mmetsp_data={}

with open(thefile,"rU") as inputfile:

headerline=next(inputfile).split(',')

#print headerline

position_name=headerline.index("ScientificName")

position_reads=headerline.index("Run")

position_mmetsp=headerline.index("SampleName")

for line in inputfile:

line_data=line.split(',')

name="_".join(line_data[position_name].split())

read_type=line_data[position_reads]

mmetsp=line_data[position_mmetsp]

test_mmetsp=mmetsp.split("_")

if len(test_mmetsp)>1:

print test_mmetsp

mmetsp=test_mmetsp[0]

name_read_tuple=(name,read_type)

#print name_read_tuple

#check to see if Scientific Name and run exist

if name_read_tuple in mmetsp_data.keys():

#check to see if ftp exists

if mmetsp in mmetsp_data[name_read_tuple]:

print "mmetsp ID already exists:", mmetsp

else:

mmetsp_data[name_read_tuple].append(mmetsp)

else:

mmetsp_data[name_read_tuple] = [mmetsp]

#os.chdir(trinity_dir)

trinity_out=trinity_dir+sample+".Trinity.fixed.fa"

fix="""

sed 's_|_-_g' {} > {}

""".format(trinity_fasta,trinity_out)

#s=subprocess.Popen(fix,shell=True)

print fix

#s.wait()

#os.chdir("/home/ubuntu/MMETSP/")

mmetsp_assembly_out=mmetsp_assembly_dir+mmetsp_assembly+".fixed.fa"

fix="""

sed 's_|_-_g' {}{} > {}

""".format(mmetsp_assembly_dir,mmetsp_assembly,mmetsp_assembly_out)

print fix

#s=subprocess.Popen(fix,shell=True)

#s.wait()

transrate_command="""

transrate -o {}{} \\

--assembly {} \\

--reference {} \\

--threads 4

""".format(transrate_dir,sample,trinity_fasta,filename)

print transrate_command

#s=subprocess.Popen(transrate_command,shell=True)

transrate_command="""

transrate -o {}{} \\

--assembly {} \\

--reference {} \\

--threads 4

""".format(transrate_dir,sample,filename,trinity_fasta)

print "This is the reverse transrate command:"

print transrate_command

#s=subprocess.Popen(transrate_command,shell=True)

#columns=["n_bases","gc","gc_skew","mean_orf_percent"]

frames=[data_frame,transrate_data]

data_frame=pd.concat(frames)

trinity_fail=[]

# construct an empty pandas dataframe to add on each assembly.csv to

for item in mmetsp_data.keys():

#print item

organism=item[0]

sample="_".join(item)

org_seq_dir=basedir+organism+"/"

mmetsp_list=mmetsp_data[item]

for mmetsp in mmetsp_list:

print mmetsp

assemblyfileslist=os.listdir(mmetsp_assemblies)

for filename in assemblyfileslist:

if filename.startswith(mmetsp):

if filename.endswith(".fixed.fa"):

print "This is not the one you want."

else:

print "MMETSP assembly found:",filename

reference_filename=filename

sra=item[1]

newdir=org_seq_dir+sra+"/"

trinitydir=newdir+"trinity/trinity_out/"

dammit_dir=trinitydir+"dammit_dir/"

transrate_dir="/mnt/comparisons/"

reverse_transrate_dir="/mnt/comparisons_reverse/"

clusterfunc.check_dir(transrate_dir)

clusterfunc.check_dir(dammit_dir)

clusterfunc.check_dir(reverse_transrate_dir)

#trinity_fasta=dammit_dir+"Trinity.fasta.dammit.fasta"

trinity_fasta=trinitydir+"Trinity.fasta"

if os.path.isfile(trinity_fasta):

print trinity_fasta

fixed_trinity=fix_fasta(trinity_fasta,trinitydir,sample)

fixed_mmetsp_ref=fix_fasta_reference(reference_filename,mmetsp_assemblies)

#transrate(transrate_dir,fixed_trinity,mmetsp_assemblies,fixed_mmetsp_ref)

transrate_reverse(reverse_transrate_dir,sample,fixed_trinity,mmetsp_assemblies_dir,fixed_mmetsp_ref)

else:

print "Trinity failed:",newdir

trinity_fail.append(newdir)

print "This is the number of times Trinity failed:"

print len(trinity_fail)

listofassemblies=os.listdir(extra_dir)

for assembly_fasta in listofassemblies:

sample_info=assembly_fasta.split("_")

sra=sample_info[2].split(".")[0]

sample_info=sample_info[:-1]

sample_info.append(sra)

sample="_".join(sample_info)

print sample

transrate_out=extra_dir+sample+"/"

fixed_trinity=fix_fasta(assembly_fasta,extra_dir,sample)

transrate(extra_dir,transrate_out,fixed_trinity)

transrate_assemblies=transrate_out+"assemblies.csv"

data=parse_transrate_stats(transrate_assemblies)

data_frame=build_DataFrame(data_frame,data)

datafile="/home/ubuntu/MMETSP/MMETSP_SRA_Run_Info_subset2.csv"

mmetsp_data=get_data(datafile)

print mmetsp_data

listofassemblies=os.listdir(extra_dir)

for filename in listofassemblies:

if filename.endswith("Trinity.dammit.fasta"):

print filename

assembly_filename=extra_dir+filename

file_info=filename.split("_")

sra_info=file_info[2].split(".")

sra=sra_info[0]

organism=file_info[0]+"_"+file_info[1]

org_sra=(organism,sra)

sample=organism+"_"+sra+".dammit"

print sample

fixed_trinity=fix_fasta(assembly_filename,extra_dir,sample)

mmetsp_list=mmetsp_data[org_sra]

mmetsp=mmetsp_list[0]

assemblyfileslist=os.listdir(mmetsp_assemblies)

for ref_filename in assemblyfileslist:

if ref_filename.startswith(mmetsp):

if ref_filename.endswith(".fixed.fa"):

print "This is not the one you want."

else:

print "MMETSP assembly found:",ref_filename

fixed_reference=fix_fasta_reference(ref_filename,mmetsp_assemblies)

reference_filename=fixed_reference

#transrate(transrate_dir,sample,fixed_trinity,mmetsp_assemblies,reference_filename)

listofdirs=os.listdir(transrate_dir)

for dirname in listofdirs:

transrate_dirname=transrate_dir+dirname+"/"

transrate_dirnames=os.listdir(transrate_dirname)

for dirname2 in transrate_dirnames:

if dirname2.endswith(".fixed"):

transrate_contigs=transrate_dirname+dirname2+"/contigs.csv"

if os.path.isfile(transrate_contigs):

print transrate_contigs

data=parse_transrate_stats(transrate_contigs)

data_frame=build_DataFrame(data_frame,data)

else:

print "File missing:",transrate_contigs

listofdirs=os.listdir(transrate_dir)

for dirname in listofdirs:

transrate_assemblies=transrate_dir+dirname+"/assemblies.csv"

print transrate_assemblies

if os.path.isfile(transrate_assemblies):

data=parse_transrate_stats(transrate_assemblies)

data_frame=build_DataFrame(data_frame,data)

else:

print "File missing:",transrate_assemblies

listdirs=os.listdir(transrate_dir)

print listdirs

for dirname in listdirs:

newdir=transrate_dir+dirname+"/"

print newdir

newfile=newdir+"assemblies.csv"

if os.path.isfile(newfile):

print "Exists:",newfile

else: