def run_tasks(tasks, cpu=4):
for t in tasks:
print(t.name)
t.stdout = os.path.join(PATH_DATABASES, t.name+'.stdout')
t.stderr = os.path.join(PATH_DATABASES, t.name+'.stderr')
s = Supervisor(tasks=tasks, force_run=False, log=database_supervisor_log, cpu=cpu)
s.run()
for t in tasks:#if everything executes properly, rm the task logs
if os.path.exists(t.stdout):
os.remove(t.stdout)
if os.path.exists(t.stderr):
os.remove(t.stderr)
def run_tasks(tasks, cpu=4):
for t in tasks:
print(t.name)
t.stdout = join(PATH_TOOLS, t.name+'.stdout')
t.stderr = join(PATH_TOOLS, t.name+'.stderr')
s = Supervisor(tasks=tasks, force_run=False, log=tool_supervisor_log, cpu=cpu)
s.run()
for t in tasks: #if everything executes properly, rm the task logs
if exists(t.stdout):
os.remove(t.stdout)
if exists(t.stderr):
os.remove(t.stderr)
def run_tasks(tasks, cpu=4):
for t in tasks:
print(t.name)
t.stdout = join(PATH_TOOLS, t.name+'.stdout')
t.stderr = join(PATH_TOOLS, t.name+'.stderr')
s = Supervisor(tasks=tasks, force_run=False, log=tool_supervisor_log, cpu=cpu)
s.run()
for t in tasks: #if everything executes properly, rm the task logs
if exists(t.stdout):
os.remove(t.stdout)
if exists(t.stderr):
os.remove(t.stderr)
def run_tasks(tasks, cpu=4):
for t in tasks:
print(t.name)
t.stdout = os.path.join(PATH_DATABASES, t.name + '.stdout')
t.stderr = os.path.join(PATH_DATABASES, t.name + '.stderr')
s = Supervisor(tasks=tasks,
force_run=False,
log=database_supervisor_log,
cpu=cpu)
s.run()
for t in tasks: #if everything executes properly, rm the task logs
if os.path.exists(t.stdout):
os.remove(t.stdout)
if os.path.exists(t.stderr):
os.remove(t.stderr)
def gen_rapclust_supervisor(opc, fastq1, fastq2, paired_names, unpaired,
unpaired_names, assembly_path, assembly_name,
bowtie2_index, gene_trans_map, sample_info, model,
out_dir, cpu_cap, deps):
rc_tasks, bowtie_rc_tasks = [], []
rc_dir = fg.make_dir_task(os.path.join(out_dir, 'rapclust_bt2'))
out_dir = rc_dir.targets[0]
for i in range(len(fastq1)):
filename = paired_names[
i] #'_'.join([paired_names[i],express_naming,assembly_name])
#filename = '_'.join([paired_names[i],express_naming,assembly_name])
bowtie_rc = fex.bowtie2_task(opc, bowtie2_index, out_dir, fastq1[i],
fastq2[i], filename, 2,
fg.round_div(cpu_cap, 2), deps)
# express = fex.express_task(opc, bowtie2_index,assembly_path,out_dir,paired_names[i],bowtie_e.targets[0],[bowtie_e])
bowtie_rc_tasks.append(bowtie_rc)
# express_tasks.append(express)
for i in range(len(unpaired)):
filename = unpaired_names[
i] #'_'.join([unpaired_names[i],express_naming,assembly_name])
bowtie_rcU = fex.bowtie2_unpaired_task(opc, bowtie2_index, out_dir,
unpaired[i], filename, 2,
fg.round_div(cpu_cap, 2), deps)
bowtie_rc_tasks.append(bowtie_rcU)
# express = fex.express_task(opc, bowtie2_index,assembly_path,out_dir,unpaired_names[i],bowtie_e.targets[0],[bowtie_e])
# express_tasks.append(express)
# transcriptName = assembly_name #'_'.join([assembly_name,express_naming])
# geneName = assembly_name + '_gene' #'_'.join([assembly_name,express_naming,'gene'])
# counts_to_table_express = fex.counts_to_table_task(opc, assembly_name,gene_trans_map,out_dir,[t.targets[0] for t in express_tasks],transcriptName,'--eXpress',express_tasks)
# deseq2_express = fex.deseq2_task(opc, assembly_name,out_dir,counts_to_table_express.targets[0],sample_info,transcriptName,model,[counts_to_table_express])
# deseq2_express_gene = fex.deseq2_task(opc, assembly_name,out_dir,counts_to_table_express.targets[1],sample_info,geneName,model,[counts_to_table_express])
rc_tasks = [rc_dir] + bowtie_rc_tasks + rc_tasks
return Supervisor(tasks=rc_tasks)
def gen_db_supervisor(force=False,
sprot=False,
uniref90=False,
nr=False,
busco_args=busco_defaults,
blast_plus=False,
idmapping=False,
cpu=float('inf'),
pfam=True,
nog_functions=True,
dep=[]):
check_db_dir()
dbs = get_dbs(defaults=force)
tasks = []
if (sprot):
tasks.append(
gen_dmnd_blast_tasks(dbs['uniprot_sprot'], force, blast_plus))
if (uniref90):
tasks.append(gen_dmnd_blast_tasks(dbs['uniref90'], force, blast_plus))
if (nr):
tasks.append(gen_dmnd_blast_tasks(dbs['nr'], force, blast_plus))
for busco_db in busco_args:
if (busco_args[busco_db]):
tasks.append(download_task_wrapper(dbs['busco_' + busco_db], []))
if (pfam):
pfam_task = download_task_wrapper(dbs['pfam'], [])
hmmpress = fdb.pfam_build_task(dbs['pfam'].download_location,
dbs['pfam'].call_path, [pfam_task])
tasks.append(pfam_task)
tasks.append(hmmpress)
if (nog_functions):
nogF_task = download_task_wrapper(dbs['nog_functions'], [])
tasks.append(nogF_task)
if (idmapping):
idmap_task = download_task_wrapper(dbs['id_mapping'], [])
tasks.append(idmap_task)
tasks.append(download_task_wrapper(dbs['id_mapping_selected'], []))
tasks.append(
fdb.subset_idmapping_task(dbs['id_mapping'].download_location,
dbs['id_mapping_biocyc'].call_path,
dbs['id_mapping_eggnog'].call_path,
dbs['id_mapping_ko'].call_path,
dbs['id_mapping_orthodb'].call_path,
[idmap_task]))
special_dbs = set([
'uniprot_sprot', 'uniref90', 'nr', 'swiss_enzyme', 'orthology_pathway',
'nog_categories', 'nog_functions', 'pfam'
])
for db_string in dbs:
if (db_string in special_dbs or db_string.startswith('busco_')
or db_string.startswith('id_mapping')):
pass
else:
tasks.append(download_task_wrapper(dbs[db_string], []))
tasks = [t for t in tasks if (t is not None)]
return Supervisor(tasks, cpu=cpu)
def gen_salmon_supervisor(opc, fastq1, fastq2, paired_names, unpaired,
unpaired_names, assembly_path, assembly_name,
gene_trans_map, sample_info, model, out_dir, cpu_cap,
deps):
salmon_tasks = []
salmon_dir = fg.make_dir_task(os.path.join(out_dir, 'salmon'))
out_dir = salmon_dir.targets[0]
build_salmon = fex.build_salmon_task(opc, assembly_path,
assembly_name, out_dir,
fg.round_div(cpu_cap,
2), [salmon_dir])
deps = deps + [build_salmon] #, salmon_gene_map]
salmon_trans_gene_map = ''
if len(gene_trans_map) > 0:
salmon_gene_map = fex.salmon_gene_map_task(opc, out_dir, assembly_name,
gene_trans_map,
[salmon_dir])
salmon_trans_gene_map = salmon_gene_map.targets[0]
deps = deps + [salmon_gene_map]
for i in range(len(fastq1)):
#filename = '_'.join([paired_names[i],salmon_naming,assembly_name])
filename = paired_names[i] #,salmon_naming,assembly_name])
salmon = fex.salmon_task(opc, build_salmon.targets[0], fastq1[i],
fastq2[i], filename, salmon_trans_gene_map,
out_dir, fg.round_div(cpu_cap, 2), deps)
salmon_tasks.append(salmon)
for i in range(len(unpaired)):
#filename = '_'.join([unpaired_names[i],salmon_naming,assembly_name])
filename = unpaired_names[i] #,salmon_naming,assembly_name])
salmon = fex.salmon_unpaired_task(opc, build_salmon.targets[0],
unpaired[i], filename,
salmon_trans_gene_map, out_dir,
fg.round_div(cpu_cap, 2), deps)
salmon_tasks.append(salmon)
transcriptName = assembly_name #'_'.join([assembly_name,salmon_naming])
geneName = assembly_name + '_gene' #'_'.join([assembly_name,salmon_naming,'gene'])
counts_to_table_salmon = fex.counts_to_table_task(
opc, assembly_name, gene_trans_map, out_dir,
[t.targets[0]
for t in salmon_tasks], transcriptName, '--salmon', salmon_tasks)
deseq2_salmon = fex.deseq2_task(opc, assembly_name, out_dir,
counts_to_table_salmon.targets[0],
sample_info, transcriptName, model,
[counts_to_table_salmon])
deseq2_salmon_gene = fex.deseq2_task(opc, assembly_name, out_dir,
counts_to_table_salmon.targets[1],
sample_info, geneName, model,
[counts_to_table_salmon])
salmon_tasks = [
salmon_dir, build_salmon, salmon_gene_map, counts_to_table_salmon,
deseq2_salmon, deseq2_salmon_gene
] + salmon_tasks
return Supervisor(tasks=salmon_tasks)
def gen_paired_prinseq_supervisor(opc, out_dir, fastq1, fastq2, unpaired,
dependency_set, rmdup):
tasks = []
prinseq_count = 0
prinseq_opts = '--derep 14' if (rmdup) else ''
for input1, input2 in zip(fastq1, fastq2):
p_task = fa.prinseq_task(out_dir, input1, input2,
'prinseq_output_' + str(prinseq_count),
prinseq_opts, [])
prinseq_count += 1
tasks.append(p_task)
return Supervisor(tasks=tasks)
def gen_paired_trimmomatic_supervisor(opc, out_dir, fq1, fq2, unpaired,
dependency_set, cpu_cap):
tasks = []
count = 0
# cpu_mod = min(len(fq1),cpu_cap)
cpu_mod = int(round(float(cpu_cap) / len(fq1)))
for i1, i2 in zip(fq1, fq2):
trim_task = fa.trimmomatic_task(opc, out_dir, i1, i2, cpu_mod,
'trimmomatic_output_' + str(count),
dependency_set)
count += 1
tasks.append(trim_task)
return Supervisor(tasks=tasks)
def gen_dmnd_blast_tasks(db, force, blast_plus):
tasks = []
sprot_download = download_task_wrapper(db, [])
tasks.append(sprot_download)
install_dmnd = fdb.build_diamond_task(db.download_location, db.call_path,
[sprot_download])
tasks.append(install_dmnd)
if (blast_plus):
install_blast = fdb.build_blast_task(db.download_location,
db.call_path, 'prot',
[sprot_download])
tasks.append(install_blast)
tasks.append(fdb.db2stitle_task(db.download_location, [sprot_download]))
return Supervisor(tasks)
def gen_unpaired_trimmomatic_supervisor(opc, out_dir, fq1, fq2, unpaired,
dependency_set, cpu_cap):
# fq2 needed for unpaired trimmomatic
tasks = []
count = len(fq1)
# cpu_mod = min(len(fq1),cpu_cap)
cpu_mod = int(round(float(cpu_cap) / len(unpaired)))
for i in unpaired:
trim_task = fa.trimmomatic_unpaired_task(
opc, out_dir, i, cpu_mod, 'trimmomatic_output_' + str(count),
dependency_set)
count += 1
tasks.append(trim_task)
return Supervisor(tasks=tasks)
def gen_filter_supervisor(opc,
dbs,
main_path_assembly,
main_assembly_name,
out_dir,
transrate_task,
dependency_set,
tpm_threshold=1):
tasks = []
filter_full = fg.filter_task(main_path_assembly, main_assembly_name,
out_dir, [transrate_task.targets[2]],
tpm_threshold, 2, [transrate_task])
tasks.append(filter_full)
if transrate_task is not None:
filter_good = fg.filter_task(transrate_task.targets[1],
'good.' + main_assembly_name, out_dir,
[transrate_task.targets[2]],
tpm_threshold, 2, [transrate_task])
tasks.append(filter_good)
return Supervisor(tasks=tasks, dependencies=dependency_set)
def gen_quality_supervisor(opc, dbs, transrate_fq1, transrate_fq2, dependency_set, busco_refs, assembly_name, assembly_path, out_dir, transrate_dir, reads_dir, filter_dir, cp_transrate=True, cpu=12, cegma_flag=False, transrate_ref=''):
tasks = []
for busco_ref in busco_refs:
tasks.append(fq.busco_task(opc, dbs, assembly_path, assembly_name, out_dir, busco_ref, int(cpu/2), []))
assembly_stats = fq.assembly_stats_task(opc, out_dir,assembly_path, [])
if transrate_fq1 == None:
transrate_fq1 = []
if transrate_fq2 == None:
transrate_fq2 = []
transrate = fq.transrate_task(opc,reads_dir,assembly_path,assembly_name,transrate_fq1,transrate_fq2,out_dir,transrate_dir,int(round(float(cpu),4)),[],transrate_ref)
tasks.append(transrate)
tasks.append(assembly_stats)
if cp_transrate:
tasks.append(fg.cp_assembly_task(join(filter_dir,'good.'+assembly_name),transrate.targets[1], [transrate]))
# for busco_ref in busco_refs:
# tasks.append(fq.busco_task(transrate.targets[1], os.path.basename(transrate.targets[1]), out_dir, busco_ref, int(cpu/2), [transrate]))
if(cegma_flag):
cegma = fq.cegma_task(out_dir,assembly_path, cpu, [])
tasks.append(cegma)
return Supervisor(tasks=tasks,dependencies=dependency_set)
def gen_assembly_supervisor(opc,
dbs,
fastq1,
fastq2,
unpaired,
dependency_set,
no_trim=False,
rnaSPAdes=False,
rmdup=False,
subset_size=50000000,
cpu=12,
subset_seed='I am a seed value',
normalize_flag=False,
truncate_opt=-1,
trimmomatic_flag=True,
trinity_memory=100):
out_dir = opc.path_assembly_files
path_assembly = opc.path_assembly
tasks = []
trim_reads, fastq1, fastq2, unpaired = gen_trimming_supervisor(
opc, out_dir, fastq1, fastq2, unpaired, no_trim, trimmomatic_flag,
rmdup, subset_size, subset_seed, truncate_opt, [], cpu)
tasks.append(trim_reads)
if (rnaSPAdes):
rnaspades = fa.rnaspades_task(path_assembly, out_dir, fastq1, fastq2,
unpaired, cpu, [trim_reads])
tasks.append(rnaspades)
else:
trinity = fa.trinity_task(opc, path_assembly,
out_dir, fastq1, fastq2, unpaired, cpu,
int(cpu / 2), trinity_memory, trinity_memory,
normalize_flag, [trim_reads])
tasks.append(trinity)
gene_trans_map = fan.gene_trans_map_task(opc, path_assembly, out_dir,
[trinity])
tasks.append(gene_trans_map)
return Supervisor(tasks=tasks)
def gen_expression_supervisor(opc,
dbs,
fastq1,
fastq2,
paired_names,
unpaired,
unpaired_names,
cpu,
sample_info,
model,
gene_trans_map,
dependency_set,
assembly_name,
assembly_path,
out_dir,
run_salmon=True,
run_express=False,
run_intersectbed=False,
run_rapclust=False):
all_tasks = []
deps = []
trim_reads = False
if trim_reads:
trimmomatic_flag = True
rmdup = False
truncate_opt = False
trim_tasks, fastq1, fastq2, unpaired = assemb.gen_trimming_supervisor(
opc, out_dir, fastq1, fastq2, unpaired, False, trimmomatic_flag,
rmdup, 10**15, 0, truncate_opt, [], cpu)
all_tasks.append(trim_tasks)
deps.append(trim_tasks)
if run_salmon:
salmon_tasks = gen_salmon_supervisor(opc, fastq1, fastq2, paired_names,
unpaired, unpaired_names,
assembly_path, assembly_name,
gene_trans_map, sample_info,
model, out_dir, cpu, deps)
all_tasks.append(salmon_tasks)
if run_express or run_intersectbed or run_rapclust:
build_bowtie = fex.build_bowtie_task(opc, assembly_path, assembly_name,
out_dir, [])
bowtie2_index = join(dirname(build_bowtie.targets[0]),
basename(build_bowtie.targets[0]).split('.')[0])
all_tasks.append(build_bowtie)
if run_express:
express_tasks = gen_express_supervisor(
opc, fastq1, fastq2, paired_names, unpaired, unpaired_names,
assembly_path, assembly_name, bowtie2_index, gene_trans_map,
sample_info, model, out_dir, cpu, [build_bowtie])
all_tasks.append(express_tasks)
if run_rapclust:
rc_tsks = gen_rapclust_supervisor(
opc, fastq1, fastq2, paired_names, unpaired, unpaired_names,
assembly_path, assembly_name, bowtie2_index, gene_trans_map,
sample_info, model, out_dir, cpu, [build_bowtie])
all_tasks.append(rc_tsks)
if run_intersectbed:
intersect_tasks = gen_intersect_supervisor(
opc, fastq1, fastq2, paired_names, unpaired, unpaired_names,
assembly_path, assembly_name, bowtie2_index, gene_trans_map,
sample_info, model, out_dir, cpu, [build_bowtie])
all_tasks.append(intersect_tasks)
return Supervisor(tasks=all_tasks, dependencies=dependency_set)
def gen_intersect_supervisor(opc, fq1, fq2, paired_names, unpaired,
unpaired_names, assembly_path, assembly_name,
bowtie2_index, gene_trans_map, sample_info, model,
out_dir, cpu_cap, deps):
intersect_tasks, bowtie_i_tasks, sam_sort_tasks = [], [], []
intersect_dir = fg.make_dir_task(os.path.join(out_dir, 'intersectBed'))
out_dir = intersect_dir.targets[0]
deps.append(intersect_dir)
fasta_to_bed = fan.assembly_to_bed_task(opc, assembly_path, out_dir,
[intersect_dir])
for i in range(len(fq1)):
filename = paired_names[
i] #'_'.join([paired_names[i],intersect_naming,assembly_name])
#filename = '_'.join([paired_names[i],intersect_naming,assembly_name])
bowtie_i = fex.bowtie2_task(opc, bowtie2_index, out_dir, fq1[i],
fq2[i], filename, 1,
fg.round_div(cpu_cap, 2), deps)
sorted_name = filename + '_sorted'
sam_sort = fex.sam_sort_task(opc, out_dir, bowtie_i.targets[0],
sorted_name, [bowtie_i])
intersect_bed = fex.intersect_bed_task(opc, out_dir,
sam_sort.targets[0],
fasta_to_bed.targets[0],
paired_names[i],
[sam_sort, fasta_to_bed])
bowtie_i_tasks.append(bowtie_i)
sam_sort_tasks.append(sam_sort)
intersect_tasks.append(intersect_bed)
for i in range(len(unpaired)):
filename = unpaired_names[
i] #'_'.join([unpaired_names[i],intersect_naming,assembly_name])
bowtie_i = fex.bowtie2_unpaired_task(opc, bowtie2_index, out_dir,
unpaired[i], filename, 1,
fg.round_div(cpu_cap, 2), deps)
bowtie_i_tasks.append(bowtie_i)
sorted_name = filename + '_sorted'
sam_sort = fex.sam_sort_task(opc, out_dir, bowtie_i.targets[0],
sorted_name, [bowtie_i])
sam_sort_tasks.append(sam_sort)
intersect_bed = fex.intersect_bed_task(opc, out_dir,
sam_sort.targets[0],
fasta_to_bed.targets[0],
unpaired_names[i],
[sam_sort, fasta_to_bed])
intersect_tasks.append(intersect_bed)
transcriptName = assembly_name #'_'.join([assembly_name,express_naming])
geneName = assembly_name + '_gene' #'_'.join([assembly_name,express_naming,'gene'])
counts_to_table_intersect = fex.counts_to_table_task(
opc, assembly_name, gene_trans_map, out_dir,
[t.targets[0] for t in intersect_tasks], transcriptName, '',
intersect_tasks)
deseq2_intersect = fex.deseq2_task(opc, assembly_name, out_dir,
counts_to_table_intersect.targets[0],
sample_info, transcriptName, model,
[counts_to_table_intersect])
deseq2_intersect_gene = fex.deseq2_task(
opc, assembly_name, out_dir, counts_to_table_intersect.targets[1],
sample_info, geneName, model, [counts_to_table_intersect])
i_tasks = [
intersect_dir, fasta_to_bed, counts_to_table_intersect,
deseq2_intersect, deseq2_intersect_gene
] + bowtie_i_tasks + sam_sort_tasks + intersect_tasks
return Supervisor(tasks=i_tasks)
def gen_annotation_supervisor(opc,
dbs,
cpu,
uniref90_flag,
nr_flag,
blast_flag,
signalp_flag,
tmhmm_flag,
rnammer_flag,
dependency_set,
gene_trans_map,
path_assembly,
assembly_name,
out_dir,
improve_orfs=False):
tasks = []
annot_table_opts = {'geneTransMap': gene_trans_map}
gff3_dependencies = []
gff3_opts = {}
def task_insert(task, name=None, index=0, gff3_flag=False):
tasks.append(task)
if (name != None):
annot_table_opts[name] = task.targets[index]
if (gff3_flag):
gff3_dependencies.append(task)
gff3_opts[name] = task.targets[index]
annot_table_opts['geneTransMap'] = gene_trans_map
transd_dir = os.path.join(out_dir, 'transdecoder')
longorfs = fan.transdecoder_longorfs_task(opc, path_assembly, transd_dir,
cpumod(cpu, 2), [])
tasks.append(longorfs)
if improve_orfs:
blastp_transd = fan.blast_task(opc, 'blastp', transd_dir,
longorfs.targets[0],
dbs['uniprot_sprot'].call_path,
int(cpu / 2), [longorfs])
pfam_transd = fan.pfam_task(opc, dbs, longorfs.targets[0], transd_dir,
cpumod(cpu, 2), [longorfs])
tasks.extend([blastp_transd, pfam_transd])
predict_orfs = fan.transdecoder_predict_orfs_task(
opc, path_assembly, transd_dir,
[longorfs, pfam_transd, blastp_transd], pfam_transd.targets[0],
blastp_transd.targets[0])
else:
predict_orfs = fan.transdecoder_predict_orfs_task(
opc, path_assembly, transd_dir, [longorfs])
gff3_dependencies.append(predict_orfs)
gff3_opts['transdecoder_gff3'] = predict_orfs.targets[2]
task_insert(predict_orfs, 'transdecoder', 1)
pfam = fan.pfam_task(opc, dbs, predict_orfs.targets[0], out_dir,
cpumod(cpu, 4), [predict_orfs])
#pfam = fan.pfam_task(predict_orfs.targets[0], out_dir,cpu, [predict_orfs])
task_insert(pfam, 'pfam', gff3_flag=True)
if (blast_flag):
blastx_sprot = fan.blast_task(opc, 'blastx', out_dir, path_assembly,
dbs['uniprot_sprot'].call_path,
cpumod(cpu, 2), [])
task_insert(blastx_sprot, 'spX', gff3_flag=True)
blastp_sprot = fan.blast_task(opc, 'blastp', out_dir,
predict_orfs.targets[0],
dbs['uniprot_sprot'].call_path,
cpumod(cpu, 2), [predict_orfs])
task_insert(blastp_sprot, 'spP', gff3_flag=True)
if (uniref90_flag):
blastx_ur90 = fan.blast_task(opc, 'blastx', out_dir, path_assembly,
dbs['uniref90'].call_path,
cpumod(cpu, 2), [])
task_insert(blastx_ur90, 'ur90X', gff3_flag=True)
blastp_ur90 = fan.blast_task(opc, 'blastp', out_dir,
predict_orfs.targets[0],
dbs['uniref90'].call_path,
cpumod(cpu, 2), [predict_orfs])
task_insert(blastp_ur90, 'ur90P', gff3_flag=True)
if (nr_flag):
blastx_nr = fan.blast_task(opc, 'blastx', out_dir, path_assembly,
dbs['nr'].call_path, cpumod(cpu, 2), [])
task_insert(blastx_nr, 'nrX', gff3_flag=True)
blastp_nr = fan.blast_task(opc, 'blastp', out_dir,
predict_orfs.targets[0],
dbs['nr'].call_path, cpumod(cpu, 2),
[predict_orfs])
task_insert(blastp_nr, 'nrP', gff3_flag=True)
else:
dmnd_dependencies = []
def dmnd_task_insert(task, name=None):
dmnd_dependencies.append(task)
task_insert(task, name)
dmnd_xsprot = fan.diamond_task(opc, 'blastx', out_dir, path_assembly,
dbs['uniprot_sprot'].call_path,
cpumod(cpu, 2), dmnd_dependencies[:])
dmnd_task_insert(dmnd_xsprot)
expand = fan.blast_augment_task(opc, dbs['uniprot_sprot'].call_path,
dmnd_xsprot.targets[0], [dmnd_xsprot])
task_insert(expand, 'spX', gff3_flag=True)
dmnd_psprot = fan.diamond_task(opc, 'blastp', out_dir,
predict_orfs.targets[0],
dbs['uniprot_sprot'].call_path,
cpumod(cpu, 2),
dmnd_dependencies + [predict_orfs])
dmnd_task_insert(dmnd_psprot)
expand = fan.blast_augment_task(opc, dbs['uniprot_sprot'].call_path,
dmnd_psprot.targets[0], [dmnd_psprot])
task_insert(expand, 'spP', gff3_flag=True)
if (uniref90_flag):
dmnd_xur90 = fan.diamond_task(opc, 'blastx', out_dir,
path_assembly,
dbs['uniref90'].call_path,
cpumod(cpu, 2), dmnd_dependencies[:])
dmnd_task_insert(dmnd_xur90)
expand = fan.blast_augment_task(opc, dbs['uniref90'].call_path,
dmnd_xur90.targets[0],
[dmnd_xur90])
task_insert(expand, 'ur90X', gff3_flag=True)
dmnd_pur90 = fan.diamond_task(opc, 'blastp', out_dir,
predict_orfs.targets[0],
dbs['uniref90'].call_path,
cpumod(cpu, 2),
dmnd_dependencies + [predict_orfs])
dmnd_task_insert(dmnd_pur90)
expand = fan.blast_augment_task(opc, dbs['uniref90'].call_path,
dmnd_pur90.targets[0],
[dmnd_pur90])
task_insert(expand, 'ur90P', gff3_flag=True)
if (nr_flag):
dmnd_xnr = fan.diamond_task(opc, 'blastx', out_dir,
path_assembly, dbs['nr'].call_path,
cpumod(cpu, 2), dmnd_dependencies[:])
dmnd_task_insert(dmnd_xnr)
expand = fan.blast_augment_task(opc, dbs['nr'].call_path,
dmnd_xnr.targets[0], [dmnd_xnr])
task_insert(expand, 'nrX', gff3_flag=True)
dmnd_pnr = fan.diamond_task(opc, 'blastp', out_dir,
predict_orfs.targets[0],
dbs['nr'].call_path, cpumod(cpu, 2),
dmnd_dependencies + [predict_orfs])
dmnd_task_insert(dmnd_pnr)
expand = fan.blast_augment_task(opc, dbs['nr'].call_path,
dmnd_pnr.targets[0], [dmnd_pnr])
task_insert(expand, 'nrP', gff3_flag=True)
if (tmhmm_flag):
tmhmm = fan.tmhmm_task(opc, predict_orfs.targets[0], out_dir,
[predict_orfs])
task_insert(tmhmm, 'tmhmm')
if (signalp_flag):
signalp = fan.signalp_task(opc, predict_orfs.targets[0], out_dir,
[predict_orfs])
task_insert(signalp, 'signalP')
# need more intelligent annot table -- if pfam fails, for example, we can still generate an annot table
annot = fan.annot_table_task(opc, dbs, path_assembly, out_dir,
annot_table_opts, tasks[:])
tasks.append(annot)
gff3_output = os.path.join(opc.path_dir, opc.assembly_name + '.gff3')
gff3 = fan.gff3_task(opc, path_assembly, gff3_output, gff3_opts,
gff3_dependencies)
tasks.append(gff3)
pipeplot = fan.pipeplot_task(opc, dbs, annot.targets[0], out_dir, [annot])
tasks.append(pipeplot)
kegg = fan.kegg_task(opc, annot.targets[0], out_dir, [annot])
tasks.append(kegg)
return Supervisor(tasks=tasks, dependencies=dependency_set)
def gen_trimming_supervisor(opc, out_dir, fq1, fq2, unpaired, no_trim,
trimmomatic_flag, rmdup, subset_size, subset_seed,
truncate_opt, dependency_set, cpu_cap):
tasks = []
deps = []
if (not no_trim):
tasks.append(
fa.fastqc_task(opc, opc.path_assembly_files, fq1 + fq2 + unpaired,
'pre_trimming',
min(cpu_cap, len(fq1 + fq2 + unpaired)), []))
if (fq1 != []):
if (trimmomatic_flag):
paired_sup = gen_paired_trimmomatic_supervisor(
opc, out_dir, fq1, fq2, unpaired, dependency_set, cpu_cap)
else:
paired_sup = gen_paired_prinseq_supervisor(
opc, out_dir, fq1, fq2, unpaired, dependency_set, rmdup)
fq1 = [
paired_sup.targets[x]
for x in range(0, len(paired_sup.targets), 2)
]
fq2 = [
paired_sup.targets[x]
for x in range(1, len(paired_sup.targets), 2)
]
tasks.append(paired_sup)
tasks.append(
fa.fastqc_task(opc, opc.path_assembly_files, fq1 + fq2,
'post_trimming_paired',
int(round(float(cpu_cap) / 2)), [paired_sup]))
deps.append(paired_sup)
if (unpaired != []):
if (trimmomatic_flag):
unpaired_sup = gen_unpaired_trimmomatic_supervisor(
opc, out_dir, fq1, fq2, unpaired, dependency_set, cpu_cap)
else:
unpaired_sup = gen_unpaired_prinseq_supervisor(
opc, out_dir, fq1, fq2, unpaired, dependency_set, rmdup)
unpaired = unpaired_sup.targets
tasks.append(unpaired_sup)
tasks.append(
fa.fastqc_task(opc, opc.path_assembly_files, unpaired,
'post_trimming_unpaired',
int(round(float(cpu_cap) / 2)), [unpaired_sup]))
deps.append(unpaired_sup)
# need to add support for unp here
#if len(fq2) <1:
#subset = fa.subset_task(out_dir, fq1,fq2,unpaired, 'final_reads', subset_size, subset_seed, deps)
#unpaired = [subset.targets[0]]
#else:
if fq1 != []:
subset = fa.subset_task(opc, out_dir, fq1, fq2, 'final_reads',
subset_size, subset_seed, deps)
fq1 = [subset.targets[0]]
fq2 = [subset.targets[1]]
tasks.append(subset)
if (truncate_opt >= 0):
truncate = fa.truncate_task(out_dir, fastq1[0], fastq2[0],
truncate_opt, [subset])
fq1 = [truncate.targets[0]]
fq2 = [truncate.targets[1]]
deps.append(truncate)
tasks.append(truncate)
late_fastqc = fa.fastqc_task(opc, out_dir, fq1 + fq2 + unpaired,
'final_reads_paired', cpu_cap, deps)
tasks.append(late_fastqc)
return (Supervisor(tasks=tasks,
dependencies=dependency_set), fq1, fq2, unpaired)