def get_read_hctg_map(asm_dir, hasm_dir, read_to_contig_map_fn):
wf = PypeProcWatcherWorkflow(
max_jobs=
12, # TODO: Why was NumThreads ever set? There is only one task!
)
rawread_id_file = makePypeLocalFile(
os.path.join(asm_dir, 'read_maps/dump_rawread_ids/rawread_ids'))
pread_id_file = makePypeLocalFile(
os.path.join(asm_dir, 'read_maps/dump_pread_ids/pread_ids'))
h_ctg_edges = makePypeLocalFile(os.path.join(hasm_dir, 'all_h_ctg_edges'))
p_ctg_edges = makePypeLocalFile(os.path.join(hasm_dir, 'all_p_ctg_edges'))
h_ctg_ids = makePypeLocalFile(os.path.join(hasm_dir, "all_h_ctg_ids"))
#make_dirs(os.path.dirname(os.path.abspath(read_to_contig_map_fn)) # Workflow does this.
read_to_contig_map_plf = makePypeLocalFile(read_to_contig_map_fn)
inputs = {
'rawread_id_file': rawread_id_file,
'pread_id_file': pread_id_file,
'h_ctg_edges': h_ctg_edges,
'p_ctg_edges': p_ctg_edges,
'h_ctg_ids': h_ctg_ids
}
make_task = PypeTask(
inputs=inputs,
outputs={'read_to_contig_map': read_to_contig_map_plf},
)
wf.addTask(make_task(generate_read_to_hctg_map))
wf.refreshTargets() # block
def get_read_ctg_map(rawread_dir, pread_dir, asm_dir):
read_map_dir = os.path.abspath(os.path.join(asm_dir, 'read_maps'))
make_dirs(read_map_dir)
wf = PypeProcWatcherWorkflow(max_jobs=12, )
"""
job_type=config['job_type'],
job_queue=config['job_queue'],
sge_option=config.get('sge_option', ''),
watcher_type=config['pwatcher_type'],
watcher_directory=config['pwatcher_directory'])
"""
rawread_db = makePypeLocalFile(os.path.join(rawread_dir, 'raw_reads.db'))
rawread_id_file = makePypeLocalFile(
os.path.join(read_map_dir, 'dump_rawread_ids', 'rawread_ids'))
task = PypeTask(inputs={'rawread_db': rawread_db},
outputs={'rawread_id_file': rawread_id_file},
TaskType=PypeThreadTaskBase,
URL='task://localhost/dump_rawread_ids')
wf.addTask(task(dump_rawread_ids))
pread_db = makePypeLocalFile(os.path.join(pread_dir, 'preads.db'))
pread_id_file = makePypeLocalFile(
os.path.join(read_map_dir, 'dump_pread_ids', 'pread_ids'))
task = PypeTask(inputs={'pread_db': pread_db},
outputs={'pread_id_file': pread_id_file},
TaskType=PypeThreadTaskBase,
URL='task://localhost/dump_pread_ids')
wf.addTask(task(dump_pread_ids))
wf.refreshTargets() # block
sg_edges_list = makePypeLocalFile(os.path.join(asm_dir, 'sg_edges_list'))
utg_data = makePypeLocalFile(os.path.join(asm_dir, 'utg_data'))
ctg_paths = makePypeLocalFile(os.path.join(asm_dir, 'ctg_paths'))
inputs = {
'rawread_id_file': rawread_id_file,
'pread_id_file': pread_id_file,
'sg_edges_list': sg_edges_list,
'utg_data': utg_data,
'ctg_paths': ctg_paths
}
read_to_contig_map = makePypeLocalFile(
os.path.join(read_map_dir, 'get_ctg_read_map', 'read_to_contig_map'))
task = PypeTask(inputs=inputs,
outputs={'read_to_contig_map': read_to_contig_map},
TaskType=PypeThreadTaskBase,
URL='task://localhost/get_ctg_read_map')
wf.addTask(task(generate_read_to_ctg_map))
wf.refreshTargets() # block
def get_read_ctg_map(rawread_dir, pread_dir, asm_dir):
read_map_dir = os.path.abspath(os.path.join(asm_dir, 'read_maps'))
make_dirs(read_map_dir)
wf = PypeProcWatcherWorkflow(
max_jobs=12,
)
"""
job_type=config['job_type'],
job_queue=config['job_queue'],
sge_option=config.get('sge_option', ''),
watcher_type=config['pwatcher_type'],
watcher_directory=config['pwatcher_directory'])
"""
rawread_db = makePypeLocalFile(os.path.join(rawread_dir, 'raw_reads.db'))
rawread_id_file = makePypeLocalFile(os.path.join(
read_map_dir, 'dump_rawread_ids', 'rawread_ids'))
task = PypeTask(
inputs={'rawread_db': rawread_db},
outputs={'rawread_id_file': rawread_id_file},
)
wf.addTask(task(pype_tasks.task_dump_rawread_ids))
pread_db = makePypeLocalFile(os.path.join(pread_dir, 'preads.db'))
pread_id_file = makePypeLocalFile(os.path.join(
read_map_dir, 'dump_pread_ids', 'pread_ids'))
task = PypeTask(
inputs={'pread_db': pread_db},
outputs={'pread_id_file': pread_id_file},
)
wf.addTask(task(pype_tasks.task_dump_pread_ids))
wf.refreshTargets() # block
sg_edges_list = makePypeLocalFile(os.path.join(asm_dir, 'sg_edges_list'))
utg_data = makePypeLocalFile(os.path.join(asm_dir, 'utg_data'))
ctg_paths = makePypeLocalFile(os.path.join(asm_dir, 'ctg_paths'))
inputs = {'rawread_id_file': rawread_id_file,
'pread_id_file': pread_id_file,
'sg_edges_list': sg_edges_list,
'utg_data': utg_data,
'ctg_paths': ctg_paths}
read_to_contig_map = makePypeLocalFile(os.path.join(
read_map_dir, 'get_ctg_read_map', 'read_to_contig_map'))
task = PypeTask(
inputs=inputs,
outputs={'read_to_contig_map': read_to_contig_map},
)
wf.addTask(task(pype_tasks.task_generate_read_to_ctg_map))
wf.refreshTargets() # block
def main(argv=sys.argv):
global LOG
LOG = support.setup_logger(None)
if len(sys.argv) < 2:
print >> sys.stderr, 'you need to provide a configuration file to specific a couple cluster running environment'
sys.exit(1)
config_fn = sys.argv[1]
config_absbasedir = os.path.dirname(os.path.abspath(config_fn))
config = ConfigParser.ConfigParser()
config.read(config_fn)
job_type = 'SGE'
if config.has_option('General', 'job_type'):
job_type = config.get('General', 'job_type')
sge_track_reads = ' -pe smp 12 -q bigmem'
if config.has_option('Unzip', 'sge_track_reads'):
sge_track_reads = config.get('Unzip', 'sge_track_reads')
sge_quiver = ' -pe smp 24 -q bigmem '
if config.has_option('Unzip', 'sge_quiver'):
sge_quiver = config.get('Unzip', 'sge_quiver')
smrt_bin = '/mnt/secondary/builds/full/3.0.0/prod/smrtanalysis_3.0.0.153854/smrtcmds/bin/'
if config.has_option('Unzip', 'smrt_bin'):
smrt_bin = config.get('Unzip', 'smrt_bin')
input_bam_fofn = 'input_bam.fofn'
if config.has_option('Unzip', 'input_bam_fofn'):
input_bam_fofn = config.get('Unzip', 'input_bam_fofn')
if not os.path.isabs(input_bam_fofn):
input_bam_fofn = os.path.join(config_absbasedir, input_bam_fofn)
quiver_concurrent_jobs = 8
if config.has_option('Unzip', 'quiver_concurrent_jobs'):
quiver_concurrent_jobs = config.getint('Unzip',
'quiver_concurrent_jobs')
config = {
'job_type': job_type,
'sge_quiver': sge_quiver,
'sge_track_reads': sge_track_reads,
'input_bam_fofn': input_bam_fofn,
'smrt_bin': smrt_bin
}
LOG.info('config={}'.format(pprint.pformat(config)))
#support.job_type = 'SGE' #tmp hack until we have a configuration parser
wf = PypeProcWatcherWorkflow(max_jobs=quiver_concurrent_jobs, )
abscwd = os.path.abspath('.')
parameters = {
'wd': os.path.join(abscwd, '4-quiver', 'track_reads_h'),
'config': config
}
hasm_done_plf = makePypeLocalFile(
'./3-unzip/1-hasm/hasm_done') # by convention
track_reads_h_done_plf = makePypeLocalFile(
os.path.join(parameters['wd'], 'track_reads_h_done'))
make_track_reads_task = PypeTask(
inputs={'hasm_done': hasm_done_plf},
outputs={'job_done': track_reads_h_done_plf},
parameters=parameters,
)
track_reads_task = make_track_reads_task(task_track_reads)
#sge_track_reads = config['sge_track_reads']
wf.addTask(track_reads_task)
scattered_quiver_plf = makePypeLocalFile(
'4-quiver/quiver_scatter/scattered.json')
make_task = PypeTask(
inputs={
'p_ctg_fa': makePypeLocalFile('3-unzip/all_p_ctg.fa'),
'h_ctg_fa': makePypeLocalFile('3-unzip/all_h_ctg.fa'),
'track_reads_h_done': track_reads_h_done_plf,
},
outputs={
'scattered_quiver_json': scattered_quiver_plf,
},
parameters={},
)
wf.addTask(make_task(task_scatter_quiver))
wf.refreshTargets()
p_ctg_out, h_ctg_out, job_done_plfs = create_quiver_jobs(
scattered_quiver_plf)
gathered_p_ctg_plf = makePypeLocalFile('4-quiver/cns_gather/p_ctg.txt')
gathered_h_ctg_plf = makePypeLocalFile('4-quiver/cns_gather/h_ctg.txt')
gather_done_plf = makePypeLocalFile('4-quiver/cns_gather/job_done')
mkdir('4-quiver/cns_gather')
with open(fn(gathered_p_ctg_plf), 'w') as ifs:
for cns_fasta_fn, cns_fastq_fn in sorted(p_ctg_out):
ifs.write('{} {}\n'.format(cns_fasta_fn, cns_fastq_fn))
with open(fn(gathered_h_ctg_plf), 'w') as ifs:
for cns_fasta_fn, cns_fastq_fn in sorted(h_ctg_out):
ifs.write('{} {}\n'.format(cns_fasta_fn, cns_fastq_fn))
make_task = PypeTask(
inputs=job_done_plfs,
outputs={
'job_done': gather_done_plf,
},
parameters={},
)
wf.addTask(make_task(task_gather_quiver))
wf.refreshTargets()
cns_p_ctg_fasta_plf = makePypeLocalFile(
'4-quiver/cns_output/cns_p_ctg.fasta')
cns_p_ctg_fastq_plf = makePypeLocalFile(
'4-quiver/cns_output/cns_p_ctg.fastq')
cns_h_ctg_fasta_plf = makePypeLocalFile(
'4-quiver/cns_output/cns_h_ctg.fasta')
cns_h_ctg_fastq_plf = makePypeLocalFile(
'4-quiver/cns_output/cns_h_ctg.fastq')
zcat_done_plf = makePypeLocalFile('4-quiver/cns_output/job_done')
make_task = PypeTask(
inputs={
'gathered_p_ctg': gathered_p_ctg_plf,
'gathered_h_ctg': gathered_h_ctg_plf,
'gather_done': gather_done_plf,
},
outputs={
'cns_p_ctg_fasta': cns_p_ctg_fasta_plf,
'cns_p_ctg_fastq': cns_p_ctg_fastq_plf,
'cns_h_ctg_fasta': cns_h_ctg_fasta_plf,
'cns_h_ctg_fastq': cns_h_ctg_fastq_plf,
'job_done': zcat_done_plf,
},
)
wf.addTask(make_task(task_cns_zcat))
wf.refreshTargets()
def unzip_all(config):
unzip_blasr_concurrent_jobs = config['unzip_blasr_concurrent_jobs']
unzip_phasing_concurrent_jobs = config['unzip_phasing_concurrent_jobs']
wf = PypeProcWatcherWorkflow(
max_jobs=unzip_blasr_concurrent_jobs,
job_type=config['job_type'],
job_queue=config.get('job_queue'),
sge_option=config.get('sge_option'),
watcher_type=config.get('pwatcher_type'),
#watcher_directory=config.get('pwatcher_directory', 'mypwatcher'),
use_tmpdir=config.get('use_tmpdir'),
)
ctg_list_file = makePypeLocalFile('./3-unzip/reads/ctg_list')
falcon_asm_done = makePypeLocalFile('./2-asm-falcon/falcon_asm_done')
wdir = os.path.abspath('./3-unzip/reads')
parameters = {
'wd': wdir,
'config': config,
'sge_option': config['sge_track_reads'],
}
job_done = makePypeLocalFile(
os.path.join(parameters['wd'], 'track_reads_done'))
make_track_reads_task = PypeTask(
inputs={'falcon_asm_done': falcon_asm_done},
outputs={
'job_done': job_done,
'ctg_list_file': ctg_list_file
},
parameters=parameters,
wdir=wdir,
)
track_reads_task = make_track_reads_task(task_track_reads)
wf.addTask(track_reads_task)
wf.refreshTargets() #force refresh now, will put proper dependence later
ctg_ids = []
with open('./3-unzip/reads/ctg_list') as f:
for row in f:
row = row.strip()
ctg_ids.append(row)
aln1_outs = {}
all_ctg_out = {}
for ctg_id in ctg_ids:
# inputs
ref_fasta = makePypeLocalFile(
'./3-unzip/reads/{ctg_id}_ref.fa'.format(ctg_id=ctg_id))
read_fasta = makePypeLocalFile(
'./3-unzip/reads/{ctg_id}_reads.fa'.format(ctg_id=ctg_id))
# outputs
wd = os.path.join(
os.getcwd(), './3-unzip/0-phasing/{ctg_id}/'.format(ctg_id=ctg_id))
#mkdir(wd)
blasr_dir = os.path.join(wd, 'blasr')
ctg_aln_out = makePypeLocalFile(
os.path.join(blasr_dir,
'{ctg_id}_sorted.bam'.format(ctg_id=ctg_id)))
job_done = makePypeLocalFile(
os.path.join(blasr_dir, 'aln_{ctg_id}_done'.format(ctg_id=ctg_id)))
parameters = {
'job_uid': 'aln-' + ctg_id,
'wd': blasr_dir,
'config': config,
'ctg_id': ctg_id,
'sge_option': config['sge_blasr_aln'],
}
make_blasr_task = PypeTask(
inputs={
'ref_fasta': ref_fasta,
'read_fasta': read_fasta
},
outputs={
'ctg_aln_out': ctg_aln_out,
'job_done': job_done
},
parameters=parameters,
)
blasr_task = make_blasr_task(task_run_blasr)
aln1_outs[ctg_id] = (ctg_aln_out, job_done)
wf.addTask(blasr_task)
wf.refreshTargets()
wf.max_jobs = unzip_phasing_concurrent_jobs
for ctg_id in ctg_ids:
# inputs
ref_fasta = makePypeLocalFile(
'./3-unzip/reads/{ctg_id}_ref.fa'.format(ctg_id=ctg_id))
read_fasta = makePypeLocalFile(
'./3-unzip/reads/{ctg_id}_reads.fa'.format(ctg_id=ctg_id))
# outputs
wd = os.path.join(
os.getcwd(), './3-unzip/0-phasing/{ctg_id}/'.format(ctg_id=ctg_id))
blasr_dir = os.path.join(wd, 'blasr')
ctg_aln_out = makePypeLocalFile(
os.path.join(blasr_dir,
'{ctg_id}_sorted.bam'.format(ctg_id=ctg_id)))
phasing_dir = os.path.join(wd, 'phasing')
job_done = makePypeLocalFile(
os.path.join(phasing_dir, 'p_{ctg_id}_done'.format(ctg_id=ctg_id)))
rid_to_phase_out = makePypeLocalFile(
os.path.join(
wd,
'rid_to_phase.{ctg_id}'.format(ctg_id=ctg_id))) # TODO: ???
all_ctg_out['r2p.{ctg_id}'.format(
ctg_id=ctg_id)] = rid_to_phase_out # implicit output?
parameters = {
'job_uid': 'ha-' + ctg_id,
'wd': wd,
'config': config,
'ctg_id': ctg_id,
'sge_option': config['sge_phasing'],
}
make_phasing_task = PypeTask(
inputs={
'ref_fasta': ref_fasta,
'aln_bam': ctg_aln_out
},
outputs={'job_done': job_done},
parameters=parameters,
)
phasing_task = make_phasing_task(task_phasing)
wf.addTask(phasing_task)
wf.refreshTargets()
hasm_wd = os.path.abspath('./3-unzip/1-hasm/')
#mkdir(hasm_wd)
rid_to_phase_all = makePypeLocalFile(
os.path.join(hasm_wd, 'rid-to-phase-all', 'rid_to_phase.all'))
task = PypeTask(
inputs=all_ctg_out,
outputs={'rid_to_phase_all': rid_to_phase_all},
)(get_rid_to_phase_all)
wf.addTask(task)
parameters['wd'] = hasm_wd
parameters['sge_option'] = config['sge_hasm']
job_done = makePypeLocalFile(os.path.join(hasm_wd, 'hasm_done'))
make_hasm_task = PypeTask(
inputs={'rid_to_phase_all': rid_to_phase_all},
outputs={'job_done': job_done},
parameters=parameters,
)
hasm_task = make_hasm_task(task_hasm)
wf.addTask(hasm_task)
wf.refreshTargets()
max_jobs=12,
)
rawread_db = makePypeLocalFile( os.path.join( rawread_dir, "raw_reads.db" ) )
rawread_id_file = makePypeLocalFile( os.path.join( rawread_dir, "raw_read_ids" ) )
@PypeTask( inputs = {"rawread_db": rawread_db},
outputs = {"rawread_id_file": rawread_id_file},
TaskType = PypeThreadTaskBase,
URL = "task://localhost/dump_rawread_ids" )
def dump_rawread_ids(self):
rawread_db = fn( self.rawread_db )
rawread_id_file = fn( self.rawread_id_file )
os.system("DBshow -n %s | tr -d '>' | awk '{print $1}' > %s" % (rawread_db, rawread_id_file) )
wf.addTask( dump_rawread_ids )
pread_db = makePypeLocalFile( os.path.join( pread_dir, "preads.db" ) )
pread_id_file = makePypeLocalFile( os.path.join( pread_dir, "pread_ids" ) )
@PypeTask( inputs = {"pread_db": pread_db},
outputs = {"pread_id_file": pread_id_file},
TaskType = PypeThreadTaskBase,
URL = "task://localhost/dump_pread_ids" )
def dump_pread_ids(self):
pread_db = fn( self.pread_db )
pread_id_file = fn( self.pread_id_file )
os.system("DBshow -n %s | tr -d '>' | awk '{print $1}' > %s" % (pread_db, pread_id_file) )
wf.addTask( dump_pread_ids )
wf.refreshTargets() # block
def main(argv=sys.argv):
global LOG
LOG = support.setup_logger(None)
if len(sys.argv) < 2:
print>>sys.stderr, 'you need to provide a configuration file to specific a couple cluster running environment'
sys.exit(1)
config_fn = sys.argv[1]
config_absbasedir = os.path.dirname(os.path.abspath(config_fn))
config = ConfigParser.ConfigParser()
config.read(config_fn)
job_type = 'SGE'
if config.has_option('General', 'job_type'):
job_type = config.get('General', 'job_type')
job_queue = 'default'
if config.has_option('General', 'job_queue'):
job_queue = config.get('General', 'job_queue')
pwatcher_type = 'fs_based'
if config.has_option('General', 'pwatcher_type'):
pwatcher_type = config.get('General', 'pwatcher_type')
sge_track_reads = ' -pe smp 12 -q bigmem'
if config.has_option('Unzip', 'sge_track_reads'):
sge_track_reads = config.get('Unzip', 'sge_track_reads')
sge_quiver = ' -pe smp 24 -q bigmem '
if config.has_option('Unzip', 'sge_quiver'):
sge_quiver = config.get('Unzip', 'sge_quiver')
smrt_bin = '/mnt/secondary/builds/full/3.0.0/prod/smrtanalysis_3.0.0.153854/smrtcmds/bin/'
if config.has_option('Unzip', 'smrt_bin'):
smrt_bin = config.get('Unzip', 'smrt_bin')
input_bam_fofn = 'input_bam.fofn'
if config.has_option('Unzip', 'input_bam_fofn'):
input_bam_fofn = config.get('Unzip', 'input_bam_fofn')
if not os.path.isabs(input_bam_fofn):
input_bam_fofn = os.path.join(config_absbasedir, input_bam_fofn)
quiver_concurrent_jobs = 8
if config.has_option('Unzip', 'quiver_concurrent_jobs'):
quiver_concurrent_jobs = config.getint('Unzip', 'quiver_concurrent_jobs')
config = {'job_type': job_type,
'job_queue': job_queue,
'sge_quiver': sge_quiver,
'sge_track_reads': sge_track_reads,
'input_bam_fofn': input_bam_fofn,
'pwatcher_type': pwatcher_type,
'smrt_bin': smrt_bin}
LOG.info('config={}'.format(pprint.pformat(config)))
#support.job_type = 'SGE' #tmp hack until we have a configuration parser
wf = PypeProcWatcherWorkflow(
max_jobs=quiver_concurrent_jobs,
job_type=config['job_type'],
job_queue=config.get('job_queue'),
sge_option=config.get('sge_option'),
watcher_type=config.get('pwatcher_type'),
#watcher_directory=config.get('pwatcher_directory', 'mypwatcher'),
use_tmpdir=config.get('use_tmpdir'),
)
abscwd = os.path.abspath('.')
parameters = {
'sge_option': config['sge_track_reads'],
}
input_bam_fofn_fn = config['input_bam_fofn']
input_bam_fofn_plf = makePypeLocalFile(input_bam_fofn_fn)
hasm_done_plf = makePypeLocalFile('./3-unzip/1-hasm/hasm_done') # by convention
track_reads_h_done_plf = makePypeLocalFile('./4-quiver/reads/track_reads_h_done')
make_track_reads_task = PypeTask(inputs = {
'input_bam_fofn': input_bam_fofn_plf,
'hasm_done': hasm_done_plf},
outputs = {'job_done': track_reads_h_done_plf},
parameters = parameters,
)
track_reads_task = make_track_reads_task(task_track_reads)
wf.addTask(track_reads_task)
scattered_quiver_plf = makePypeLocalFile('4-quiver/quiver_scatter/scattered.json')
parameters = {
'config': config,
}
make_task = PypeTask(
inputs = {
'p_ctg_fa': makePypeLocalFile('3-unzip/all_p_ctg.fa'),
'h_ctg_fa': makePypeLocalFile('3-unzip/all_h_ctg.fa'),
'track_reads_h_done': track_reads_h_done_plf,
},
outputs = {
'scattered_quiver_json': scattered_quiver_plf,
},
parameters = parameters,
)
wf.addTask(make_task(task_scatter_quiver))
wf.refreshTargets()
p_ctg_out, h_ctg_out, job_done_plfs = create_quiver_jobs(wf, scattered_quiver_plf)
gathered_p_ctg_plf = makePypeLocalFile('4-quiver/cns_gather/p_ctg.txt')
gathered_h_ctg_plf = makePypeLocalFile('4-quiver/cns_gather/h_ctg.txt')
gather_done_plf = makePypeLocalFile('4-quiver/cns_gather/job_done')
mkdir('4-quiver/cns_gather')
with open(fn(gathered_p_ctg_plf), 'w') as ifs:
for cns_fasta_fn, cns_fastq_fn in sorted(p_ctg_out):
ifs.write('{} {}\n'.format(cns_fasta_fn, cns_fastq_fn))
with open(fn(gathered_h_ctg_plf), 'w') as ifs:
for cns_fasta_fn, cns_fastq_fn in sorted(h_ctg_out):
ifs.write('{} {}\n'.format(cns_fasta_fn, cns_fastq_fn))
make_task = PypeTask(
inputs = job_done_plfs,
outputs = {
'job_done': gather_done_plf,
},
parameters = {},
)
wf.addTask(make_task(task_gather_quiver))
wf.refreshTargets()
cns_p_ctg_fasta_plf = makePypeLocalFile('4-quiver/cns_output/cns_p_ctg.fasta')
cns_p_ctg_fastq_plf = makePypeLocalFile('4-quiver/cns_output/cns_p_ctg.fastq')
cns_h_ctg_fasta_plf = makePypeLocalFile('4-quiver/cns_output/cns_h_ctg.fasta')
cns_h_ctg_fastq_plf = makePypeLocalFile('4-quiver/cns_output/cns_h_ctg.fastq')
zcat_done_plf = makePypeLocalFile('4-quiver/cns_output/job_done')
make_task = PypeTask(
inputs = {
'gathered_p_ctg': gathered_p_ctg_plf,
'gathered_h_ctg': gathered_h_ctg_plf,
'gather_done': gather_done_plf,
},
outputs = {
'cns_p_ctg_fasta': cns_p_ctg_fasta_plf,
'cns_p_ctg_fastq': cns_p_ctg_fastq_plf,
'cns_h_ctg_fasta': cns_h_ctg_fasta_plf,
'cns_h_ctg_fastq': cns_h_ctg_fastq_plf,
'job_done': zcat_done_plf,
},
)
wf.addTask(make_task(task_cns_zcat))
wf.refreshTargets()
def unzip_all(config):
unzip_blasr_concurrent_jobs = config['unzip_blasr_concurrent_jobs']
unzip_phasing_concurrent_jobs = config['unzip_phasing_concurrent_jobs']
wf = PypeProcWatcherWorkflow(
max_jobs=unzip_blasr_concurrent_jobs,
job_type=config['job_type'],
job_queue=config.get('job_queue'),
sge_option=config.get('sge_option'),
watcher_type=config.get('pwatcher_type'),
#watcher_directory=config.get('pwatcher_directory', 'mypwatcher'),
use_tmpdir=config.get('use_tmpdir'),
)
ctg_list_file = makePypeLocalFile('./3-unzip/reads/ctg_list')
falcon_asm_done = makePypeLocalFile('./2-asm-falcon/falcon_asm_done')
wdir = os.path.abspath('./3-unzip/reads')
parameters = {'wd': wdir, 'config': config,
'sge_option': config['sge_track_reads'],
}
job_done = makePypeLocalFile(os.path.join(parameters['wd'], 'track_reads_done'))
make_track_reads_task = PypeTask(inputs = {'falcon_asm_done': falcon_asm_done},
outputs = {'job_done': job_done, 'ctg_list_file': ctg_list_file},
parameters = parameters,
wdir = wdir,
)
track_reads_task = make_track_reads_task(task_track_reads)
wf.addTask(track_reads_task)
wf.refreshTargets() #force refresh now, will put proper dependence later
ctg_ids = []
with open('./3-unzip/reads/ctg_list') as f:
for row in f:
row = row.strip()
ctg_ids.append(row)
aln1_outs = {}
all_ctg_out = {}
for ctg_id in ctg_ids:
# inputs
ref_fasta = makePypeLocalFile('./3-unzip/reads/{ctg_id}_ref.fa'.format(ctg_id = ctg_id))
read_fasta = makePypeLocalFile('./3-unzip/reads/{ctg_id}_reads.fa'.format(ctg_id = ctg_id))
# outputs
wd = os.path.join(os.getcwd(), './3-unzip/0-phasing/{ctg_id}/'.format(ctg_id = ctg_id))
#mkdir(wd)
blasr_dir = os.path.join(wd, 'blasr')
ctg_aln_out = makePypeLocalFile(os.path.join(blasr_dir, '{ctg_id}_sorted.bam'.format(ctg_id = ctg_id)))
job_done = makePypeLocalFile(os.path.join(blasr_dir, 'aln_{ctg_id}_done'.format(ctg_id = ctg_id)))
parameters = {'job_uid':'aln-'+ctg_id, 'wd': blasr_dir, 'config':config, 'ctg_id': ctg_id,
'sge_option': config['sge_blasr_aln'],
}
make_blasr_task = PypeTask(inputs = {'ref_fasta': ref_fasta, 'read_fasta': read_fasta},
outputs = {'ctg_aln_out': ctg_aln_out, 'job_done': job_done},
parameters = parameters,
)
blasr_task = make_blasr_task(task_run_blasr)
aln1_outs[ctg_id] = (ctg_aln_out, job_done)
wf.addTask(blasr_task)
wf.refreshTargets()
wf.max_jobs = unzip_phasing_concurrent_jobs
for ctg_id in ctg_ids:
# inputs
ref_fasta = makePypeLocalFile('./3-unzip/reads/{ctg_id}_ref.fa'.format(ctg_id = ctg_id))
read_fasta = makePypeLocalFile('./3-unzip/reads/{ctg_id}_reads.fa'.format(ctg_id = ctg_id))
# outputs
wd = os.path.join(os.getcwd(), './3-unzip/0-phasing/{ctg_id}/'.format(ctg_id = ctg_id))
blasr_dir = os.path.join(wd, 'blasr')
ctg_aln_out = makePypeLocalFile(os.path.join(blasr_dir, '{ctg_id}_sorted.bam'.format(ctg_id = ctg_id)))
phasing_dir = os.path.join(wd, 'phasing')
job_done = makePypeLocalFile(os.path.join(phasing_dir, 'p_{ctg_id}_done'.format(ctg_id = ctg_id)))
rid_to_phase_out = makePypeLocalFile(os.path.join(wd, 'rid_to_phase.{ctg_id}'.format(ctg_id = ctg_id))) # TODO: ???
all_ctg_out[ 'r2p.{ctg_id}'.format(ctg_id = ctg_id) ] = rid_to_phase_out # implicit output?
parameters = {'job_uid':'ha-'+ctg_id, 'wd': wd, 'config':config, 'ctg_id': ctg_id,
'sge_option': config['sge_phasing'],
}
make_phasing_task = PypeTask(inputs = {'ref_fasta': ref_fasta, 'aln_bam':ctg_aln_out},
outputs = {'job_done': job_done},
parameters = parameters,
)
phasing_task = make_phasing_task(task_phasing)
wf.addTask(phasing_task)
wf.refreshTargets()
hasm_wd = os.path.abspath('./3-unzip/1-hasm/')
#mkdir(hasm_wd)
rid_to_phase_all = makePypeLocalFile(os.path.join(hasm_wd, 'rid-to-phase-all', 'rid_to_phase.all'))
task = PypeTask(inputs = all_ctg_out, outputs = {'rid_to_phase_all': rid_to_phase_all},
) (get_rid_to_phase_all)
wf.addTask(task)
parameters['wd'] = hasm_wd
parameters['sge_option'] = config['sge_hasm']
job_done = makePypeLocalFile(os.path.join(hasm_wd, 'hasm_done'))
make_hasm_task = PypeTask(inputs = {'rid_to_phase_all': rid_to_phase_all},
outputs = {'job_done': job_done},
parameters = parameters,
)
hasm_task = make_hasm_task(task_hasm)
wf.addTask(hasm_task)
wf.refreshTargets()