def run(args):
logging.basicConfig(level=int(round(10*args.verbose_level)))
assert args.n_core <= multiprocessing.cpu_count(), 'Requested n_core={} > cpu_count={}'.format(
args.n_core, multiprocessing.cpu_count())
def Start():
LOG.info('Started a worker in {} from parent {}'.format(
os.getpid(), os.getppid()))
exe_pool = Pool(args.n_core, initializer=Start)
if args.trim:
get_consensus = get_consensus_with_trim
else:
get_consensus = get_consensus_without_trim
K = 8
config = args.min_cov, K, \
args.max_n_read, args.min_idt, args.edge_tolerance, args.trim_size, args.min_cov_aln, args.max_cov_aln
# TODO: pass config object, not tuple, so we can add fields
inputs = []
for datum in get_seq_data(config, args.min_n_read, args.min_len_aln):
inputs.append((get_consensus, datum))
try:
LOG.info('running {!r}'.format(get_consensus))
for res in exe_pool.imap(io.run_func, inputs):
process_get_consensus_result(res, args)
LOG.info('finished {!r}'.format(get_consensus))
except:
LOG.exception('failed gen_consensus')
exe_pool.terminate()
raise
def run(args):
logging.basicConfig(level=int(round(10*args.verbose_level)))
assert args.n_core <= multiprocessing.cpu_count(), 'Requested n_core={} > cpu_count={}'.format(
args.n_core, multiprocessing.cpu_count())
def Start():
LOG.info('Started a worker in {} from parent {}'.format(
os.getpid(), os.getppid()))
exe_pool = Pool(args.n_core, initializer=Start)
if args.trim:
get_consensus = get_consensus_with_trim
else:
get_consensus = get_consensus_without_trim
K = 8
config = args.min_cov, K, \
args.max_n_read, args.min_idt, args.edge_tolerance, \
args.trim_size, args.min_cov_aln, args.max_cov_aln, \
args.allow_external_mapping
# TODO: pass config object, not tuple, so we can add fields
inputs = []
for datum in get_seq_data(config, args.min_n_read, args.min_len_aln):
inputs.append((get_consensus, datum))
try:
LOG.info('running {!r}'.format(get_consensus))
for res in exe_pool.imap(io.run_func, inputs):
process_get_consensus_result(res, args)
LOG.info('finished {!r}'.format(get_consensus))
except:
LOG.exception('failed gen_consensus')
exe_pool.terminate()
raise
def try_run_ovlp_stats(n_core, fofn, min_len):
io.LOG('starting ovlp_stats')
file_list = io.validated_fns(fofn)
io.LOG('fofn %r: %r' %(fofn, file_list))
n_core = min(n_core, len(file_list))
exe_pool = Pool(n_core)
try:
run_ovlp_stats(exe_pool, file_list, min_len)
io.LOG('finished ovlp_stats')
except KeyboardInterrupt:
io.LOG('terminating ovlp_stats workers...')
exe_pool.terminate()
def try_run_ovlp_stats(n_core, fofn, min_len):
io.LOG('starting ovlp_stats')
file_list = io.validated_fns(fofn)
io.LOG('fofn %r: %r' % (fofn, file_list))
n_core = min(n_core, len(file_list))
exe_pool = Pool(n_core)
try:
run_ovlp_stats(exe_pool, file_list, min_len)
io.LOG('finished ovlp_stats')
except KeyboardInterrupt:
io.LOG('terminating ovlp_stats workers...')
exe_pool.terminate()
def try_run_ovlp_filter(n_core, fofn, max_diff, max_cov, min_cov, min_len, bestn, db_fn):
io.LOG("starting ovlp_filter")
file_list = io.validated_fns(fofn)
io.LOG("fofn %r: %r" % (fofn, file_list))
n_core = min(n_core, len(file_list))
exe_pool = Pool(n_core)
try:
run_ovlp_filter(exe_pool, file_list, max_diff, max_cov, min_cov, min_len, bestn, db_fn)
io.LOG("finished ovlp_filter")
except KeyboardInterrupt:
io.LOG("terminating ovlp_filter workers...")
exe_pool.terminate()
def try_run_ovlp_filter(n_core, fofn, max_diff, max_cov, min_cov, min_len, bestn, db_fn):
io.LOG('starting ovlp_filter')
file_list = io.validated_fns(fofn)
io.LOG('fofn %r: %r' %(fofn, file_list))
n_core = min(n_core, len(file_list))
exe_pool = Pool(n_core)
try:
run_ovlp_filter(exe_pool, file_list, max_diff, max_cov, min_cov, min_len, bestn, db_fn)
io.LOG('finished ovlp_filter')
except:
io.LOG('terminating ovlp_filter workers...')
exe_pool.terminate()
raise
def try_run_ovlp_filter(n_core, fofn, max_diff, max_cov, min_cov, min_len, bestn, db_fn):
io.LOG('starting ovlp_filter')
file_list = io.validated_fns(fofn)
io.LOG('fofn %r: %r' %(fofn, file_list))
n_core = min(n_core, len(file_list))
exe_pool = Pool(n_core)
try:
run_ovlp_filter(exe_pool, file_list, max_diff, max_cov, min_cov, min_len, bestn, db_fn)
io.LOG('finished ovlp_filter')
except:
io.LOG('terminating ovlp_filter workers...')
exe_pool.terminate()
raise
def try_run_ovlp_stats(n_core, db_fn, fofn, min_len):
io.LOG('starting ovlp_stats')
file_list = io.validated_fns(fofn)
io.LOG('fofn {!r}: {}'.format(fofn, file_list))
io.LOG('db {!r}; n_core={}'.format(db_fn, n_core))
n_core = min(n_core, len(file_list))
exe_pool = Pool(n_core)
try:
run_ovlp_stats(exe_pool, db_fn, file_list, min_len)
io.LOG('finished ovlp_stats')
except KeyboardInterrupt:
io.LOG('terminating ovlp_stats workers...')
exe_pool.terminate()
def try_run_ovlp_stats(n_core, db_fn, fofn, min_len):
io.LOG('starting ovlp_stats')
file_list = io.validated_fns(fofn)
io.LOG('fofn {!r}: {}'.format(fofn, file_list))
io.LOG('db {!r}; n_core={}'.format(db_fn, n_core))
n_core = min(n_core, len(file_list))
exe_pool = Pool(n_core)
try:
run_ovlp_stats(exe_pool, db_fn, file_list, min_len)
io.LOG('finished ovlp_stats')
except KeyboardInterrupt:
io.LOG('terminating ovlp_stats workers...')
exe_pool.terminate()
def try_run_track_reads(n_core, base_dir, min_len, bestn):
io.LOG('starting track_reads')
pread_dir = os.path.abspath(os.path.join(base_dir, "1-preads_ovl"))
file_list = glob.glob(os.path.join(pread_dir, "m*/preads.*.las"))
io.LOG('file list: %r' % file_list)
db_fn = os.path.join(pread_dir, "preads.db")
n_core = min(n_core, len(file_list))
exe_pool = Pool(n_core)
try:
run_track_reads(exe_pool, base_dir, file_list, min_len, bestn, db_fn)
io.LOG('finished track_reads')
except:
io.LOG('terminating track_reads workers...')
exe_pool.terminate()
raise
def try_run_track_reads(n_core, base_dir, min_len, bestn):
io.LOG('starting track_reads')
pread_dir = os.path.abspath(os.path.join(base_dir, "1-preads_ovl"))
file_list = glob.glob(os.path.join(pread_dir, "m*/preads.*.las"))
io.LOG('file list: %r' % file_list)
db_fn = os.path.join(pread_dir, "preads.db")
n_core = min(n_core, len(file_list))
exe_pool = Pool(n_core)
try:
run_track_reads(exe_pool, base_dir, file_list, min_len, bestn, db_fn)
io.LOG('finished track_reads')
except:
io.LOG('terminating track_reads workers...')
exe_pool.terminate()
raise
def try_run_ovlp_filter(out_fn, n_core, fofn, max_diff, max_cov, min_cov, min_len, min_idt, ignore_indels, bestn, db_fn):
io.LOG('starting ovlp_filter')
file_list = io.validated_fns(fofn)
io.LOG('fofn %r: %r' % (fofn, file_list))
n_core = min(n_core, len(file_list))
exe_pool = Pool(n_core)
tmp_out_fn = out_fn + '.tmp'
try:
with open(tmp_out_fn, 'w') as outs:
run_ovlp_filter(outs, exe_pool, file_list, max_diff, max_cov,
min_cov, min_len, min_idt, ignore_indels, bestn, db_fn)
os.rename(tmp_out_fn, out_fn)
io.LOG('finished ovlp_filter')
except:
io.LOG('terminating ovlp_filter workers...')
exe_pool.terminate()
raise
def try_run_ovlp_filter(out_fn, n_core, fofn, max_diff, max_cov, min_cov, min_len, bestn, db_fn):
io.LOG('starting ovlp_filter')
file_list = io.validated_fns(fofn)
io.LOG('fofn %r: %r' % (fofn, file_list))
n_core = min(n_core, len(file_list))
exe_pool = Pool(n_core)
tmp_out_fn = out_fn + '.tmp'
try:
with open(tmp_out_fn, 'w') as outs:
run_ovlp_filter(outs, exe_pool, file_list, max_diff, max_cov,
min_cov, min_len, bestn, db_fn)
outs.write('---\n')
os.rename(tmp_out_fn, out_fn)
io.LOG('finished ovlp_filter')
except:
io.LOG('terminating ovlp_filter workers...')
exe_pool.terminate()
raise
def try_run_track_reads(n_core, phased_read_file, read_to_contig_map, rawread_ids, min_len, bestn, output):
io.LOG('starting track_reads')
rawread_dir = os.path.abspath('0-rawreads')
# better logic for finding the las files path or move the logic to extern (taking the --fofn option?)
file_list = glob.glob( os.path.join(rawread_dir, 'm*/raw_reads.*.las')) # TODO: More inputs
io.LOG('file list: %r' % file_list)
db_fn = os.path.join(rawread_dir, 'raw_reads.db') # TODO: Another input
n_core = min(n_core, len(file_list))
exe_pool = Pool(n_core)
try:
run_track_reads(exe_pool, phased_read_file, read_to_contig_map, rawread_ids, file_list, min_len, bestn, db_fn, output)
io.LOG('finished track_reads')
except:
io.LOG('terminating track_reads workers...')
exe_pool.terminate()
raise
def try_run_track_reads(n_core, base_dir, min_len, bestn):
io.LOG('starting track_reads')
rawread_dir = os.path.abspath(os.path.join(base_dir, "0-rawreads"))
# better logic for finding the las files path or move the logic to extern (taking the --fofn option?)
file_list = glob.glob(os.path.join(rawread_dir, "m*/raw_reads.*.las"))
io.LOG('file list: %r' % file_list)
# same, shoud we decide this as a parameter
db_fn = os.path.join(rawread_dir, "raw_reads.db")
n_core = min(n_core, len(file_list))
exe_pool = Pool(n_core)
try:
run_track_reads(exe_pool, base_dir, file_list, min_len, bestn, db_fn)
io.LOG('finished track_reads')
except:
io.LOG('terminating track_reads workers...')
exe_pool.terminate()
raise
def try_run_track_reads(n_core, base_dir, min_len, bestn):
io.LOG('starting track_reads')
rawread_dir = os.path.abspath(os.path.join(base_dir, "0-rawreads"))
# better logic for finding the las files path or move the logic to extern (taking the --fofn option?)
file_list = glob.glob(os.path.join(rawread_dir, "m*/raw_reads.*.las"))
io.LOG('file list: %r' % file_list)
# same, shoud we decide this as a parameter
db_fn = os.path.join(rawread_dir, "raw_reads.db")
n_core = min(n_core, len(file_list))
exe_pool = Pool(n_core)
try:
run_track_reads(exe_pool, base_dir, file_list, min_len, bestn, db_fn)
io.LOG('finished track_reads')
except:
io.LOG('terminating track_reads workers...')
exe_pool.terminate()
raise