def main(argv):
args = parser.parse_args(argv[1:])
if not args.overwrite:
if os.path.exists(args.output):
print("Cowardly refusing to overwrite {}".format(args.output))
sys.exit(1)
fast5_files = fast5.iterate_fast5(args.input_folder,
paths=True,
limit=args.limit,
strand_list=args.input_strand_list)
print('* Processing data using', args.jobs, 'threads')
i = 0
kwarg_names = ['section']
with open(args.output, 'w') as file_handle:
for res in imap_mp(reference_extraction_worker,
fast5_files,
threads=args.jobs,
unordered=True,
fix_kwargs=util.get_kwargs(args, kwarg_names)):
if res is not None:
i = util.progress_report(i)
file_name, reference = res
header = '>{}\n'.format(
os.path.basename(os.path.splitext(file_name)[0]))
file_handle.write(header)
file_handle.write(reference + '\n')
def raw_chunkify_with_identity_main(args):
""" Main function for `chunkify.py raw_identity` producing batch file for model training
"""
if not args.overwrite:
if os.path.exists(args.output):
print("Cowardly refusing to overwrite {}".format(args.output))
sys.exit(1)
fast5_files = fast5.iterate_fast5(args.input_folder,
paths=True,
limit=args.limit,
strand_list=args.input_strand_list)
print('* Processing data using', args.jobs, 'threads')
kwarg_names = [
'chunk_len', 'kmer_len', 'min_length', 'trim', 'normalisation',
'downsample_factor', 'interpolation'
]
i = 0
bad_list = []
chunk_list = []
label_list = []
for res in imap_mp(raw_chunk_worker,
fast5_files,
threads=args.jobs,
unordered=True,
fix_kwargs=util.get_kwargs(args, kwarg_names),
init=batch.init_chunk_identity_worker,
initargs=[args.kmer_len, args.alphabet]):
if res is not None:
i = util.progress_report(i)
(chunks, labels, bad_ev) = res
chunk_list.append(chunks)
label_list.append(labels)
bad_list.append(bad_ev)
if chunk_list == []:
print("no chunks were produced", file=sys.stderr)
sys.exit(1)
else:
print('\n* Writing out to HDF5')
hdf5_attributes = {
'chunk': args.chunk_len,
'downsample_factor': args.downsample_factor,
'input_type': 'raw',
'interpolation': args.interpolation,
'kmer': args.kmer_len,
'normalisation': args.normalisation,
'section': 'template',
'trim': args.trim,
'alphabet': args.alphabet,
}
blanks_per_chunk = np.concatenate([(l == 0).mean(1)
for l in label_list])
blanks = np.percentile(blanks_per_chunk, args.blanks_percentile)
util.create_labelled_chunks_hdf5(args.output, blanks, hdf5_attributes,
chunk_list, label_list, bad_list)
def chunkify_with_identity_main(args):
if not args.overwrite:
if os.path.exists(args.output):
print("Cowardly refusing to overwrite {}".format(args.output))
sys.exit(1)
fast5_files = fast5.iterate_fast5(args.input_folder, paths=True,
limit=args.limit,
strand_list=args.input_strand_list)
print('* Processing data using', args.jobs, 'threads')
kwarg_names = ['section', 'chunk_len', 'kmer_len', 'min_length', 'trim', 'use_scaled', 'normalisation']
i = 0
bad_list = []
chunk_list = []
label_list = []
for res in imap_mp(batch.chunk_worker, fast5_files, threads=args.jobs,
unordered=True, fix_kwargs=util.get_kwargs(args, kwarg_names),
init=batch.init_chunk_identity_worker, initargs=[args.kmer_len, args.alphabet]):
if res is not None:
i = util.progress_report(i)
(chunks, labels, bad_ev) = res
chunk_list.append(chunks)
label_list.append(labels)
bad_list.append(bad_ev)
if chunk_list == []:
print("no chunks were produced", file=sys.stderr)
sys.exit(1)
else:
print('\n* Writing out to HDF5')
hdf5_attributes = {
'chunk': args.chunk_len,
'input_type': 'events',
'kmer': args.kmer_len,
'normalisation': args.normalisation,
'scaled': args.use_scaled,
'section': args.section,
'trim': args.trim,
'alphabet': args.alphabet,
}
util.create_labelled_chunks_hdf5(args.output, args.blanks, hdf5_attributes, chunk_list, label_list, bad_list)
assert args.datatype in ["events", "samples"]
assert args.command in ["events", "raw"]
basecall_worker = getattr(basecall, args.command + "_worker")
if args.command == "events":
kwarg_names = ['section', 'segmentation', 'trim', 'kmer_len', 'transducer', 'bad', 'min_prob', 'skip', 'trans', 'alphabet']
else:
kwarg_names = ['trim', 'open_pore_fraction', 'kmer_len', 'transducer', 'bad', 'min_prob', 'skip', 'trans', 'alphabet']
compiled_file = helpers.compile_model(args.model, args.compile)
seq_printer = basecall.SeqPrinter(args.kmer_len, datatype=args.datatype,
transducer=args.transducer, alphabet=args.alphabet.decode('ascii'))
files = fast5.iterate_fast5(args.input_folder, paths=True, limit=args.limit,
strand_list=args.input_strand_list)
nbases = nevents = 0
t0 = time.time()
for res in imap_mp(basecall_worker, files, threads=args.jobs, fix_kwargs=util.get_kwargs(args, kwarg_names),
unordered=True, init=basecall.init_worker, initargs=[compiled_file]):
if res is None:
continue
read, score, call, nev = res
seq_len = seq_printer.write(read, score, call, nev)
nbases += seq_len
nevents += nev
dt = time.time() - t0
t = 'Called {} bases in {:.1f} s ({:.1f} bases/s or {:.1f} {}/s)\n'
sys.stderr.write(t.format(nbases, dt, nbases / dt, nevents / dt, args.datatype))
def raw_chunkify_with_remap_main(args):
""" Main function for `chunkify.py raw_remap` producing batch file for model training
"""
if not args.overwrite:
if os.path.exists(args.output):
print("Cowardly refusing to overwrite {}".format(args.output))
sys.exit(1)
if os.path.exists(args.output_strand_list):
print("Cowardly refusing to overwrite {}".format(
args.output_strand_list))
sys.exit(2)
fast5_files = fast5.iterate_fast5(args.input_folder,
paths=True,
limit=args.limit,
strand_list=args.input_strand_list)
references = util.fasta_file_to_dict(args.references)
print('* Processing data using', args.jobs, 'threads')
kwarg_names = [
'trim', 'min_prob', 'kmer_len', 'min_length', 'prior', 'slip',
'chunk_len', 'normalisation', 'downsample_factor', 'interpolation',
'open_pore_fraction'
]
kwargs = util.get_kwargs(args, kwarg_names)
kwargs['references'] = references
i = 0
compiled_file = helpers.compile_model(args.model, args.compile)
output_strand_list_entries = []
bad_list = []
chunk_list = []
label_list = []
with open(args.output_strand_list, 'w') as slfh:
slfh.write(u'\t'.join([
'filename', 'nblocks', 'score', 'nstay', 'seqlen', 'start', 'end'
]) + u'\n')
for res in imap_mp(
raw_chunk_remap_worker,
fast5_files,
threads=args.jobs,
fix_kwargs=kwargs,
unordered=True,
init=batch.init_chunk_remap_worker,
initargs=[compiled_file, args.kmer_len, args.alphabet]):
if res is not None:
i = util.progress_report(i)
read, score, nblocks, path, seq, chunks, labels, bad_ev = res
chunk_list.append(chunks)
label_list.append(labels)
bad_list.append(bad_ev)
strand_data = [
read, nblocks, -score / nblocks,
np.sum(np.ediff1d(path, to_begin=1) == 0),
len(seq),
min(path),
max(path)
]
slfh.write('\t'.join([str(x) for x in strand_data]) + '\n')
if compiled_file != args.compile:
os.remove(compiled_file)
if chunk_list == []:
print("no chunks were produced", file=sys.stderr)
sys.exit(1)
else:
print('\n* Writing out to HDF5')
hdf5_attributes = {
'chunk': args.chunk_len,
'downsample_factor': args.downsample_factor,
'input_type': 'raw',
'interpolation': args.interpolation,
'kmer': args.kmer_len,
'normalisation': args.normalisation,
'section': 'template',
'trim': args.trim,
'alphabet': args.alphabet,
}
blanks_per_chunk = np.concatenate([(l == 0).mean(1)
for l in label_list])
blanks = np.percentile(blanks_per_chunk, args.blanks_percentile)
util.create_labelled_chunks_hdf5(args.output, blanks, hdf5_attributes,
chunk_list, label_list, bad_list)
def chunkify_with_remap_main(args):
if not args.overwrite:
if os.path.exists(args.output):
print("Cowardly refusing to overwrite {}".format(args.output))
sys.exit(1)
if os.path.exists(args.output_strand_list):
print("Cowardly refusing to overwrite {}".format(
args.output_strand_list))
sys.exit(2)
fast5_files = fast5.iterate_fast5(args.input_folder,
paths=True,
limit=args.limit,
strand_list=args.input_strand_list)
references = util.fasta_file_to_dict(args.references)
print('* Processing data using', args.jobs, 'threads')
kwarg_names = [
'trim', 'min_prob', 'kmer_len', 'min_length', 'prior', 'slip',
'chunk_len', 'use_scaled', 'normalisation', 'section', 'segmentation'
]
kwargs = util.get_kwargs(args, kwarg_names)
kwargs['references'] = references
i = 0
compiled_file = helpers.compile_model(args.model, args.compile)
output_strand_list_entries = []
bad_list = []
chunk_list = []
label_list = []
with open(args.output_strand_list, 'w') as slfh:
slfh.write(u'\t'.join(
['filename', 'nev', 'score', 'nstay', 'seqlen', 'start', 'end']) +
u'\n')
for res in imap_mp(
batch.chunk_remap_worker,
fast5_files,
threads=args.jobs,
fix_kwargs=kwargs,
unordered=True,
init=batch.init_chunk_remap_worker,
initargs=[compiled_file, args.kmer_len, args.alphabet]):
if res is not None:
i = util.progress_report(i)
read, score, nev, path, seq, chunks, labels, bad_ev = res
chunk_list.append(chunks)
label_list.append(labels)
bad_list.append(bad_ev)
strand_data = [
read, nev, -score / nev,
np.sum(np.ediff1d(path, to_begin=1) == 0),
len(seq),
min(path),
max(path)
]
slfh.write('\t'.join([str(x) for x in strand_data]) + '\n')
if compiled_file != args.compile:
os.remove(compiled_file)
if chunk_list == []:
print("no chunks were produced", file=sys.stderr)
sys.exit(1)
else:
print('\n* Creating HDF5 file')
hdf5_attributes = {
'chunk': args.chunk_len,
'input_type': 'events',
'kmer': args.kmer_len,
'normalisation': args.normalisation,
'scaled': args.use_scaled,
'section': args.section,
'trim': args.trim,
'alphabet': args.alphabet,
}
util.create_labelled_chunks_hdf5(args.output, args.blanks,
hdf5_attributes, chunk_list,
label_list, bad_list)
def test_iterate_works_with_strandlist(self):
fast5_files = set(fast5.iterate_fast5(self.readdir, paths=True,
strand_list=self.strand_list))
self.assertTrue(self.strands == fast5_files)
def test_iterate_respects_limits(self):
_LIMIT = 2
fast5_files = set(fast5.iterate_fast5(self.readdir, paths=True, limit=_LIMIT))
self.assertTrue(len(fast5_files) == _LIMIT)
def test_iterate_returns_all(self):
fast5_files = set(fast5.iterate_fast5(self.readdir, paths=True))
dir_list = set(glob.glob(os.path.join(self.readdir, '*.fast5')))
self.assertTrue(fast5_files == dir_list)