def main(args):
info('build-graph.py', ['graph', 'SeqAn'])
report_on_config(args, hashtype='nodegraph')
base = args.output_filename
filenames = args.input_filenames
for fname in args.input_filenames:
check_input_files(fname, args.force)
# if optimization args are given, do optimization
args = functions.do_sanity_checking(args, 0.01)
check_space(args.input_filenames, args.force)
check_space_for_hashtable(args, 'nodegraph', args.force)
print('Saving k-mer presence table to %s' % base, file=sys.stderr)
print('Loading kmers from sequences in %s' %
repr(filenames), file=sys.stderr)
if args.no_build_tagset:
print('We WILL NOT build the tagset.', file=sys.stderr)
else:
print('We WILL build the tagset (for partitioning/traversal).',
file=sys.stderr)
print('making nodegraph', file=sys.stderr)
htable = khmer_args.create_nodegraph(args)
functions.build_graph(filenames, htable, args.threads,
not args.no_build_tagset)
print('Total number of unique k-mers: {0}'.format(htable.n_unique_kmers()),
file=sys.stderr)
print('saving k-mer presence table in', base + '.pt', file=sys.stderr)
htable.save(base + '.pt')
if not args.no_build_tagset:
print('saving tagset in', base + '.tagset', file=sys.stderr)
htable.save_tagset(base + '.tagset')
info_fp = open(base + '.info', 'w')
info_fp.write('%d unique k-mers' % htable.n_unique_kmers())
fp_rate = \
khmer.calc_expected_collisions(htable, args.force, max_false_pos=.15)
# 0.18 is ACTUAL MAX. Do not change.
print('false positive rate estimated to be %1.3f' % fp_rate,
file=sys.stderr)
print('\nfalse positive rate estimated to be %1.3f' % fp_rate,
file=info_fp)
print('wrote to', base + '.info and', base + '.pt', file=sys.stderr)
if not args.no_build_tagset:
print('and ' + base + '.tagset', file=sys.stderr)
sys.exit(0)
def main(): # pylint: disable=too-many-branches,too-many-statements
info('normalize-by-median.py', ['diginorm'])
args = get_parser().parse_args()
report_on_config(args)
report_fp = args.report
force_single = args.force_single
# if optimization args are given, do optimization
args = oxutils.do_sanity_checking(args, 0.1)
# check for similar filenames
# if we're using a single output file only check for identical filenames
# otherwise, check for identical BASE names as well.
filenames = []
basenames = []
for pathfilename in args.input_filenames:
filenames.append(pathfilename)
if args.single_output_file:
continue # nothing more to worry about
basename = os.path.basename(pathfilename)
if basename in basenames:
print('ERROR: Duplicate filename--Cannot handle this!',
file=sys.stderr)
print('** Exiting!', file=sys.stderr)
sys.exit(1)
basenames.append(basename)
# check that files exist and there is sufficient output disk space.
check_valid_file_exists(args.input_filenames)
check_space(args.input_filenames, args.force)
if args.savetable:
check_space_for_hashtable(args, 'countgraph', args.force)
# load or create counting table.
if args.loadtable:
print('loading k-mer counting table from ' + args.loadtable,
file=sys.stderr)
htable = khmer.load_counting_hash(args.loadtable)
if args.unique_kmers != 0:
print('Warning: You have specified a number of unique kmers'
' but are loading a precreated counting table--'
'argument optimization will NOT be done.', file=sys.stderr)
else:
print('making countgraph', file=sys.stderr)
htable = khmer_args.create_countgraph(args)
# create an object to handle diginorm of all files
norm = Normalizer(args.cutoff, htable)
with_diagnostics = WithDiagnostics(norm, report_fp, args.report_frequency)
# make a list of all filenames and if they're paired or not;
# if we don't know if they're paired, default to allowing but not
# forcing pairing.
files = []
for element in filenames:
files.append([element, args.paired])
if args.unpaired_reads:
files.append([args.unpaired_reads, False])
corrupt_files = []
outfp = None
output_name = None
if args.single_output_file:
if args.single_output_file is sys.stdout:
output_name = '/dev/stdout'
else:
output_name = args.single_output_file.name
outfp = args.single_output_file
#
# main loop: iterate over all files given, do diginorm.
#
for filename, require_paired in files:
if not args.single_output_file:
output_name = os.path.basename(filename) + '.keep'
outfp = open(output_name, 'w')
# failsafe context manager in case an input file breaks
with catch_io_errors(filename, outfp, args.single_output_file,
args.force, corrupt_files):
screed_iter = screed.open(filename, parse_description=False)
reader = broken_paired_reader(screed_iter, min_length=args.ksize,
force_single=force_single,
require_paired=require_paired)
# actually do diginorm
for record in with_diagnostics(reader, filename):
if record is not None:
write_record(record, outfp)
print('output in ' + output_name, file=sys.stderr)
if output_name is not '/dev/stdout':
outfp.close()
# finished - print out some diagnostics.
print('Total number of unique k-mers: {0}'
.format(htable.n_unique_kmers()),
file=sys.stderr)
if args.savetable:
print('...saving to ' + args.savetable, file=sys.stderr)
htable.save(args.savetable)
fp_rate = \
khmer.calc_expected_collisions(htable, args.force, max_false_pos=.8)
# for max_false_pos see Zhang et al., http://arxiv.org/abs/1309.2975
print('fp rate estimated to be {fpr:1.3f}'.format(fpr=fp_rate),
file=sys.stderr)
if args.force and len(corrupt_files) > 0:
print("** WARNING: Finished with errors!", file=sys.stderr)
print("** I/O Errors occurred in the following files:",
file=sys.stderr)
print("\t", " ".join(corrupt_files), file=sys.stderr)
def main(): # pylint: disable=too-many-branches,too-many-statements
info('normalize-by-median.py', ['diginorm'])
args = get_parser().parse_args()
report_on_config(args)
report_fp = args.report
force_single = args.force_single
# if optimization args are given, do optimization
args = oxutils.do_sanity_checking(args, 0.1)
# check for similar filenames
# if we're using a single output file only check for identical filenames
# otherwise, check for identical BASE names as well.
filenames = []
basenames = []
for pathfilename in args.input_filenames:
filenames.append(pathfilename)
if args.single_output_file:
continue # nothing more to worry about
basename = os.path.basename(pathfilename)
if basename in basenames:
print('ERROR: Duplicate filename--Cannot handle this!',
file=sys.stderr)
print('** Exiting!', file=sys.stderr)
sys.exit(1)
basenames.append(basename)
# check that files exist and there is sufficient output disk space.
check_valid_file_exists(args.input_filenames)
check_space(args.input_filenames, args.force)
if args.savetable:
check_space_for_hashtable(args, 'countgraph', args.force)
# load or create counting table.
if args.loadtable:
print('loading k-mer counting table from ' + args.loadtable,
file=sys.stderr)
htable = khmer.load_counting_hash(args.loadtable)
if args.unique_kmers != 0:
print('Warning: You have specified a number of unique kmers'
' but are loading a precreated counting table--'
'argument optimization will NOT be done.', file=sys.stderr)
else:
print('making countgraph', file=sys.stderr)
htable = khmer_args.create_countgraph(args)
# create an object to handle diginorm of all files
norm = Normalizer(args.cutoff, htable)
# make a list of all filenames and if they're paired or not;
# if we don't know if they're paired, default to allowing but not
# forcing pairing.
files = []
for element in filenames:
files.append([element, args.paired])
if args.unpaired_reads:
files.append([args.unpaired_reads, False])
corrupt_files = []
outfp = None
output_name = None
if args.single_output_file:
if args.single_output_file is sys.stdout:
output_name = '/dev/stdout'
else:
output_name = args.single_output_file.name
outfp = args.single_output_file
#
# main loop: iterate over all files given, do diginorm.
#
for filename, require_paired in files:
if not args.single_output_file:
output_name = os.path.basename(filename) + '.keep'
outfp = open(output_name, 'w')
# failsafe context manager in case an input file breaks
with catch_io_errors(filename, outfp, args.single_output_file,
args.force, corrupt_files):
screed_iter = screed.open(filename, parse_description=False)
reader = broken_paired_reader(screed_iter, min_length=args.ksize,
force_single=force_single,
require_paired=require_paired)
# actually do diginorm
for record in with_diagnostics(filename, norm, reader, report_fp):
if record is not None:
write_record(record, outfp)
print('output in ' + output_name, file=sys.stderr)
if output_name is not '/dev/stdout':
outfp.close()
# finished - print out some diagnostics.
print('Total number of unique k-mers: {0}'
.format(htable.n_unique_kmers()),
file=sys.stderr)
if args.savetable:
print('...saving to ' + args.savetable, file=sys.stderr)
htable.save(args.savetable)
fp_rate = \
khmer.calc_expected_collisions(htable, args.force, max_false_pos=.8)
# for max_false_pos see Zhang et al., http://arxiv.org/abs/1309.2975
print('fp rate estimated to be {fpr:1.3f}'.format(fpr=fp_rate),
file=sys.stderr)
if args.force and len(corrupt_files) > 0:
print("** WARNING: Finished with errors!", file=sys.stderr)
print("** I/O Errors occurred in the following files:",
file=sys.stderr)
print("\t", " ".join(corrupt_files), file=sys.stderr)