def main(): # pylint: disable=too-many-locals,too-many-branches args = sanitize_help(get_parser()).parse_args() graph_type = 'smallcountgraph' if args.small_count else 'countgraph' configure_logging(args.quiet) report_on_config(args, graph_type) check_input_files(args.input_sequence_filename, args.force) if args.savegraph is not None: graphsize = calculate_graphsize(args, graph_type) check_space_for_graph(args.savegraph, graphsize, args.force) if (not args.squash_output and os.path.exists(args.output_histogram_filename)): log_error('ERROR: {output} exists; not squashing.', output=args.output_histogram_filename) sys.exit(1) else: hist_fp = open(args.output_histogram_filename, 'w') hist_fp_csv = csv.writer(hist_fp) # write headers: hist_fp_csv.writerow( ['abundance', 'count', 'cumulative', 'cumulative_fraction']) log_info('making countgraph') # In case the user specified a maximum memory usage, use 8/(9+eps) of that # for the countgraph and 1/(9+eps) for the tracking nodegraph # `eps` is used to account for the memory used by the python interpreter countgraph = khmer_args.create_countgraph(args, multiplier=8 / (9. + 0.3)) countgraph.set_use_bigcount(args.bigcount) log_info('building k-mer tracking graph') tracking = khmer_args.create_matching_nodegraph(countgraph) log_info('kmer_size: {ksize}', ksize=countgraph.ksize()) log_info('k-mer countgraph sizes: {sizes}', sizes=countgraph.hashsizes()) log_info('outputting to {output}', output=args.output_histogram_filename) # start loading rparser = khmer.ReadParser(args.input_sequence_filename) threads = [] log_info('consuming input, round 1 -- {input}', input=args.input_sequence_filename) for _ in range(args.threads): thread = \ threading.Thread( target=countgraph.consume_seqfile_with_reads_parser, args=(rparser, ) ) threads.append(thread) thread.start() for thread in threads: thread.join() log_info('Total number of unique k-mers: {nk}', nk=countgraph.n_unique_kmers()) abundance_lists = [] def __do_abundance_dist__(read_parser): abundances = countgraph.abundance_distribution_with_reads_parser( read_parser, tracking) abundance_lists.append(abundances) log_info('preparing hist from {seqfile}...', seqfile=args.input_sequence_filename) rparser = khmer.ReadParser(args.input_sequence_filename) threads = [] log_info('consuming input, round 2 -- {filename}', filename=args.input_sequence_filename) for _ in range(args.threads): thread = \ threading.Thread( target=__do_abundance_dist__, args=(rparser, ) ) threads.append(thread) thread.start() for thread in threads: thread.join() assert len(abundance_lists) == args.threads, len(abundance_lists) abundance = {} for abundance_list in abundance_lists: for i, count in enumerate(abundance_list): abundance[i] = abundance.get(i, 0) + count total = sum(abundance.values()) if 0 == total: log_error("ERROR: abundance distribution is uniformly zero; " "nothing to report.") log_error("\tPlease verify that the input files are valid.") sys.exit(1) sofar = 0 for _, i in sorted(abundance.items()): if i == 0 and not args.output_zero: continue sofar += i frac = sofar / float(total) hist_fp_csv.writerow([_, i, sofar, round(frac, 3)]) if sofar == total: break if args.savegraph is not None: log_info('Saving k-mer countgraph to {savegraph}', savegraph=args.savegraph) countgraph.save(args.savegraph) log_info('wrote to: {output}', output=args.output_histogram_filename)
def test_create_matching_nodegraph(ntables, targetsize): cg = khmer.Countgraph(31, targetsize, ntables) ng = create_matching_nodegraph(cg) assert cg.hashsizes() == ng.hashsizes()
def main(): # pylint: disable=too-many-locals,too-many-branches args = sanitize_help(get_parser()).parse_args() graph_type = 'smallcountgraph' if args.small_count else 'countgraph' configure_logging(args.quiet) report_on_config(args, graph_type) check_input_files(args.input_sequence_filename, args.force) if args.savegraph is not None: graphsize = calculate_graphsize(args, graph_type) check_space_for_graph(args.savegraph, graphsize, args.force) if (not args.squash_output and os.path.exists(args.output_histogram_filename)): log_error('ERROR: {output} exists; not squashing.', output=args.output_histogram_filename) sys.exit(1) else: hist_fp = open(args.output_histogram_filename, 'w') hist_fp_csv = csv.writer(hist_fp) # write headers: hist_fp_csv.writerow(['abundance', 'count', 'cumulative', 'cumulative_fraction']) log_info('making countgraph') # In case the user specified a maximum memory usage, use 8/(9+eps) of that # for the countgraph and 1/(9+eps) for the tracking nodegraph # `eps` is used to account for the memory used by the python interpreter countgraph = khmer_args.create_countgraph(args, multiplier=8 / (9. + 0.3)) log_info('building k-mer tracking graph') tracking = khmer_args.create_matching_nodegraph(countgraph) log_info('kmer_size: {ksize}', ksize=countgraph.ksize()) log_info('k-mer countgraph sizes: {sizes}', sizes=countgraph.hashsizes()) log_info('outputting to {output}', output=args.output_histogram_filename) # start loading rparser = khmer.ReadParser(args.input_sequence_filename) threads = [] log_info('consuming input, round 1 -- {input}', input=args.input_sequence_filename) for _ in range(args.threads): thread = \ threading.Thread( target=countgraph.consume_seqfile, args=(rparser, ) ) threads.append(thread) thread.start() for thread in threads: thread.join() log_info('Total number of unique k-mers: {nk}', nk=countgraph.n_unique_kmers()) abundance_lists = [] def __do_abundance_dist__(read_parser): abundances = countgraph.abundance_distribution( read_parser, tracking) abundance_lists.append(abundances) log_info('preparing hist from {seqfile}...', seqfile=args.input_sequence_filename) rparser = khmer.ReadParser(args.input_sequence_filename) threads = [] log_info('consuming input, round 2 -- {filename}', filename=args.input_sequence_filename) for _ in range(args.threads): thread = \ threading.Thread( target=__do_abundance_dist__, args=(rparser, ) ) threads.append(thread) thread.start() for thread in threads: thread.join() assert len(abundance_lists) == args.threads, len(abundance_lists) abundance = {} for abundance_list in abundance_lists: for i, count in enumerate(abundance_list): abundance[i] = abundance.get(i, 0) + count total = sum(abundance.values()) if 0 == total: log_error("ERROR: abundance distribution is uniformly zero; " "nothing to report.") log_error("\tPlease verify that the input files are valid.") sys.exit(1) sofar = 0 for _, i in sorted(abundance.items()): if i == 0 and not args.output_zero: continue sofar += i frac = sofar / float(total) hist_fp_csv.writerow([_, i, sofar, round(frac, 3)]) if sofar == total: break if args.savegraph is not None: log_info('Saving k-mer countgraph to {savegraph}', savegraph=args.savegraph) countgraph.save(args.savegraph) log_info('wrote to: {output}', output=args.output_histogram_filename)
def sequenceToHistograma(nome, sequence): ksize = 3 nkmers = 4**ksize tablesize = nkmers + 10 # Initialize countgraph cg = khmer.Countgraph(ksize, tablesize, 1) # print('Created a countgraph with', cg.hashsizes(), 'buckets') # start loading auxNome = "sequenciaAuxliar.fa" aux = open(auxNome, 'w') aux.write(nome) aux.write(sequence + "\n") aux.close() rparser = khmer.ReadParser(auxNome) # rparser2 = rparser # aux.closes # os.remove(auxNome) # rparser = khmer.ReadParser(sequence) threads = [] for _ in range(1): thread = \ threading.Thread( target=cg.consume_seqfile_with_reads_parser, args=(rparser, ) ) threads.append(thread) thread.start() for thread in threads: thread.join() # print('unique', cg.n_unique_kmers()) h = Histograma(nome, cg.n_unique_kmers(), nkmers, tablesize, len(sequence)) abundance_lists = [] tracking = khmer_args.create_matching_nodegraph(cg) def __do_abundance_dist__(read_parser): abundances = cg.abundance_distribution_with_reads_parser( read_parser, tracking) abundance_lists.append(abundances) # aux = open(auxNome, 'r') rparser2 = khmer.ReadParser(auxNome) # TESTE AQUI os.remove(auxNome) threads = [] for _ in range(1): thread = \ threading.Thread( target=__do_abundance_dist__, args=(rparser2, ) ) threads.append(thread) thread.start() for thread in threads: thread.join() assert len(abundance_lists) == 1, len(abundance_lists) abundance = {} for abundance_list in abundance_lists: for i, count in enumerate(abundance_list): abundance[i] = abundance.get(i, 0) + count total = sum(abundance.values()) if 0 == total: print("ERROR: abundance distribution is uniformly zero; " "nothing to report.") print("\tPlease verify that the input files are valid.") return 0 sofar = 0 line = 0 for _, i in sorted(abundance.items()): if i == 0 and line < h.tablesize: continue sofar += i frac = sofar / float(total) #hist_fp_csv.writerow([_, i, sofar, round(frac, 3)]) # print(line, tablesize, [_, i, sofar, round(frac, 3)]) h.histo[line][0] = _ h.histo[line][1] = i h.histo[line][2] = sofar h.histo[line][3] = round(frac, 3) line = line + 1 if sofar == total: break return h
def test_create_matching_nodegraph(ntables, targetsize): cg = khmer.Countgraph(31, targetsize, ntables) ng = create_matching_nodegraph(cg) assert cg.hashsizes() == ng.hashsizes()