def main():
args = sanitize_help(get_parser()).parse_args()
ksize = args.ksize
filenames = args.input_filenames
nodegraph = Nodegraph(ksize, 1, 1)
partitionmap_file = args.graphbase + '.pmap.merged'
check_input_files(partitionmap_file, args.force)
for _ in filenames:
check_input_files(_, args.force)
check_space(filenames, args.force)
print('loading partition map from:', partitionmap_file, file=sys.stderr)
nodegraph.load_partitionmap(partitionmap_file)
for infile in filenames:
print('outputting partitions for', infile, file=sys.stderr)
outfile = os.path.basename(infile) + '.part'
part_count = nodegraph.output_partitions(infile, outfile)
print('output %d partitions for %s' % (part_count, infile),
file=sys.stderr)
print('partitions are in', outfile, file=sys.stderr)
def main():
info('annotate-partitions.py', ['graph'])
args = sanitize_help(get_parser()).parse_args()
ksize = args.ksize
filenames = args.input_filenames
nodegraph = Nodegraph(ksize, 1, 1)
partitionmap_file = args.graphbase + '.pmap.merged'
check_input_files(partitionmap_file, args.force)
for _ in filenames:
check_input_files(_, args.force)
check_space(filenames, args.force)
print('loading partition map from:', partitionmap_file, file=sys.stderr)
nodegraph.load_partitionmap(partitionmap_file)
for infile in filenames:
print('outputting partitions for', infile, file=sys.stderr)
outfile = os.path.basename(infile) + '.part'
part_count = nodegraph.output_partitions(infile, outfile)
print('output %d partitions for %s' % (
part_count, infile), file=sys.stderr)
print('partitions are in', outfile, file=sys.stderr)
def test_load_graph_1():
in1 = utils.get_test_data('test-abund-read-2.fa')
out1 = utils.get_temp_filename('out.ct')
cmd = """
cat {in1} |
{scripts}/load-graph.py -x 1e3 -N 2 -k 20 {out1} - \
2> /dev/null
"""
cmd = cmd.format(scripts=scriptpath(), in1=in1, out1=out1)
print(cmd)
run_shell_cmd(cmd)
assert os.path.exists(out1)
Nodegraph.load(out1)
def test_load_graph_1():
in1 = utils.get_test_data('test-abund-read-2.fa')
out1 = utils.get_temp_filename('out.ct')
cmd = """
cat {in1} |
{scripts}/load-graph.py -x 1e3 -N 2 -k 20 {out1} - \
2> /dev/null
"""
cmd = cmd.format(scripts=scriptpath(), in1=in1, out1=out1)
print(cmd)
run_shell_cmd(cmd)
assert os.path.exists(out1)
Nodegraph.load(out1)
def test_hashbits_file_version_check():
inpath = utils.get_test_data('badversion-k12.htable')
try:
nodegraph = Nodegraph.load(inpath)
assert 0, "this should fail"
except OSError as e:
print(str(e))
def test_hashbits_file_version_check():
inpath = utils.get_test_data('badversion-k12.htable')
try:
nodegraph = Nodegraph.load(inpath)
assert 0, "this should fail"
except OSError as e:
print(str(e))
def test_nodegraph_file_type_check():
kh = khmer.Countgraph(12, 1, 1)
savepath = utils.get_temp_filename('tempcountingsave0.ct')
kh.save(savepath)
try:
nodegraph = Nodegraph.load(savepath)
assert 0, "this should fail"
except OSError as e:
print(str(e))
def test_nodegraph_file_type_check():
kh = khmer.Countgraph(12, 1, 1)
savepath = utils.get_temp_filename('tempcountingsave0.ct')
kh.save(savepath)
try:
nodegraph = Nodegraph.load(savepath)
assert 0, "this should fail"
except OSError as e:
print(str(e))
def test_read_cleaning_abundance_distribution(Countingtype):
infile = utils.get_test_data('valid-read-testing.fq')
x = Countingtype(15, *params_1m)
y = Nodegraph(15, 1, 1, primes=PRIMES_1m)
x.consume_seqfile(infile)
dist = x.abundance_distribution(infile, y)
assert dist[1] == 35 # k-mers with non-ACGTN => ignored.
assert dist[2] == 69
def test_abund_dist_A(AnyTabletype):
A_filename = utils.get_test_data('all-A.fa')
kh = AnyTabletype(4)
tracking = Nodegraph(4, 1, 1, primes=PRIMES_1m)
kh.consume_seqfile(A_filename)
dist = kh.abundance_distribution(A_filename, tracking)
print(dist[:10])
assert sum(dist) == 1
assert dist[0] == 0
def main():
info('filter-stoptags.py', ['graph'])
args = sanitize_help(get_parser()).parse_args()
stoptags = args.stoptags_file
infiles = args.input_filenames
for _ in infiles:
check_input_files(_, args.force)
check_space(infiles, args.force)
print('loading stop tags, with K', args.ksize, file=sys.stderr)
nodegraph = Nodegraph(args.ksize, 1, 1)
nodegraph.load_stop_tags(stoptags)
def process_fn(record):
name = record['name']
seq = record['sequence']
if 'N' in seq:
return None, None
trim_seq, trim_at = nodegraph.trim_on_stoptags(seq)
if trim_at >= args.ksize:
return name, trim_seq
return None, None
# the filtering loop
for infile in infiles:
print('filtering', infile, file=sys.stderr)
outfile = os.path.basename(infile) + '.stopfilt'
outfp = open(outfile, 'w')
tsp = ThreadedSequenceProcessor(process_fn)
tsp.start(verbose_loader(infile), outfp)
print('output in', outfile, file=sys.stderr)
def main():
info('filter-stoptags.py', ['graph'])
args = sanitize_help(get_parser()).parse_args()
stoptags = args.stoptags_file
infiles = args.input_filenames
for _ in infiles:
check_input_files(_, args.force)
check_space(infiles, args.force)
print('loading stop tags, with K', args.ksize, file=sys.stderr)
nodegraph = Nodegraph(args.ksize, 1, 1)
nodegraph.load_stop_tags(stoptags)
def process_fn(record):
name = record['name']
seq = record['sequence']
if 'N' in seq:
return None, None
trim_seq, trim_at = nodegraph.trim_on_stoptags(seq)
if trim_at >= args.ksize:
return name, trim_seq
return None, None
# the filtering loop
for infile in infiles:
print('filtering', infile, file=sys.stderr)
outfile = os.path.basename(infile) + '.stopfilt'
outfp = open(outfile, 'w')
tsp = ThreadedSequenceProcessor(process_fn)
tsp.start(verbose_loader(infile), outfp)
print('output in', outfile, file=sys.stderr)
def main():
args = sanitize_help(get_parser()).parse_args()
graphbase = args.graphbase
# @RamRS: This might need some more work
infiles = [graphbase, graphbase + '.tagset']
if args.stoptags:
infiles.append(args.stoptags)
for _ in infiles:
check_input_files(_, args.force)
print('loading nodegraph %s.pt' % graphbase, file=sys.stderr)
nodegraph = Nodegraph.load(graphbase)
# do we want to load stop tags, and do they exist?
if args.stoptags:
print('loading stoptags from', args.stoptags, file=sys.stderr)
nodegraph.load_stop_tags(args.stoptags)
print('loading tagset %s.tagset...' % graphbase, file=sys.stderr)
nodegraph.load_tagset(graphbase + '.tagset')
counting = khmer_args.create_countgraph(args)
# divide up into SUBSET_SIZE fragments
divvy = nodegraph.divide_tags_into_subsets(args.subset_size)
divvy = list(divvy)
# pick off the first one
if len(divvy) == 1:
start, end = 0, 0
else:
start, end = divvy[:2]
# partition!
print('doing pre-partitioning from', start, 'to', end, file=sys.stderr)
subset = nodegraph.do_subset_partition(start, end)
# now, repartition...
print('repartitioning to find HCKs.', file=sys.stderr)
nodegraph.repartition_largest_partition(counting,
EXCURSION_DISTANCE,
EXCURSION_KMER_THRESHOLD,
EXCURSION_KMER_COUNT_THRESHOLD,
subs=subset)
print('saving stop tags', file=sys.stderr)
nodegraph.save_stop_tags(graphbase + '.stoptags')
print('wrote to:', graphbase + '.stoptags', file=sys.stderr)
def test_n_occupied_save_load():
filename = utils.get_test_data('random-20-a.fa')
nodegraph = khmer.Nodegraph(20, 100000, 3)
for _, record in enumerate(screed.open(filename)):
nodegraph.consume(record.sequence)
assert nodegraph.n_occupied() == 3884
assert nodegraph.n_unique_kmers() == 3960
savefile = utils.get_temp_filename('out')
nodegraph.save(savefile)
ng2 = Nodegraph.load(savefile)
assert ng2.n_occupied() == 3884, ng2.n_occupied()
assert ng2.n_unique_kmers() == 0 # this is intended behavior, sigh.
def test_n_occupied_save_load():
filename = utils.get_test_data('random-20-a.fa')
nodegraph = khmer.Nodegraph(20, 100000, 3)
for _, record in enumerate(screed.open(filename)):
nodegraph.consume(record.sequence)
assert nodegraph.n_occupied() == 3884
assert nodegraph.n_unique_kmers() == 3960
savefile = utils.get_temp_filename('out')
nodegraph.save(savefile)
ng2 = Nodegraph.load(savefile)
assert ng2.n_occupied() == 3884, ng2.n_occupied()
assert ng2.n_unique_kmers() == 0 # this is intended behavior, sigh.
def test_get_labels_save_load():
lb_pre = GraphLabels.NodeGraphLabels(20, 1e7, 4)
filename = utils.get_test_data('test-labels.fa')
lb_pre.consume_seqfile_and_tag_with_labels(filename)
# save labels to a file
savepath = utils.get_temp_filename('saved.labels')
lb_pre.save_labels_and_tags(savepath)
# trash the old GraphLabels
del lb_pre
# create new, load labels & tags
graph = Nodegraph(20, 1e7, 4)
lb = GraphLabels.load(savepath, graph)
labels = list(lb.labels())
expected = [0, 1, 2, 3]
for e_label in expected:
assert e_label in labels
for a_label in labels:
assert a_label in expected
def main():
args = sanitize_help(get_parser()).parse_args()
basename = args.basename
filenames = [basename, basename + '.tagset']
for _ in filenames:
check_input_files(_, args.force)
print('--', file=sys.stderr)
print('SUBSET SIZE', args.subset_size, file=sys.stderr)
print('N THREADS', args.threads, file=sys.stderr)
if args.stoptags:
print('stoptag file:', args.stoptags, file=sys.stderr)
print('--', file=sys.stderr)
print('loading nodegraph %s' % basename, file=sys.stderr)
nodegraph = Nodegraph.load(basename)
nodegraph.load_tagset(basename + '.tagset')
# do we want to load stop tags, and do they exist?
if args.stoptags:
print('loading stoptags from', args.stoptags, file=sys.stderr)
nodegraph.load_stop_tags(args.stoptags)
# do we want to exhaustively traverse the graph?
stop_big_traversals = args.no_big_traverse
if stop_big_traversals:
print('** This script brakes for lumps:',
' stop_big_traversals is true.', file=sys.stderr)
else:
print('** Traverse all the things:',
' stop_big_traversals is false.', file=sys.stderr)
#
# now, partition!
#
# divide the tags up into subsets
divvy = nodegraph.divide_tags_into_subsets(int(args.subset_size))
divvy = list(divvy)
n_subsets = len(divvy)
divvy.append(0)
# build a queue of tasks:
worker_q = queue.Queue()
# break up the subsets into a list of worker tasks
for _ in range(0, n_subsets):
start = divvy[_]
end = divvy[_ + 1]
worker_q.put((nodegraph, _, start, end))
print('enqueued %d subset tasks' % n_subsets, file=sys.stderr)
open('%s.info' % basename, 'w').write('%d subsets total\n' % (n_subsets))
n_threads = args.threads
if n_subsets < n_threads:
n_threads = n_subsets
# start threads!
print('starting %d threads' % n_threads, file=sys.stderr)
print('---', file=sys.stderr)
threads = []
for _ in range(n_threads):
cur_thrd = threading.Thread(target=worker, args=(worker_q, basename,
stop_big_traversals))
threads.append(cur_thrd)
cur_thrd.start()
print('done starting threads', file=sys.stderr)
# wait for threads
for _ in threads:
_.join()
print('---', file=sys.stderr)
print('done making subsets! see %s.subset.*.pmap' %
(basename,), file=sys.stderr)
def test_bad_primes():
try:
Nodegraph(6, 1, 1, primes=["a", "b", "c"])
assert 0, "this should fail"
except TypeError as e:
print(str(e))
def main():
parser = get_parser()
parser.epilog = parser.epilog.replace(
":doc:`partitioning-big-data`",
"http://khmer.readthedocs.io/en/stable/user/"
"partitioning-big-data.html"
)
args = sanitize_help(parser).parse_args()
graphbase = args.graphbase
# @RamRS: This might need some more work
infiles = [graphbase, graphbase + '.tagset']
if os.path.exists(graphbase + '.stoptags'):
infiles.append(graphbase + '.stoptags')
for _ in infiles:
check_input_files(_, args.force)
check_space(infiles, args.force)
print('loading k-mer nodegraph %s' % graphbase, file=sys.stderr)
graph = Nodegraph.load(graphbase)
print('loading tagset %s.tagset...' % graphbase, file=sys.stderr)
graph.load_tagset(graphbase + '.tagset')
initial_stoptags = False # @CTB regularize with make-initial
if os.path.exists(graphbase + '.stoptags'):
print('loading stoptags %s.stoptags' % graphbase, file=sys.stderr)
graph.load_stop_tags(graphbase + '.stoptags')
initial_stoptags = True
pmap_files = glob.glob(args.graphbase + '.subset.*.pmap')
print('loading %d pmap files (first one: %s)' %
(len(pmap_files), pmap_files[0]), file=sys.stderr)
print('---', file=sys.stderr)
print('output stoptags will be in',
graphbase + '.stoptags', file=sys.stderr)
if initial_stoptags:
print(
'(these output stoptags will include the already-loaded set)',
file=sys.stderr)
print('---', file=sys.stderr)
# create countgraph
ksize = graph.ksize()
counting = khmer_args.create_countgraph(args, ksize=ksize)
# load & merge
for index, subset_file in enumerate(pmap_files):
print('<-', subset_file, file=sys.stderr)
subset = SubsetPartition.load(subset_file, graph)
print('** repartitioning subset... %s' % subset_file, file=sys.stderr)
graph.repartition_largest_partition(counting,
EXCURSION_DISTANCE,
EXCURSION_KMER_THRESHOLD,
EXCURSION_KMER_COUNT_THRESHOLD,
subs=subset)
print('** merging subset... %s' % subset_file, file=sys.stderr)
graph.merge_subset(subset)
print('** repartitioning, round 2... %s' %
subset_file, file=sys.stderr)
size = \
graph.repartition_largest_partition(counting,
EXCURSION_DISTANCE,
EXCURSION_KMER_THRESHOLD,
EXCURSION_KMER_COUNT_THRESHOLD)
print('** repartitioned size:', size, file=sys.stderr)
print('saving stoptags binary', file=sys.stderr)
graph.save_stop_tags(graphbase + '.stoptags')
os.rename(subset_file, subset_file + '.processed')
print('(%d of %d)\n' % (index, len(pmap_files)), file=sys.stderr)
print('done!', file=sys.stderr)
def main():
parser = get_parser()
parser.epilog = parser.epilog.replace(
":doc:`partitioning-big-data`",
"http://khmer.readthedocs.io/en/stable/user/"
"partitioning-big-data.html")
args = sanitize_help(parser).parse_args()
graphbase = args.graphbase
# @RamRS: This might need some more work
infiles = [graphbase, graphbase + '.tagset']
if os.path.exists(graphbase + '.stoptags'):
infiles.append(graphbase + '.stoptags')
for _ in infiles:
check_input_files(_, args.force)
check_space(infiles, args.force)
print('loading k-mer nodegraph %s' % graphbase, file=sys.stderr)
graph = Nodegraph.load(graphbase)
print('loading tagset %s.tagset...' % graphbase, file=sys.stderr)
graph.load_tagset(graphbase + '.tagset')
initial_stoptags = False # @CTB regularize with make-initial
if os.path.exists(graphbase + '.stoptags'):
print('loading stoptags %s.stoptags' % graphbase, file=sys.stderr)
graph.load_stop_tags(graphbase + '.stoptags')
initial_stoptags = True
pmap_files = glob.glob(args.graphbase + '.subset.*.pmap')
print('loading %d pmap files (first one: %s)' %
(len(pmap_files), pmap_files[0]),
file=sys.stderr)
print('---', file=sys.stderr)
print('output stoptags will be in',
graphbase + '.stoptags',
file=sys.stderr)
if initial_stoptags:
print('(these output stoptags will include the already-loaded set)',
file=sys.stderr)
print('---', file=sys.stderr)
# create countgraph
ksize = graph.ksize()
counting = khmer_args.create_countgraph(args, ksize=ksize)
# load & merge
for index, subset_file in enumerate(pmap_files):
print('<-', subset_file, file=sys.stderr)
subset = SubsetPartition.load(subset_file, graph)
print('** repartitioning subset... %s' % subset_file, file=sys.stderr)
graph.repartition_largest_partition(counting,
EXCURSION_DISTANCE,
EXCURSION_KMER_THRESHOLD,
EXCURSION_KMER_COUNT_THRESHOLD,
subs=subset)
print('** merging subset... %s' % subset_file, file=sys.stderr)
graph.merge_subset(subset)
print('** repartitioning, round 2... %s' % subset_file,
file=sys.stderr)
size = \
graph.repartition_largest_partition(counting,
EXCURSION_DISTANCE,
EXCURSION_KMER_THRESHOLD,
EXCURSION_KMER_COUNT_THRESHOLD)
print('** repartitioned size:', size, file=sys.stderr)
print('saving stoptags binary', file=sys.stderr)
graph.save_stop_tags(graphbase + '.stoptags')
os.rename(subset_file, subset_file + '.processed')
print('(%d of %d)\n' % (index, len(pmap_files)), file=sys.stderr)
print('done!', file=sys.stderr)
