def add_batch(batch_index, pCS, orphans, fasta_d, cpus, dun_use_partial):
"""
1. align batch<i>.fasta against seed<i>.S.fasta, process -> write remains to batch<i>.remains.fasta
2. align batch<i>.remains.fasta against seed<i>.orphans.fasta -> write remains to batch<i>.remains2.fasta
3. self align batch<i>.remains2.fasta -> combine remains+orphans to new orphans
4. write out seed<i+1>.S.fasta and seed<i+1>.orphans.fasta
"""
cur_file = "batch{0}.fasta".format(batch_index)
seqids = set([r.id for r in SeqIO.parse(open(cur_file), 'fasta')])
o = ar.run_minimap(cur_file,
"seed{0}.S.fasta".format(batch_index),
cpus=cpus)
print("processing", o, file=sys.stderr)
pCS, remains = sp.process_align_to_pCS(o,
seqids,
pCS,
MiniReader,
dun_use_partial=dun_use_partial)
print("pCS: {0}, tucked: {1}, orphans: {2}, remains: {3}".format( \
len(pCS.S), sum(v == 'T' for v in pCS.seq_stat.values()), len(orphans), len(remains)), file=sys.stderr)
# write batch<i>.remains.fasta
cur_file = "batch{0}.remains.fasta".format(batch_index)
FileIO.write_seqids_to_fasta(remains, cur_file, fasta_d)
o = ar.run_minimap(cur_file,
"seed{0}.orphans.fasta".format(batch_index),
cpus=cpus)
print("processing", o, file=sys.stderr)
pCS, orphans, remains = sp.process_align_to_orphan(
o, remains, orphans, pCS, MiniReader, dun_use_partial=dun_use_partial)
print("pCS: {0}, tucked: {1}, orphans: {2}, remains: {3}".format( \
len(pCS.S), sum(v == 'T' for v in pCS.seq_stat.values()), len(orphans), len(remains)), file=sys.stderr)
# write batch<i>.remains2.fasta and self align
cur_file = "batch{0}.remains2.fasta".format(batch_index)
FileIO.write_seqids_to_fasta(remains, cur_file, fasta_d)
o = ar.run_minimap(cur_file, cur_file, cpus=cpus)
print("processing", o, file=sys.stderr)
pCS, remains = sp.process_self_align_into_seed(
o, remains, MiniReader, pCS, dun_use_partial=dun_use_partial)
print("pCS: {0}, tucked: {1}, orphans: {2}, remains: {3}".format( \
len(pCS.S), sum(v == 'T' for v in pCS.seq_stat.values()), len(orphans), len(remains)), file=sys.stderr)
# combine remains+orphans to new orphans
orphans = orphans.union(remains)
FileIO.write_preClusterSet_to_fasta(
pCS, "seed{0}.S.fasta".format(batch_index + 1), fasta_d)
FileIO.write_seqids_to_fasta(
orphans, "seed{0}.orphans.fasta".format(batch_index + 1), fasta_d)
return pCS, orphans
def add_batch(batch_index, pCS, orphans, fasta_d, cpus, dun_use_partial):
"""
1. align batch<i>.fasta against seed<i>.S.fasta, process -> write remains to batch<i>.remains.fasta
2. align batch<i>.remains.fasta against seed<i>.orphans.fasta -> write remains to batch<i>.remains2.fasta
3. self align batch<i>.remains2.fasta -> combine remains+orphans to new orphans
4. write out seed<i+1>.S.fasta and seed<i+1>.orphans.fasta
"""
cur_file = "batch{0}.fasta".format(batch_index)
seqids = set([r.id for r in SeqIO.parse(open(cur_file), 'fasta')])
o = ar.run_minimap(cur_file, "seed{0}.S.fasta".format(batch_index), cpus=cpus)
print >> sys.stderr, "processing", o
pCS, remains = sp.process_align_to_pCS(o, seqids, pCS, MiniReader, dun_use_partial=dun_use_partial)
print >> sys.stderr, "pCS: {0}, tucked: {1}, orphans: {2}, remains: {3}".format( \
len(pCS.S), sum(v == 'T' for v in pCS.seq_stat.itervalues()), len(orphans), len(remains))
# write batch<i>.remains.fasta
cur_file = "batch{0}.remains.fasta".format(batch_index)
FileIO.write_seqids_to_fasta(remains, cur_file, fasta_d)
o = ar.run_minimap(cur_file, "seed{0}.orphans.fasta".format(batch_index), cpus=cpus)
print >> sys.stderr, "processing", o
pCS, orphans, remains = sp.process_align_to_orphan(o, remains, orphans, pCS, MiniReader, dun_use_partial=dun_use_partial)
print >> sys.stderr, "pCS: {0}, tucked: {1}, orphans: {2}, remains: {3}".format( \
len(pCS.S), sum(v == 'T' for v in pCS.seq_stat.itervalues()), len(orphans), len(remains))
# write batch<i>.remains2.fasta and self align
cur_file = "batch{0}.remains2.fasta".format(batch_index)
FileIO.write_seqids_to_fasta(remains, cur_file, fasta_d)
o = ar.run_minimap(cur_file, cur_file, cpus=cpus)
print >> sys.stderr, "processing", o
pCS, remains = sp.process_self_align_into_seed(o, remains, MiniReader, pCS, dun_use_partial=dun_use_partial)
print >> sys.stderr, "pCS: {0}, tucked: {1}, orphans: {2}, remains: {3}".format( \
len(pCS.S), sum(v == 'T' for v in pCS.seq_stat.itervalues()), len(orphans), len(remains))
# combine remains+orphans to new orphans
orphans = orphans.union(remains)
FileIO.write_preClusterSet_to_fasta(pCS, "seed{0}.S.fasta".format(batch_index+1), fasta_d)
FileIO.write_seqids_to_fasta(orphans, "seed{0}.orphans.fasta".format(batch_index+1), fasta_d)
return pCS, orphans
def main(cpus, dun_make_bins=False, dun_use_partial=False, num_seqs_per_batch=100000, dun_cleanup_files=False):
print "Indexing isoseq_flnc.fasta using LazyFastaReader..."
d = LazyFastaReader('isoseq_flnc.fasta')
print "Splitting input isoseq_flnc.fasta into seed/batches..."
num_batchs = create_seed_n_batch_files(input='isoseq_flnc.fasta', fasta_d=d, seed_filename='seed0.fasta', batch_pre='batch', num_seqs_per_batch=num_seqs_per_batch)
# step1. run minimap of seed0 against itself and process
o = ar.run_minimap('seed0.fasta', 'seed0.fasta', cpus=cpus)
seqids = set([r.id for r in SeqIO.parse(open('seed0.fasta'),'fasta')])
pCS, orphans = sp.process_self_align_into_seed(o, seqids, MiniReader, dun_use_partial=dun_use_partial)
# keep stats
size_S, size_tucked, size_orphans = len(pCS.S), sum(v=='T' for v in pCS.seq_stat.itervalues()), len(orphans)
print "seed 0 initial: S {0}, tucked {1}, orphans {2}".format(size_S, size_tucked, size_orphans)
# write out seed1.S.fasta and seed1.orphans.fasta
FileIO.write_preClusterSet_to_fasta(pCS, 'seed1.S.fasta', d)
FileIO.write_seqids_to_fasta(orphans, 'seed1.orphans.fasta', d)
# step 2a. minimap batch1 against seed1.S and process
for i in xrange(1, num_batchs):
pCS, orphans = add_batch(i, pCS, orphans, d, cpus=cpus, dun_use_partial=dun_use_partial)
cleanup_precluster_intermediate_files(i)
# detect PCR chimeras from orphans
chimeras = detect_PCR_chimeras(orphans, d)
orphans = orphans.difference(chimeras)
FileIO.write_seqids_to_fasta(orphans, "preCluster_out.orphans.fasta", d)
FileIO.write_seqids_to_fasta(chimeras, "preCluster_out.chimeras.fasta", d)
tucked_seqids = []
# dump pCS, orphans, chimeras to a pickle
# can't dump yet --- since pCS is an object
#with open('preCluster.output.pickle', 'w') as f:
# dump({'pCS': pCS, 'chimeras': chimeras, 'orphans': orphans}, f)
# write CSV file
with open('preCluster.output.csv', 'w') as f:
f.write("seqid,stat\n")
for x, stat in pCS.seq_stat.iteritems():
if stat == 'T':
f.write("{0},tucked\n".format(x))
tucked_seqids.append(x)
elif stat == 'M': f.write("{0},{1}\n".format(x, pCS.seq_map[x]))
for x in orphans: f.write("{0},orphan\n".format(x))
for x in chimeras: f.write("{0},chimera\n".format(x))
# Liz: currently not using tucked...
#FileIO.write_seqids_to_fasta(tucked_seqids, "preCluster_out.tucked.fasta", d)
infof = open('preCluster.cluster_info.csv', 'w')
infof.write("cluster,size\n")
# write out a directory per preCluster cid in preCluster_out/<cid>
# Liz note: right now, write out even directories with just 1 sequence
# (we know they have "tucked" support, so can run Partial/Arrow on it)
#singlef = open("preCluster_out.singles.fasta", 'w')
for cid in pCS.S:
# if pCS.S[cid].size == 1:
# r = d[pCS.S[cid].members[0]]
# singlef.write(">{0}\n{1}\n".format(r.id, r.seq))
# else:
if True:
if not dun_make_bins:
dirname = os.path.join("preCluster_out", str(cid))
os.makedirs(dirname)
file = os.path.join(dirname, 'isoseq_flnc.fasta')
FileIO.write_seqids_to_fasta(pCS.S[cid].members, file, d)
infof.write("{0},{1}\n".format(cid, len(pCS.S[cid].members)))
#singlef.close()
infof.close()
if not dun_cleanup_files: # clean up all seed* and batch* files
for file in glob.glob('batch*fasta*'):
os.remove(file)
for file in glob.glob('seed*fasta*'):
os.remove(file)
def main(cpus, dun_make_bins=False, dun_use_partial=False, num_seqs_per_batch=100000, dun_cleanup_files=False):
print "Indexing isoseq_flnc.fasta using LazyFastaReader..."
d = LazyFastaReader('isoseq_flnc.fasta')
print "Splitting input isoseq_flnc.fasta into seed/batches..."
num_batchs = create_seed_n_batch_files(input='isoseq_flnc.fasta', fasta_d=d, seed_filename='seed0.fasta', batch_pre='batch', num_seqs_per_batch=num_seqs_per_batch)
# step1. run minimap of seed0 against itself and process
o = ar.run_minimap('seed0.fasta', 'seed0.fasta', cpus=cpus)
seqids = set([r.id for r in SeqIO.parse(open('seed0.fasta'),'fasta')])
pCS, orphans = sp.process_self_align_into_seed(o, seqids, MiniReader, dun_use_partial=dun_use_partial)
# keep stats
size_S, size_tucked, size_orphans = len(pCS.S), sum(v=='T' for v in pCS.seq_stat.itervalues()), len(orphans)
print "seed 0 initial: S {0}, tucked {1}, orphans {2}".format(size_S, size_tucked, size_orphans)
# write out seed1.S.fasta and seed1.orphans.fasta
FileIO.write_preClusterSet_to_fasta(pCS, 'seed1.S.fasta', d)
FileIO.write_seqids_to_fasta(orphans, 'seed1.orphans.fasta', d)
# step 2a. minimap batch1 against seed1.S and process
for i in xrange(1, num_batchs):
pCS, orphans = add_batch(i, pCS, orphans, d, cpus=cpus, dun_use_partial=dun_use_partial)
cleanup_precluster_intermediate_files(i)
# detect PCR chimeras from orphans
chimeras = detect_PCR_chimeras(orphans, d)
orphans = orphans.difference(chimeras)
FileIO.write_seqids_to_fasta(orphans, "preCluster_out.orphans.fasta", d)
FileIO.write_seqids_to_fasta(chimeras, "preCluster_out.chimeras.fasta", d)
tucked_seqids = []
# dump pCS, orphans, chimeras to a pickle
# can't dump yet --- since pCS is an object
#with open('preCluster.output.pickle', 'w') as f:
# dump({'pCS': pCS, 'chimeras': chimeras, 'orphans': orphans}, f)
# write CSV file
with open('preCluster.output.csv', 'w') as f:
f.write("seqid,stat\n")
for x, stat in pCS.seq_stat.iteritems():
if stat == 'T':
f.write("{0},tucked\n".format(x))
tucked_seqids.append(x)
elif stat == 'M': f.write("{0},{1}\n".format(x, pCS.seq_map[x]))
for x in orphans: f.write("{0},orphan\n".format(x))
for x in chimeras: f.write("{0},chimera\n".format(x))
# Liz: currently not using tucked...
#FileIO.write_seqids_to_fasta(tucked_seqids, "preCluster_out.tucked.fasta", d)
infof = open('preCluster.cluster_info.csv', 'w')
infof.write("cluster,size\n")
# write out a directory per preCluster cid in preCluster_out/<cid>
# Liz note: right now, write out even directories with just 1 sequence
# (we know they have "tucked" support, so can run Partial/Arrow on it)
#singlef = open("preCluster_out.singles.fasta", 'w')
for cid in pCS.S:
# if pCS.S[cid].size == 1:
# r = d[pCS.S[cid].members[0]]
# singlef.write(">{0}\n{1}\n".format(r.id, r.seq))
# else:
if True:
if not dun_make_bins:
dirname = os.path.join("preCluster_out", str(cid))
os.makedirs(dirname)
file = os.path.join(dirname, 'isoseq_flnc.fasta')
FileIO.write_seqids_to_fasta(pCS.S[cid].members, file, d)
infof.write("{0},{1}\n".format(cid, len(pCS.S[cid].members)))
#singlef.close()
infof.close()
if not dun_cleanup_files: # clean up all seed* and batch* files
for file in glob.glob('batch*fasta*'):
os.remove(file)
for file in glob.glob('seed*fasta*'):
os.remove(file)