def main(args): rf = args[2] dir = args[3] CreateLog(dir) disjointigs = ContigCollection().loadFromFasta(open(args[1], "r"), num_names=False) basic.ensure_dir_existance(dir) aligner = Aligner(DirDistributor(os.path.join(dir, "alignments"))) clen = 5000000 reads = ReadCollection().loadFromFasta(open(rf, "r")) tlen0 = sum(map(len, reads)) for i in range(10): good_reads = set() for al in aligner.localAlign(reads, disjointigs): if not al.contradictingRTC(al.seg_to.contig.asSegment(), 500): good_reads.add(al.seg_from.contig.id) rf = os.path.join(dir, "reads" + str(i) + ".fasta") reads = reads.filter(lambda read: read.id not in good_reads).cleanCopy() tlen = sum(map(len, reads)) reads.print_fasta(open(rf, "w")) l = tlen * clen / tlen0 assembly_dir = os.path.join(dir, "assembly" + str(i)) subprocess.check_call(["./bin/flye", "-o", assembly_dir, "-t", "8", "--pacbio-raw", rf, "--genome-size", str(l), "--no-trestle"]) df= os.path.join(assembly_dir, "10-consensus", "consensus.fasta") disjointigs.addAll(ContigCollection().loadFromFasta(open(df, "r"), num_names=False)) df = os.path.join(dir, "df" + str(i) + ".fasta") disjointigs.print_fasta(open(df, "w"))
def main(dir, contigs_file1, contigs_file2, unique_contigs_file): CreateLog(dir) sys.stdout.level = LogPriority.warning unique = ContigCollection().loadFromFasta( open(unique_contigs_file, "r"), False).filter(lambda contig: len(contig) > 5000) aligner = Aligner(DirDistributor(os.path.join(dir, "alignments"))) contigs1 = ContigCollection().loadFromFasta(open(contigs_file1, "r"), False) cals1 = list(aligner.overlapAlign(unique.unique(), contigs1)) transfers1, term1, all1 = extract_transfers(contigs1, cals1) contigs2 = ContigCollection().loadFromFasta(open(contigs_file2, "r"), False) cals2 = list(aligner.overlapAlign(unique.unique(), contigs2)) transfers2, term2, all2 = extract_transfers(contigs2, cals2) missing1 = [] missing2 = [] different = dict() unresolved1 = [] unresolved2 = [] same = [] for ucontig in list(unique) + [contig.rc for contig in unique]: uid = ucontig.id in1 = uid in all1 in2 = uid in all2 if not in1 and not in2: continue if not in1: missing1.append(uid) elif not in2: missing2.append(uid) else: if all1[uid][0] == all2[uid][0]: same.append(uid) elif uid in transfers1 and uid in transfers2: different[uid] = (all1[uid][0], all2[uid][0]) elif uid in transfers1: unresolved2.append(uid) elif uid in transfers2: unresolved1.append(uid) out = open(os.path.join(dir, "contigs.txt"), "w") out.write("Different: " + str(different) + "\n") out.write("Unresolved1: " + str(unresolved1) + "\n") out.write("Unresolved2: " + str(unresolved2) + "\n") out.write("Same: " + str(same) + "\n") out.write("Missing1: " + str(missing1) + "\n") out.write("Missing2: " + str(missing2) + "\n") out.write("Contig1 transfers: " + str(transfers1) + "\n") out.write("Contig1 term: " + str(term1) + "\n") out.write("Contig2 transfers: " + str(transfers2) + "\n") out.write("Contig2 term: " + str(term2) + "\n") out.close() print contigs_file1, contigs_file2 print len(different), len(unresolved1), len(unresolved2), len( missing1), len(missing2), len(same)
def main(dir, contigs_files, reference_file, unique_contigs_file): CreateLog(dir) sys.stdout.level = LogPriority.warning ref = ContigCollection().loadFromFasta(open(reference_file, "r"), False) unique = ContigCollection().loadFromFasta(open(unique_contigs_file, "r"), False).filter(lambda contig: len(contig) > 5000) aligner = Aligner(DirDistributor(os.path.join(dir, "alignments"))) ref_als= list(aligner.overlapAlign(unique.unique(), ref)) ref_transfers, ref_term, all_ref_als = extract_transfers(ref, ref_als) for uid in ref_term: ref_transfers[uid] = ref_term[uid] print "#", "file", "wrong", "unresolved", "correct", "missing" for i, contigs_file in enumerate(contigs_files): contigs = ContigCollection().loadFromFasta(open(contigs_file, "r"), False) contig_als = list(aligner.overlapAlign(unique.unique(), contigs)) contig_transfers, contig_term, all_contig_als = extract_transfers(contigs, contig_als) missing = [] wrong = dict() unresolved = [] correct = [] for uid in ref_transfers: if uid not in contig_transfers and uid not in contig_term: # print uid, "missing" missing.append(uid) elif uid in contig_transfers: if ref_transfers[uid][0] == contig_transfers[uid][0]: # print uid, "correct" correct.append(uid) else: # print uid, "wrong", ref_transfers[uid][0].id, contig_transfers[uid][0].id wrong[uid] = (ref_transfers[uid][0], contig_transfers[uid][0]) else: if ref_transfers[uid][0] == contig_term[uid][0]: # print uid, "correct" correct.append(uid) else: # print uid, "unresolved" unresolved.append(uid) out = open(os.path.join(dir, "contigs_" + str(i) +".txt"), "w") out.write("Wrong: " + str(wrong) + "\n") out.write("Unresolved: " + str(unresolved) + "\n") out.write("Correct: " + str(correct) + "\n") out.write("Missing: " + str(missing) + "\n") out.write("Contig transfers: " + str(contig_transfers) + "\n") out.write("Contig term: " + str(contig_term) + "\n") out.write("Ref transfers: " + str(ref_transfers) + "\n") out.write("Ref als:\n") for c in all_ref_als: out.write(str(c) + "\n") out.write("Contig als:\n") for c in all_contig_als: out.write(str(c) + "\n") out.close() print "result", i, contigs_file, len(wrong), len(unresolved), len(correct), len(missing)
def main(contig_file, reads_file, sam_file, dir, contig_id): # type: (str, str, str, str, str) -> None basic.ensure_dir_existance(dir) contigs = ContigCollection() contigs.loadFromFasta(open(contig_file, "r")) print "Contigs loaded" contig = contigs[contig_id] read_names = set() for rec in Samfile(open(sam_file, "r")): read_names.add(rec.query_name) reads = ReadCollection() cnt = 0 for rec in SeqIO.parse_fasta(open(reads_file, "r")): if rec.id in read_names: rec.id = "Read" + str(cnt) reads.add(AlignedRead(rec)) cnt += 1 reads.print_fasta(open(os.path.join(dir, "reads.fasta"), "w")) print "Reads loaded", len(reads) reads.addAllRC() print "RC added", len(reads) aligner = Aligner(DirDistributor(os.path.join(dir, "alignments"))) aligner.alignReadCollection(reads, contigs) print "Reads aligned", len(reads) reads = reads.inter(contig.asSegment()) print "Reads filtered", len(reads) sorted_reads = sorted(list(reads.reads.values()), key = lambda read: read.alignmentsTo(contig.asSegment()).next().seg_to.left) for read in sorted_reads: print read for al in read.alignmentsTo(contig.asSegment()): print "\n".join(al.asMatchingStrings())
def main(contigs_file, parts_file, dir): contigs = ContigCollection().loadFromFasta(open(contigs_file, "r")) parts = ContigCollection().loadFromFasta(open(parts_file, "r")) basic.CreateLog(dir) aligner = Aligner(DirDistributor(dir)) res = dict() for al in aligner.localAlign(parts, contigs): if al.seg_to.contig.id not in res: res[al.seg_to.contig.id] = [] res[al.seg_to.contig.rc.id] = [] res[al.seg_to.contig.id].append(al) res[al.seg_to.contig.rc.id].append(al.rc) for cname, arr in res.items(): print cname arr = filter( lambda al: len(al.seg_to) > min( len(al.seg_to.contig) - 1000, 5000), arr) arr = sorted(arr, key=lambda al: al.seg_to.left) print arr
def alignReadsToSegments(self, reads, segments): # type: (ReadCollection, Iterable[Segment]) -> None segments = list(segments) seg_dict = dict() for i, seg in enumerate(segments): seg_dict[str(i + 1)] = seg contigs = map(lambda (i, seg): Contig(seg.Seq(), str(i + 1)), enumerate(segments)) read_collection = ReadCollection().extendClean(reads) self.alignReadCollection(read_collection, ContigCollection(contigs)) read_collection.contigsAsSegments(seg_dict) reads.mergeAlignments(read_collection)
def main(reads_file, ref_file, dir, error_rate): sys.stderr.write("Reading reference" + "\n") ref = sorted(list(SeqIO.parse_fasta(open(ref_file, "r"))), key=lambda rec: len(rec))[-1] ref = Contig(ref.seq, ref.id) refs = ContigCollection() for i in range(0, len(ref) - 500, 500): if random.random() > 0.95: tmp = list(ref.segment(i, i + 500).Seq()) for j in range(error_rate * 500 / 100): pos = random.randint(0, 499) tmp[pos] = basic.rc[tmp[pos]] refs.add( Contig("".join(tmp), ref.id + "(" + str(i) + "," + str(i + 500) + ")")) refs.print_names(sys.stderr) sys.stderr.write("Reading reads" + "\n") reads = ReadCollection() reads.loadFromFasta(open(reads_file, "r")) sys.stderr.write("Aligning reads" + "\n") basic.ensure_dir_existance(dir) aligner = Aligner(DirDistributor(dir)) aligner.alignReadCollection(reads, refs) sys.stderr.write("Analysing alignments" + "\n") alignments = [] for read in reads: alignments.extend(read.alignments) alignments = filter(lambda al: len(al) > 450, alignments) alignments = sorted(alignments, key=lambda al: (al.seg_to.contig.id, al.seg_from.contig.id)) scorer = Scorer() scorer.scores.homo_score = 3 scorer.scores.ins_score = 5 scorer.scores.del_score = 5 cnt = 0 for contig, iter in itertools.groupby(alignments, key=lambda al: al.seg_to.contig): iter = list(iter) sys.stderr.write(str(contig) + " " + str(len(iter)) + "\n") if len(iter) < 150: for al in iter: print scorer.accurateScore(al.matchingSequence(), params.alignment_correction_radius) cnt += 1 if cnt >= 5000: break if cnt >= 5000: break
def alignReadCollection(self, reads_collection, contigs): # type: (ReadCollection, Iterable[Contig]) -> None contig_collection = ContigCollection(contigs) contig_ids = set() for contig in contigs: if contig.rc.id not in contig_ids: contig_ids.add(contig.id) read_ids = set() for read in reads_collection: if read.rc.id not in read_ids: read_ids.add(read.id) contigs = filter(lambda contig: contig.id in contig_ids, contigs) reads = filter(lambda read: read.id in read_ids, reads_collection) reads_collection.fillFromSam(self.align(reads, contigs, "local"), contig_collection)
def main(ref_file, contig_size, rlen, cov, dir): basic.ensure_dir_existance(dir) all_contigs = ContigCollection().loadFromFasta(open(ref_file, "r"), False) contig_file_name = os.path.join(dir, "contigs.fasta") contig_file = open(contig_file_name, "w") reads_file_name = os.path.join(dir, "reads.fasta") reads_file = open(reads_file_name, "w") for ref in all_contigs.unique(): if len(ref) < contig_size: continue SeqIO.write(ref, contig_file, "fasta") for i in range(0, len(ref), max(1, rlen / cov)): read = ref.segment(i, min(i + rlen, len(ref))).asNamedSequence() SeqIO.write(read, reads_file, "fasta") reads_file.close() contig_file.close() print "Done" print contig_file_name print reads_file_name
def main(ref_file, segment, dir): ref = ContigCollection().loadFromFasta(open(ref_file, "r"), False) chr1 = ref["chr1"] if segment[0] < 0: segment = (-segment[0], -segment[1]) chr1 = chr1.rc reads = ReadCollection() reads_list = [] for i in range(segment[0], segment[1], 500): read = reads.addNewRead(Segment(chr1, i, i + 500).asNamedSequence()) reads_list.append(read) chr1.seq = chr1.seq[:segment[0]] + "N" * (segment[1] - segment[0]) + chr1.seq[segment[1]:] chr1.rc.seq = basic.RC(chr1.seq) basic.ensure_dir_existance(dir) aligner = Aligner(DirDistributor(dir)) aligner.alignReadCollection(reads, ref) out = sys.stdout for read in reads_list: # print read out.write(str(len(read.alignments)) + " " + str(max([0] + map(lambda al: al.percentIdentity(), read.alignments))) + "\n") out.close()
def loadAll(handler): # type: (TokenReader) -> Tuple[Params, Aligner, ContigCollection, ReadCollection, DisjointigCollection, NewLineStorage, LineDotPlot] cl_params = Params() cl_params.load(handler) aligner = Aligner.load(handler) sys.stdout.info("Loading contigs") contigs = ContigCollection() contigs.load(handler) sys.stdout.info("Loading reads") reads = CreateReadCollection(cl_params.reads_file, cl_params.downsample) reads.loadFromFasta(open(cl_params.reads_file, "r"), downsample=params.downsample) tmp_reads = reads.copy().addAllRC() sys.stdout.info("Loading disjointigs") disjointigs = DisjointigCollection() disjointigs.load(handler, tmp_reads) sys.stdout.info("Loading lines") lines = NewLineStorage(disjointigs, aligner) lines.load(handler, tmp_reads, contigs) sys.stdout.info("Loading dot plot") dot_plot = LineDotPlot(lines, aligner) dot_plot.load(handler) sys.stdout.info("Loading finished") return cl_params, aligner, contigs, reads, disjointigs, lines, dot_plot
def CreateContigCollection(graph_file, contigs_file, min_cov, aligner, polisher, reads, force_unique, all_unique): sys.stdout.info("Creating contig collection") if force_unique is None and not all_unique: graph = SimpleGraph().ReadDot(graph_file) graph.FillSeq(contigs_file) covs = [] for e in graph.e.values(): covs.append((e.len, e.cov)) tmp_cov = [] total = sum(l for c,l in covs) / 2 for l, c in sorted(covs)[::-1]: if total < 0: break tmp_cov.append((l, c)) total -= l avg_cov = float(sum([l * c for l, c in tmp_cov])) / sum(l for l, c in tmp_cov) sys.stdout.info("Average coverage determined:", avg_cov) nonunique = set() for edge in graph.e.values(): if edge.unique and edge.len < 20000 and len(graph.v[edge.start].out) > 1: if edge.cov >= min_cov and (edge.cov < 0.8 * avg_cov or edge.len > 40000): alter = ContigStorage() for e in graph.v[edge.start].out: if e != edge: alter.add(Contig(e.seq, e.id)) for al in aligner.localAlign([Contig(edge.seq, edge.id)], alter):#type: AlignmentPiece if al.percentIdentity() > 0.98 and (al.seg_from.left < 100 and al.seg_to.left < 100 and len(al) > min(500, edge.len)): nonunique.add(edge.id) nonunique.add(basic.Reverse(edge.id)) contigs = ContigCollection() for edge in graph.e.values(): if basic.isCanonocal(edge.id): if edge.unique and (edge.len > params.min_isolated_length or len(graph.v[edge.end].out) > 0 or len(graph.v[edge.start].inc) > 0): if edge.cov >= min_cov and (edge.cov < 1.5 * avg_cov or edge.len > 40000): if edge.id in nonunique: sys.stdout.info("Edge removed based on alignment to alternative:", edge.id, edge.cov, edge.len) else: contigs.add(Contig(edge.seq, edge.id)) else: sys.stdout.info("Edge removed based on coverage:", edge.id, edge.cov, edge.len) elif (edge.len > 100000 and edge.cov < 1.5 * avg_cov) or (edge.len > 40000 and 1.3 * avg_cov > edge.cov > 0.7 * avg_cov): contigs.add(Contig(edge.seq, edge.id)) sys.stdout.info("Edge added based on length and coverage:", edge.id, edge.cov, edge.len) elif force_unique is not None: sys.stdout.info("Using forced unique edge set") sys.stdout.trace(force_unique) contigs = ContigCollection().loadFromFile(contigs_file).filter(lambda contig: contig.id in force_unique) else: sys.stdout.info("Considering all contigs unique") contigs = ContigCollection().loadFromFile(contigs_file) # contigs.loadFromFasta(open(contigs_file, "r"), num_names=True) # contigs = contigs.filter(lambda contig: contig.id not in nonunique and len(contig) > params.k + 20) sys.stdout.info("Created", len(contigs), "initial contigs") if not all_unique or force_unique is not None: sys.stdout.info("Polishing contigs") polished_contigs = polisher.polishMany(reads, list(contigs.unique())) contigs = ContigCollection().addAll(polished_contigs) else: sys.stdout.info("Skipping contig polishing step since manual unique contig initialization was used") return contigs
def main(argv): sys.stdout.write("Started\n") dot_file = argv[1] edge_sequences = argv[2] reference_file = argv[3] alignment_file = argv[4] edges = ParseVertices(argv[5]) output_file = argv[6] sys.stdout.write("Loading dot\n") dot = DotParser(open(dot_file, "r")).parse() edge_collection = ContigCollection().loadFromFasta( open(edge_sequences, "r"), True) graph = Graph().loadFromDot(edge_collection, dot) vertices = [graph.E[id].start.id for id in edges] graph.printToFile(sys.stdout) print vertices ref = ContigCollection().loadFromFasta(open(reference_file, "r"), False) print "Looking for relevant" pq = PriorityQueue() for v in graph.V.values(): if v.id in vertices: pq.push((0, v)) visited = [] while not pq.empty(): d, v = pq.pop() if v in visited: continue visited.append(v) for e in v.inc: print e.id, e.start.id, e.end.id if d + len(e) < dist: pq.push((d + len(e), e.start)) for e in v.out: print e.id, e.start.id, e.end.id if d + len(e) < dist: pq.push((d + len(e), e.end)) print "Visited", len(visited) print map(str, list(visited)) relevant = [] edge_alignments = ReadCollection().loadFromFasta(open(edge_sequences, "r")).addAllRC() for edge in graph.E.values(): if edge.start in visited or edge.start.rc in visited: relevant.append(edge_alignments[edge.id]) print "Loading sam" edge_alignments.fillFromSam(Samfile(open(alignment_file, "r")), ref) for rel in relevant: print rel.__str__() print "Collecting segments" segments = [] chr1 = ref["chr1"] for edge in relevant: for al in edge.alignments: print al if al.seg_from.inter(edge.prefix(dist)): l = dist - al.seg_from.left contig = al.seg_to.contig start = al.seg_to.left segments.append( Segment(contig, start, min(start + l, len(contig)))) print segments[-1] tmp = [] print "Rotating" for seg in segments: if seg.contig != chr1: seg = seg.RC() if seg.contig != chr1: print "WARNING", seg tmp.append(seg) segments = sorted(tmp, key=lambda seg: seg.left) print "All relevant segments" print "\n".join(map(str, segments)) cur_seg = None interesting_segments = [] print "Gluing" for seg in segments: if cur_seg is None: cur_seg = seg.copy() continue if cur_seg.right + 20000 < seg.left: interesting_segments.append(cur_seg.copy()) cur_seg = seg.copy() else: cur_seg.right = max(cur_seg.right, seg.right) if cur_seg is not None: interesting_segments.append(cur_seg.copy()) alignments = [] for edge in edge_alignments: for al in edge.alignments: ok = False for seg in interesting_segments: if al.seg_to.inter(seg): alignments.append(al) alignments = sorted(alignments, key=lambda al: al.seg_to.left) print "All relevant alignments" print "\n".join(map(str, alignments)) print "Interesting segments:", len(interesting_segments), sum( map(len, interesting_segments)) for seg in interesting_segments: print seg f = open(output_file, "w") tmp = [] for seg in interesting_segments: SeqIO.write(SeqIO.SeqRecord(seg.Seq(), seg.__str__()), f, "fasta") tmp.append(seg.Seq()) f.close() f1 = open(output_file + "1", "w") SeqIO.write(SeqIO.SeqRecord(("N" * 20000).join(tmp), "concat"), f1, "fasta")
als = filter(lambda al: not al.contradictingRTC(tail_size=params.bad_end_length), als) return self.filterLocal(als) if __name__ == "__main__": dir = sys.argv[1] query = sys.argv[2] target = sys.argv[3] extra_params = sys.argv[4:] contra = "contra" in extra_params over = "over" in extra_params long = "long" in extra_params start = "start" in extra_params forward = "forward" in extra_params aln = Aligner(DirDistributor(dir)) basic.CreateLog(dir) contigs = ContigCollection().loadFromFasta(open(target, "r"), False) for al in aln.localAlign(ReadCollection().loadFromFile(query), contigs): if start: if al.seg_to.contig.id.startswith("-"): al = al.rc if al.seg_to.left > 50: continue if over and al.contradictingRTC(): continue if forward: if al.seg_to.contig.id.startswith("-"): al = al.rc if contra and (len(al) < 8000 or not al.contradictingRTC()): continue if long and len(al) < 5000: continue