def main(args): size, args = grace.get_option_value(args,'--size',int,200) stride, args = grace.get_option_value(args,'--stride',int,50) grace.expect_no_further_options(args) if not args: print USAGE return 1 for filename in args: for name, seq in io.read_sequences(filename): name_parts = name.split(None, 1) name = name_parts[0] if len(name_parts) > 1: desc = ' ' + name_parts[1] else: desc = '' for i in xrange(-size+stride,len(seq),stride): start = max(0,min(len(seq),i)) end = max(0,min(len(seq), i+size)) io.write_fasta( sys.stdout, '%s:%d..%d' % (name,start+1,end) + desc, seq[start:end] ) return 0
def normalize(args): min_depth, args = grace.get_option_value(args, '--min-depth', int, 5) grace.expect_no_further_options(args) if len(args) < 2: print NORMALIZE_HELP raise grace.Help_shown() dirnames = args filenames = [] for dirname in dirnames: assert os.path.isdir(dirname), dirname + ' is not a directory' filenames.append( sorted( item for item in os.listdir(dirname) #if item.endswith('.userplot') and not item.endswith('-norm.userplot') if item.endswith('-depth.userplot') and not item.endswith('-ambiguous-depth.userplot') and not item.endswith('-pairspan-depth.userplot'))) for i in xrange(1, len(dirnames)): if filenames[i] != filenames[0]: raise grace.Error('Userplots in %s differ from those in %s' % (dirnames[i], dirnames[0])) filenames = filenames[0] for filename in filenames: normalize_files(dirnames, filename[:-15], min_depth)
def main(args): size, args = grace.get_option_value(args, '--size', int, 200) stride, args = grace.get_option_value(args, '--stride', int, 50) grace.expect_no_further_options(args) if not args: print USAGE return 1 for filename in args: for name, seq in io.read_sequences(filename): name_parts = name.split(None, 1) name = name_parts[0] if len(name_parts) > 1: desc = ' ' + name_parts[1] else: desc = '' for i in xrange(-size + stride, len(seq), stride): start = max(0, min(len(seq), i)) end = max(0, min(len(seq), i + size)) io.write_fasta(sys.stdout, '%s:%d..%d' % (name, start + 1, end) + desc, seq[start:end]) return 0
def normalize(args): min_depth, args = grace.get_option_value(args, '--min-depth', int, 5) grace.expect_no_further_options(args) if len(args) < 2: print NORMALIZE_HELP raise grace.Help_shown() dirnames = args filenames = [ ] for dirname in dirnames: assert os.path.isdir(dirname), dirname + ' is not a directory' filenames.append(sorted( item for item in os.listdir(dirname) #if item.endswith('.userplot') and not item.endswith('-norm.userplot') if item.endswith('-depth.userplot') and not item.endswith('-ambiguous-depth.userplot') and not item.endswith('-pairspan-depth.userplot') )) for i in xrange(1,len(dirnames)): if filenames[i] != filenames[0]: raise grace.Error('Userplots in %s differ from those in %s' % (dirnames[i], dirnames[0])) filenames = filenames[0] for filename in filenames: normalize_files(dirnames, filename[:-15], min_depth)
def import_(args): grace.expect_no_further_options(args) for item in args: sample = Options() options.samples.append(sample) sample.imported = True sample.clip_dest = None sample.align_dest = absolutize(item)
def front_command(args): grace.expect_no_further_options(args) if len(args) < 1: return output_dir.append(args[0]) input_reference_filenames.extend( [os.path.abspath(filename) for filename in args[1:]])
def front_command(args): grace.expect_no_further_options(args) if len(args) < 1: return output_dir.append(args[0]) input_reference_filenames.extend( [ os.path.abspath(filename) for filename in args[1:] ])
def plot(args): log_it, args = grace.get_option_value(args, '--log', grace.as_bool, False) grace.expect_no_further_options(args) import numpy, pylab pylab.rcParams['axes.formatter.limits'] = [-20, 20] pylab.figure(figsize=(10, 4)) maximum = 0 for filename in args: parts = filename.split('~~', 1) data = [] f = open(parts[0], 'rb') for line in f: data.append(float(line.strip())) f.close() data = numpy.array(data) maximum = max(maximum, numpy.maximum.reduce(data)) #if log_it: # data = numpy.log(data + 1.0) / numpy.log(2.0) if log_it: pylab.semilogy(numpy.arange(1, len(data) + 1), data, label=parts[-1]) else: pylab.plot(numpy.arange(1, len(data) + 1), data, label=parts[-1]) if len(args) > 1: pylab.legend() if log_it: pylab.ylim((1, maximum**1.2)) else: pylab.ylim((0, maximum * 1.2)) pylab.show()
def plot(args): log_it, args = grace.get_option_value(args, '--log', grace.as_bool, False) grace.expect_no_further_options(args) import numpy, pylab pylab.rcParams['axes.formatter.limits'] = [ -20, 20 ] pylab.figure(figsize=(10,4)) maximum = 0 for filename in args: parts = filename.split('~~', 1) data = [ ] f = open(parts[0],'rb') for line in f: data.append(float(line.strip())) f.close() data = numpy.array(data) maximum = max(maximum,numpy.maximum.reduce(data)) #if log_it: # data = numpy.log(data + 1.0) / numpy.log(2.0) if log_it: pylab.semilogy( numpy.arange(1,len(data)+1), data, label=parts[-1] ) else: pylab.plot( numpy.arange(1,len(data)+1), data, label=parts[-1] ) if len(args) > 1: pylab.legend() if log_it: pylab.ylim( (1,maximum**1.2) ) else: pylab.ylim( (0,maximum*1.2) ) pylab.show()
def pairs_command(args): grace.expect_no_further_options(args) assert len(args) == 2, 'Expected exactly two files in "pairs"' reads_filenames.append([ os.path.abspath(filename) for filename in args ])
def interleaved(args): grace.expect_no_further_options(args) sample.interleaved.extend(args)
def pairs(args): grace.expect_no_further_options(args) assert len(args) == 2, 'Expected exactly two files in "pairs:"' sample.pairs.append(args)
def main(args): default_transl_table, args = grace.get_option_value( args, '--transl_table', int, 11) use_coverage, args = grace.get_flag(args, '--use-coverage') coverage_cutoff, args = grace.get_option_value(args, '--coverage-cutoff', float, 0.1) tabular, args = grace.get_flag(args, '--tabular') noheader, args = grace.get_flag(args, '--noheader') verbose, args = grace.get_flag(args, '--verbose') bandwidth, args = grace.get_option_value(args, '--band', int, 20) grace.expect_no_further_options(args) if len(args) != 2: print USAGE return 1 genbank_filename = args[0] alignment_filename = args[1] if os.path.isdir(alignment_filename): alignment_filename = os.path.join(alignment_filename, 'alignment.maf') working_dir = os.path.split(alignment_filename)[0] alignments = load_alignments(alignment_filename) summaries = [] details = [] if not noheader: fields = 'Sequence\tLocus tag\tOld length (aa)\tNew length (aa)\tAmino acid changes\t' if use_coverage: fields += 'Unambiguous coverage vs expected\t\tAmbiguous coverage vs expected\t\tAmbiguous percent with any hits\t' fields += 'Gene\tProduct' if tabular: fields += '\tChanges of note' print fields for record in SeqIO.parse( io.open_possibly_compressed_file(genbank_filename), 'genbank'): sequence = record.seq.tostring() for name, seq1, seq2, alignment in alignments: if seq1 == sequence: break else: raise grace.Error( 'Genbank record %s sequence not identical to any reference sequence' % record.id) if use_coverage: depth = get_graph(working_dir, name, 'depth') ambiguous_depth = get_graph(working_dir, name, 'ambiguous-depth') median_depth = numpy.median(depth) median_ambiguous_depth = numpy.median(ambiguous_depth) ambiguous_factor = float(median_ambiguous_depth) / median_depth depth_expect = expected_depth(name, sequence, depth, ambiguous_depth) for feature in record.features: if feature.type != 'CDS': continue if 'locus_tag' not in feature.qualifiers: locus_tag = '%d..%d' % (feature.location.nofuzzy_start + 1, feature.location.nofuzzy_end) else: locus_tag = feature.qualifiers['locus_tag'][0] if 'transl_table' in feature.qualifiers: transl_table_no = int(feature.qualifiers['transl_table'][0]) else: assert default_transl_table is not None, 'No /transl_table for CDS, and default transl_table not given' transl_table_no = default_transl_table transl_table = CodonTable.ambiguous_dna_by_id[transl_table_no] start_codons = transl_table.start_codons try: feature_alignment = alignment_from_feature(sequence, feature) except Weird_alignment: warn('%s has a location I could not handle, skipping, sorry' % locus_tag) continue dna = [] new_dna = [] shifts = [] for i in xrange(feature_alignment.end2): p1 = feature_alignment.back_project(i, left=False) p2 = feature_alignment.back_project(i + 1, left=True) assert abs(p2 - p1) < 2 dna.append(sequence_slice(sequence, p1, p2)) p1a = alignment.project(p1, left=False) p2a = alignment.project(p2, left=False) #Hmm diff = (p2 - p1) - (p2a - p1a) #if diff: # if diff%3: # frame_shift = True # else: # frame_preserving_shift = True new_dna.append(sequence_slice(seq2, p1a, p2a)) if diff: shifts.append((i, dna[-1], new_dna[-1])) dna = ''.join(dna) new_dna = ''.join(new_dna) # This usually indicated a CDS truncated at the start? # in which case, will probably fail some way or other down the line. if 'codon_start' in feature.qualifiers: codon_start = int(feature.qualifiers['codon_start'][0]) - 1 else: codon_start = 0 dna = dna[codon_start:] new_dna = new_dna[codon_start:] if len(dna) % 3 != 0: warn(locus_tag + ' length not a multiple of 3') #assert len(new_dna) % 3 == 0 protein = Seq.Seq(dna).translate(table=transl_table_no).tostring() # http://en.wikipedia.org/wiki/Start_codon is always translated to M protein = 'M' + protein[1:] if dna[:3] not in start_codons: warn(locus_tag + ' has unknown start codon: ' + dna[:3]) original_lacks_stop_codon = not protein.endswith('*') if original_lacks_stop_codon: warn(locus_tag + ' lacks end codon') original_stops_before_end = '*' in protein[:-1] if original_stops_before_end: warn(locus_tag + ' contains stop codon before end') if 'translation' in feature.qualifiers: expect = feature.qualifiers['translation'][0] if protein[:-1] != expect: warn( locus_tag + ' translation given in feature does not match translation from DNA' ) new_protein = Seq.Seq(new_dna).translate( table=transl_table_no).tostring() new_protein = 'M' + new_protein[1:] # If end codon changed, find new end # Don't bother if there are unknown amino acids or # the original protein lacks a stop codon if 'X' not in new_protein and '*' not in new_protein and not original_lacks_stop_codon: #This is very inefficient i = feature_alignment.end2 while True: p1 = feature_alignment.back_project(i, left=False) p2 = feature_alignment.back_project(i + 1, left=True) p1a = alignment.project(p1, left=False) p2a = alignment.project(p2, left=False) #Hmm if p1a < 0 or p2a < 0 or p1a > len(seq2) or p2a > len( seq2): break new_dna += sequence_slice(seq2, p1a, p2a) new_protein = Seq.Seq(new_dna).translate( table=transl_table_no).tostring() new_protein = 'M' + new_protein[1:] if 'X' in new_protein or '*' in new_protein: break i += 1 # Is the protein shorter? # Don't bother checking if the original protein has extra stop codons if '*' in new_protein and not original_stops_before_end: new_protein = new_protein[:new_protein.index('*') + 1] # If indels occurred, do an alignment # Don't bother otherwise if shifts: # Penalize gaps with cost 2 (vs 1 for mismatch) # If lengths don't match, pad with spaces (won't match longer seq), # aligner prefers mismatch to gaps #result = pairwise2.align.globalxs(protein + ' '*max(0,len(new_protein)-len(protein)), # new_protein + ' '*max(0,len(protein)-len(new_protein)), # -2.001,-2.000)[0] # 2.001 : very slightly prefer contiguous gaps. Also much faster! result = band_limited_align( protein + ' ' * max(0, len(new_protein) - len(protein)), new_protein + ' ' * max(0, len(protein) - len(new_protein)), bandwidth) protein_ali = result[0] new_protein_ali = result[1] else: protein_ali = protein new_protein_ali = new_protein diffs = [] j = 0 k = 0 for i in xrange(min(len(new_protein_ali), len(protein_ali))): if protein_ali[i] != ' ' and new_protein_ali[i] != ' ' and ( protein_ali[i] == '-' or new_protein_ali[i] == '-' or not bio.might_be_same_amino(protein_ali[i], new_protein_ali[i])): diffs.append((i, j, k)) if protein_ali[i] != '-': j += 1 if new_protein_ali[i] != '-': k += 1 diff_start = not bio.might_be_same_base(new_dna[0],dna[0]) or \ not bio.might_be_same_base(new_dna[1],dna[1]) or \ not bio.might_be_same_base(new_dna[2],dna[2]) interesting_coverage = False if use_coverage: cds_depth = depth[feature_alignment.start1: feature_alignment.end1] #/ median_depth if not feature_alignment.forward1: cds_depth = cds_depth[::-1] cds_ambiguous_depth = ambiguous_depth[ feature_alignment.start1: feature_alignment.end1] #/ median_ambiguous_depth if not feature_alignment.forward1: cds_ambiguous_depth = cds_ambiguous_depth[::-1] cds_depth_expect = depth_expect[feature_alignment. start1:feature_alignment.end1] if not feature_alignment.forward1: cds_depth_expect = cds_depth_expect[::-1] #cds_average_depth_ratio = numpy.average(depth[feature_alignment.start1:feature_alignment.end1]) / median_depth #cds_average_ambiguous_depth_ratio = numpy.average(ambiguous_depth[feature_alignment.start1:feature_alignment.end1]) / median_ambiguous_depth #line += '%.1f\t' % cds_average_depth_ratio #line += '%.1f\t' % cds_average_ambiguous_depth_ratio #line += '%.1f..%.1f\t' % (numpy.minimum.reduce(cds_depth)/median_depth, numpy.maximum.reduce(cds_depth)/median_depth) #line += '%.1f+/-%.1f\t' % (numpy.average(cds_depth)/median_depth, numpy.var(cds_depth)**0.5/median_depth) #line += '%.1f..%.1f\t' % (numpy.minimum.reduce(cds_ambiguous_depth)/median_ambiguous_depth, numpy.maximum.reduce(cds_ambiguous_depth)/median_ambiguous_depth) avg_expect = numpy.average(cds_depth_expect) if avg_expect > 0.0: cds_avg_depth = numpy.average(cds_depth) / avg_expect cds_avg_ambiguous_depth = numpy.average( cds_ambiguous_depth) / avg_expect / ambiguous_factor strange = ((cds_depth >= cds_depth_expect * 1.5) | (cds_ambiguous_depth <= cds_depth_expect * (0.5 * ambiguous_factor))) interesting_coverage = numpy.average( strange) >= coverage_cutoff if interesting_coverage or diffs or diff_start or shifts or len( new_protein) != len(protein): line = name + '\t' + locus_tag + '\t' + \ '%d\t' % (len(protein)-1) + \ '%d\t' % (len(new_protein)-1) + \ '%d\t' % len(diffs) if use_coverage: if avg_expect <= 0.0: line += '\t\t\t' else: line += '%.1f\t' % (cds_avg_depth) + graphlet( cds_depth, cds_depth_expect) + '\t' line += '%.1f\t' % ( cds_avg_ambiguous_depth) + graphlet( cds_ambiguous_depth, cds_depth_expect * ambiguous_factor) + '\t' line += '%.1f%%\t' % ( numpy.average(cds_ambiguous_depth > 0.0) * 100.0) line += '%s\t' % feature.qualifiers.get('gene',[''])[0] + \ '%s' % feature.qualifiers.get('product',[''])[0] notes = [] if use_coverage and 'X' in new_protein: xs = new_protein.count('X') if xs == len(new_protein) - 1: #First is M, so len-1 notes.append('\ No consensus') else: notes.append('\ No consensus for %d aa' % (new_protein.count('X'))) if len(new_protein) < len(protein): notes.append('\ Shorter by %d aa' % (len(protein) - len(new_protein))) if len(new_protein) > len(protein): notes.append('\ Longer by %d aa' % (len(new_protein) - len(protein))) if diff_start: notes.append('\ Start changed: %s -> %s' % (dna[:3], new_dna[:3])) if new_dna[:3] not in start_codons: notes.append(' No longer a start codon!') if shifts: notes.append('\ Indels:') for pos, old, new in shifts: notes.append(' base %5d / codon %5d %s -> %s' % (pos + 1, (pos // 3) + 1, old, new or '-')) if diffs: if verbose: notes.append('\ Amino acid changes:') for i, j, k in diffs: notes.append( ' codon %5d %s->%s (%s->%s)' % (j + 1, protein_ali[i], new_protein_ali[i], dna[j * 3:j * 3 + 3] if protein_ali[i] != '-' else '-', new_dna[k * 3:k * 3 + 3] if new_protein_ali[i] != '-' else '-')) #if len(new_protein) > len(protein): # print 'New protein is longer:', new_protein[len(protein):] #if len(new_protein) < len(protein): # print 'New protein is shorter:', protein[len(new_protein):] #print protein #print new_protein if tabular: print line + '\t' + ' '.join( [' '.join(note.strip().split()) for note in notes]) else: print line for note in notes: print '\t' + note return 0
def main(args): default_transl_table, args = grace.get_option_value(args, '--transl_table', int, 11) use_coverage, args = grace.get_flag(args, '--use-coverage') coverage_cutoff, args = grace.get_option_value(args, '--coverage-cutoff', float, 0.1) tabular, args = grace.get_flag(args, '--tabular') noheader, args = grace.get_flag(args, '--noheader') verbose, args = grace.get_flag(args, '--verbose') bandwidth, args = grace.get_option_value(args, '--band', int, 20) grace.expect_no_further_options(args) if len(args) != 2: print USAGE return 1 genbank_filename = args[0] alignment_filename = args[1] if os.path.isdir(alignment_filename): alignment_filename = os.path.join(alignment_filename, 'alignment.maf') working_dir = os.path.split(alignment_filename)[0] alignments = load_alignments(alignment_filename) summaries = [ ] details = [ ] if not noheader: fields = 'Sequence\tLocus tag\tOld length (aa)\tNew length (aa)\tAmino acid changes\t' if use_coverage: fields += 'Unambiguous coverage vs expected\t\tAmbiguous coverage vs expected\t\tAmbiguous percent with any hits\t' fields += 'Gene\tProduct' if tabular: fields += '\tChanges of note' print fields for record in SeqIO.parse(io.open_possibly_compressed_file(genbank_filename),'genbank'): sequence = record.seq.tostring() for name, seq1, seq2, alignment in alignments: if seq1 == sequence: break else: raise grace.Error('Genbank record %s sequence not identical to any reference sequence' % record.id) if use_coverage: depth = get_graph(working_dir, name, 'depth') ambiguous_depth = get_graph(working_dir, name, 'ambiguous-depth') median_depth = numpy.median(depth) median_ambiguous_depth = numpy.median(ambiguous_depth) ambiguous_factor = float(median_ambiguous_depth) / median_depth depth_expect = expected_depth(name, sequence, depth, ambiguous_depth) for feature in record.features: if feature.type != 'CDS': continue if 'locus_tag' not in feature.qualifiers: locus_tag = '%d..%d' % (feature.location.nofuzzy_start+1,feature.location.nofuzzy_end) else: locus_tag = feature.qualifiers['locus_tag'][0] if 'transl_table' in feature.qualifiers: transl_table_no = int(feature.qualifiers['transl_table'][0]) else: assert default_transl_table is not None, 'No /transl_table for CDS, and default transl_table not given' transl_table_no = default_transl_table transl_table = CodonTable.ambiguous_dna_by_id[transl_table_no] start_codons = transl_table.start_codons try: feature_alignment = alignment_from_feature(sequence, feature) except Weird_alignment: warn('%s has a location I could not handle, skipping, sorry' % locus_tag) continue dna = [ ] new_dna = [ ] shifts = [ ] for i in xrange(feature_alignment.end2): p1 = feature_alignment.back_project(i, left=False) p2 = feature_alignment.back_project(i+1, left=True) assert abs(p2-p1) < 2 dna.append( sequence_slice(sequence,p1,p2) ) p1a = alignment.project(p1, left=False) p2a = alignment.project(p2, left=False) #Hmm diff = (p2-p1)-(p2a-p1a) #if diff: # if diff%3: # frame_shift = True # else: # frame_preserving_shift = True new_dna.append( sequence_slice(seq2,p1a,p2a) ) if diff: shifts.append((i,dna[-1],new_dna[-1])) dna = ''.join(dna) new_dna = ''.join(new_dna) # This usually indicated a CDS truncated at the start? # in which case, will probably fail some way or other down the line. if 'codon_start' in feature.qualifiers: codon_start = int(feature.qualifiers['codon_start'][0]) - 1 else: codon_start = 0 dna = dna[codon_start:] new_dna = new_dna[codon_start:] if len(dna) % 3 != 0: warn(locus_tag + ' length not a multiple of 3') #assert len(new_dna) % 3 == 0 protein = Seq.Seq(dna).translate(table=transl_table_no).tostring() # http://en.wikipedia.org/wiki/Start_codon is always translated to M protein = 'M' + protein[1:] if dna[:3] not in start_codons: warn(locus_tag + ' has unknown start codon: ' + dna[:3]) original_lacks_stop_codon = not protein.endswith('*') if original_lacks_stop_codon: warn(locus_tag + ' lacks end codon') original_stops_before_end = '*' in protein[:-1] if original_stops_before_end: warn(locus_tag + ' contains stop codon before end') if 'translation' in feature.qualifiers: expect = feature.qualifiers['translation'][0] if protein[:-1] != expect: warn(locus_tag + ' translation given in feature does not match translation from DNA') new_protein = Seq.Seq(new_dna).translate(table=transl_table_no).tostring() new_protein = 'M' + new_protein[1:] # If end codon changed, find new end # Don't bother if there are unknown amino acids or # the original protein lacks a stop codon if 'X' not in new_protein and '*' not in new_protein and not original_lacks_stop_codon: #This is very inefficient i = feature_alignment.end2 while True: p1 = feature_alignment.back_project(i, left=False) p2 = feature_alignment.back_project(i+1, left=True) p1a = alignment.project(p1, left=False) p2a = alignment.project(p2, left=False) #Hmm if p1a < 0 or p2a < 0 or p1a > len(seq2) or p2a > len(seq2): break new_dna += sequence_slice(seq2,p1a,p2a) new_protein = Seq.Seq(new_dna).translate(table=transl_table_no).tostring() new_protein = 'M' + new_protein[1:] if 'X' in new_protein or '*' in new_protein: break i += 1 # Is the protein shorter? # Don't bother checking if the original protein has extra stop codons if '*' in new_protein and not original_stops_before_end: new_protein = new_protein[:new_protein.index('*')+1] # If indels occurred, do an alignment # Don't bother otherwise if shifts: # Penalize gaps with cost 2 (vs 1 for mismatch) # If lengths don't match, pad with spaces (won't match longer seq), # aligner prefers mismatch to gaps #result = pairwise2.align.globalxs(protein + ' '*max(0,len(new_protein)-len(protein)), # new_protein + ' '*max(0,len(protein)-len(new_protein)), # -2.001,-2.000)[0] # 2.001 : very slightly prefer contiguous gaps. Also much faster! result = band_limited_align(protein + ' '*max(0,len(new_protein)-len(protein)), new_protein + ' '*max(0,len(protein)-len(new_protein)), bandwidth) protein_ali = result[0] new_protein_ali = result[1] else: protein_ali = protein new_protein_ali = new_protein diffs = [ ] j = 0 k = 0 for i in xrange(min(len(new_protein_ali),len(protein_ali))): if protein_ali[i] != ' ' and new_protein_ali[i] != ' ' and ( protein_ali[i] == '-' or new_protein_ali[i] == '-' or not bio.might_be_same_amino(protein_ali[i], new_protein_ali[i]) ): diffs.append((i,j,k)) if protein_ali[i] != '-': j += 1 if new_protein_ali[i] != '-': k += 1 diff_start = not bio.might_be_same_base(new_dna[0],dna[0]) or \ not bio.might_be_same_base(new_dna[1],dna[1]) or \ not bio.might_be_same_base(new_dna[2],dna[2]) interesting_coverage = False if use_coverage: cds_depth = depth[feature_alignment.start1:feature_alignment.end1] #/ median_depth if not feature_alignment.forward1: cds_depth = cds_depth[::-1] cds_ambiguous_depth = ambiguous_depth[feature_alignment.start1:feature_alignment.end1] #/ median_ambiguous_depth if not feature_alignment.forward1: cds_ambiguous_depth = cds_ambiguous_depth[::-1] cds_depth_expect = depth_expect[feature_alignment.start1:feature_alignment.end1] if not feature_alignment.forward1: cds_depth_expect = cds_depth_expect[::-1] #cds_average_depth_ratio = numpy.average(depth[feature_alignment.start1:feature_alignment.end1]) / median_depth #cds_average_ambiguous_depth_ratio = numpy.average(ambiguous_depth[feature_alignment.start1:feature_alignment.end1]) / median_ambiguous_depth #line += '%.1f\t' % cds_average_depth_ratio #line += '%.1f\t' % cds_average_ambiguous_depth_ratio #line += '%.1f..%.1f\t' % (numpy.minimum.reduce(cds_depth)/median_depth, numpy.maximum.reduce(cds_depth)/median_depth) #line += '%.1f+/-%.1f\t' % (numpy.average(cds_depth)/median_depth, numpy.var(cds_depth)**0.5/median_depth) #line += '%.1f..%.1f\t' % (numpy.minimum.reduce(cds_ambiguous_depth)/median_ambiguous_depth, numpy.maximum.reduce(cds_ambiguous_depth)/median_ambiguous_depth) avg_expect = numpy.average(cds_depth_expect) if avg_expect > 0.0: cds_avg_depth = numpy.average(cds_depth)/avg_expect cds_avg_ambiguous_depth = numpy.average(cds_ambiguous_depth)/avg_expect/ambiguous_factor strange = ( (cds_depth >= cds_depth_expect*1.5) | (cds_ambiguous_depth <= cds_depth_expect*(0.5*ambiguous_factor)) ) interesting_coverage = numpy.average(strange) >= coverage_cutoff if interesting_coverage or diffs or diff_start or shifts or len(new_protein) != len(protein): line = name + '\t' + locus_tag + '\t' + \ '%d\t' % (len(protein)-1) + \ '%d\t' % (len(new_protein)-1) + \ '%d\t' % len(diffs) if use_coverage: if avg_expect <= 0.0: line += '\t\t\t' else: line += '%.1f\t' % (cds_avg_depth) + graphlet(cds_depth, cds_depth_expect)+'\t' line += '%.1f\t' % (cds_avg_ambiguous_depth) + graphlet(cds_ambiguous_depth, cds_depth_expect*ambiguous_factor)+'\t' line += '%.1f%%\t' % (numpy.average(cds_ambiguous_depth > 0.0)*100.0) line += '%s\t' % feature.qualifiers.get('gene',[''])[0] + \ '%s' % feature.qualifiers.get('product',[''])[0] notes = [ ] if use_coverage and 'X' in new_protein: xs = new_protein.count('X') if xs == len(new_protein)-1: #First is M, so len-1 notes.append('\ No consensus') else: notes.append('\ No consensus for %d aa' % (new_protein.count('X'))) if len(new_protein) < len(protein): notes.append('\ Shorter by %d aa' % (len(protein)-len(new_protein))) if len(new_protein) > len(protein): notes.append('\ Longer by %d aa' % (len(new_protein)-len(protein))) if diff_start: notes.append('\ Start changed: %s -> %s' % (dna[:3], new_dna[:3])) if new_dna[:3] not in start_codons: notes.append(' No longer a start codon!') if shifts: notes.append('\ Indels:') for pos, old, new in shifts: notes.append(' base %5d / codon %5d %s -> %s' % (pos+1,(pos//3)+1,old,new or '-')) if diffs: if verbose: notes.append('\ Amino acid changes:') for i, j, k in diffs: notes.append(' codon %5d %s->%s (%s->%s)' % ( j+1, protein_ali[i], new_protein_ali[i], dna[j*3:j*3+3] if protein_ali[i] != '-' else '-', new_dna[k*3:k*3+3] if new_protein_ali[i] != '-' else '-' )) #if len(new_protein) > len(protein): # print 'New protein is longer:', new_protein[len(protein):] #if len(new_protein) < len(protein): # print 'New protein is shorter:', protein[len(new_protein):] #print protein #print new_protein if tabular: print line + '\t' + ' '.join([ ' '.join(note.strip().split()) for note in notes ]) else: print line for note in notes: print '\t' + note return 0
def pairs_command(args): grace.expect_no_further_options(args) assert len(args) == 2, 'Expected exactly two files in "pairs"' reads_filenames.append( [os.path.abspath(filename) for filename in args])
def main(args): title1, args = grace.get_option_value(args, '--title1', str, None) title2, args = grace.get_option_value(args, '--title2', str, None) grace.expect_no_further_options(args) if len(args) != 3: print >> sys.stderr, USAGE return 1 working_dir1 = args[0] working_dir2 = args[1] cutoff = float(args[2]) sequence_names = [ name for name, sequence in io.read_sequences( os.path.join(working_dir1, 'reference.fa')) ] if title1 is None: title1 = working_dir1 if title2 is None: title2 = working_dir2 n = 1 while significance([('A', n)], [('T', n)], 1.0) > cutoff: n += 1 print '%g\tsignificance cutoff' % cutoff print '%d\tdepth required to call substitution (greater if there are errors in the reads)' % n print 'Sequence\tPosition in reference\tChange type\tReference\t%s\t%s\tp-value (no correction for multiple testing)\t%s\t%s' % ( title1, title2, title1, title2) for sequence_name in sequence_names: filename1 = os.path.join( working_dir1, grace.filesystem_friendly_name(sequence_name) + '-evidence.txt') filename2 = os.path.join( working_dir2, grace.filesystem_friendly_name(sequence_name) + '-evidence.txt') for (pos1, ins1, sub1, ref1, conins1, consub1), (pos2, ins2, sub2, ref2, conins2, consub2) in itertools.izip(read_file(filename1), read_file(filename2)): assert pos1 == pos2 and ref1 == ref2 if pos1 % 1000 == 0: grace.status('Testing %s %d' % (sequence_name, pos1)) dec_ins1 = io.decode_evidence(ins1) dec_ins2 = io.decode_evidence(ins2) if dec_ins1 and dec_ins2: sig = significance(io.decode_evidence(ins1), io.decode_evidence(ins2), cutoff) if sig is not None and sig <= cutoff: grace.status('') print '%s\t%d\t%s\t\t%s\t%s\t%g\t%s\t%s' % ( sequence_name, pos1, 'insertion-before', ins1, ins2, sig, conins1, conins2) dec_sub1 = io.decode_evidence(sub1) dec_sub2 = io.decode_evidence(sub2) if dec_sub1 and dec_sub2: sig = significance(dec_sub1, dec_sub2, cutoff) if sig is not None and sig <= cutoff: if dec_sub1[0][0] == '-' or dec_sub2[0][0] == '-': what = 'deletion' elif dec_sub1[0][0] != dec_sub2[0][0]: what = 'substitution' else: what = 'different mix' grace.status('') print '%s\t%d\t%s\t%s\t%s\t%s\t%g\t%s\t%s' % ( sequence_name, pos1, what, ref1, sub1, sub2, sig, consub1, consub2) grace.status('') return 0
def reads_command(args): grace.expect_no_further_options(args) reads_filenames.extend([ [ os.path.abspath(filename) ] for filename in args])
def main(args): title1, args = grace.get_option_value(args, "--title1", str, None) title2, args = grace.get_option_value(args, "--title2", str, None) grace.expect_no_further_options(args) if len(args) != 3: print >> sys.stderr, USAGE return 1 working_dir1 = args[0] working_dir2 = args[1] cutoff = float(args[2]) sequence_names = [name for name, sequence in io.read_sequences(os.path.join(working_dir1, "reference.fa"))] if title1 is None: title1 = working_dir1 if title2 is None: title2 = working_dir2 n = 1 while significance([("A", n)], [("T", n)], 1.0) > cutoff: n += 1 print "%g\tsignificance cutoff" % cutoff print "%d\tdepth required to call substitution (greater if there are errors in the reads)" % n print "Sequence\tPosition in reference\tChange type\tReference\t%s\t%s\tp-value (no correction for multiple testing)\t%s\t%s" % ( title1, title2, title1, title2, ) for sequence_name in sequence_names: filename1 = os.path.join(working_dir1, grace.filesystem_friendly_name(sequence_name) + "-evidence.txt") filename2 = os.path.join(working_dir2, grace.filesystem_friendly_name(sequence_name) + "-evidence.txt") for (pos1, ins1, sub1, ref1, conins1, consub1), (pos2, ins2, sub2, ref2, conins2, consub2) in itertools.izip( read_file(filename1), read_file(filename2) ): assert pos1 == pos2 and ref1 == ref2 if pos1 % 1000 == 0: grace.status("Testing %s %d" % (sequence_name, pos1)) dec_ins1 = io.decode_evidence(ins1) dec_ins2 = io.decode_evidence(ins2) if dec_ins1 and dec_ins2: sig = significance(io.decode_evidence(ins1), io.decode_evidence(ins2), cutoff) if sig is not None and sig <= cutoff: grace.status("") print "%s\t%d\t%s\t\t%s\t%s\t%g\t%s\t%s" % ( sequence_name, pos1, "insertion-before", ins1, ins2, sig, conins1, conins2, ) dec_sub1 = io.decode_evidence(sub1) dec_sub2 = io.decode_evidence(sub2) if dec_sub1 and dec_sub2: sig = significance(dec_sub1, dec_sub2, cutoff) if sig is not None and sig <= cutoff: if dec_sub1[0][0] == "-" or dec_sub2[0][0] == "-": what = "deletion" elif dec_sub1[0][0] != dec_sub2[0][0]: what = "substitution" else: what = "different mix" grace.status("") print "%s\t%d\t%s\t%s\t%s\t%s\t%g\t%s\t%s" % ( sequence_name, pos1, what, ref1, sub1, sub2, sig, consub1, consub2, ) grace.status("") return 0
def main(args): genbank_filename, args = grace.get_option_value(args,'--gbk',str,None) use_indels, args = grace.get_option_value(args,'--indels',grace.as_bool,True) use_reference, args = grace.get_option_value(args,'--reference',grace.as_bool,True) give_evidence, args = grace.get_option_value(args,'--evidence',grace.as_bool,True) give_consequences, args = grace.get_option_value(args,'--consequences',grace.as_bool,True) require_all, args = grace.get_option_value(args,'--require-all',grace.as_bool,False) require_bisect, args = grace.get_option_value(args,'--require-bisect',grace.as_bool,False) full_output, args = grace.get_option_value(args,'--full',grace.as_bool,False) format, args = grace.get_option_value(args,'--as',str,'table') # Secret option! limit, args = grace.get_option_value(args,'--limit',int,None) grace.expect_no_further_options(args) if len(args) < 1: sys.stderr.write(USAGE) return 1 working_dirs = [ ] split_a = [ ] split_b = [ ] def default(args): working_dirs.extend(args) def splitting(args): split_a.extend(args) def splitting_from(args): split_b.extend(args) grace.execute(args, { 'splitting' : splitting, 'from' : splitting_from }, default ) if use_reference: names = ['reference'] evidence_start = 1 else: names = [ ] evidence_start = 0 names.extend( norm_name(item) for item in working_dirs ) references = io.read_sequences(os.path.join(working_dirs[0], 'reference.fa')) annotations = { } if genbank_filename: from Bio import SeqIO for record in SeqIO.parse(io.open_possibly_compressed_file(genbank_filename),'genbank'): sequence = record.seq.tostring() features = [ item for item in record.features if item.type != 'source' ] features.sort(key=lambda item: item.location.nofuzzy_start) annotations[sequence] = features iterator = reader(working_dirs, references, use_reference, annotations) if not use_indels: iterator = itertools.ifilter(has_no_indels, iterator) if require_all or require_bisect or format == 'counts': iterator = itertools.ifilter(fully_unambiguous, iterator) if require_bisect: iterator = itertools.ifilter(is_binary_partition, iterator) if not require_bisect: if full_output: iterator = itertools.ifilter(not_boring_insertion, iterator) else: iterator = itertools.ifilter(is_interesting, iterator) if split_a or split_b: assert len(names) == len(set(names)), 'Two samples with the same name' try: split_a = [ names.index(norm_name(item)) for item in split_a ] split_b = [ names.index(norm_name(item)) for item in split_b ] except ValueError: raise grace.Error('Sample to be split is not amongst samples given') iterator = itertools.ifilter(is_split(split_a, split_b), iterator) if limit: iterator = itertools.islice(iterator, limit) if format == 'table': line = 'Reference\tPosition\tChange type' line += '\t' + '\t'.join(names) if give_evidence: line += '\t' + '\t'.join(names[evidence_start:]) if give_consequences: line += '\t' + '\t'.join(names[evidence_start:]) if annotations: line += '\tAnnotations' print line for calls in iterator: line = '%s\t%d\t%s\t%s' % ( calls.ref_name, calls.ref_pos+1, change_type(calls), '\t'.join(item.consensus for item in calls.calls)) if give_evidence: line += '\t' + '\t'.join(item.evidence for item in calls.calls[evidence_start:]) if give_consequences: line += '\t' + '\t'.join(item.consequences for item in calls.calls[evidence_start:]) if annotations: line += '\t' + describe_features(calls.features) print line elif format == 'compact': for line in transpose_strings(names): print line print for calls in iterator: if calls.is_insertion: footer = '%12d.5 %s' % (calls.ref_pos, calls.ref_name) else: footer = '%12d %s' % (calls.ref_pos+1, calls.ref_name) t = transpose_strings([ item.consensus for item in calls.calls ], '-', 1) top = t[0] + ' ' + footer if give_consequences: consequences = [ ] for call in calls.calls: if call.consequences: for item in call.consequences.split(', '): item = ' '.join(item.split()[:3]) if item not in consequences: consequences.append(item) if consequences: top += ' ' + ' / '.join(sorted(consequences)) top += ' ' + describe_features(calls.features) print top for line in t[1:]: print line elif format == 'nexus': buckets = [ [ ] for name in names ] for calls in iterator: for i, char in enumerate(partition_string(calls)): buckets[i].append(char) print '#NEXUS' print 'begin taxa;' print 'dimensions ntax=%d;' % len(names) print 'taxlabels' for name in names: print name print ';' print 'end;' print 'begin characters;' print 'dimensions nchar=%d;' % len(buckets[0]) print 'format datatype=STANDARD symbols="ACGT-0123456789" missing=N;' print 'matrix' for name, bucket in itertools.izip(names, buckets): print name, ''.join(bucket) print ';' print 'end;' elif format == 'counts': for line in transpose_strings(names): print line print counts = { } for calls in iterator: count_str = partition_string(calls) if count_str not in counts: counts[count_str] = 1 else: counts[count_str] += 1 for count_str in sorted(counts, key=lambda x: (counts[x], x), reverse=True): print '%s %d' % (transpose_strings(count_str)[0], counts[count_str]) else: raise grace.Error('Unknown output format: ' + format)
def reads_command(args): grace.expect_no_further_options(args) reads_filenames.extend([[os.path.abspath(filename)] for filename in args])
def old_main(args): use_indels, args = grace.get_option_value(args,'--indels',int,1) use_reference, args = grace.get_option_value(args,'--reference',int,1) make_list, args = grace.get_option_value(args,'--list',int,0) fasta_output, args = grace.get_option_value(args,'--fasta',int,0) grace.expect_no_further_options(args) if len(args) < 1: sys.stderr.write(USAGE) return 1 if fasta_output and use_indels: print >> sys.stderr, 'Indels will not be included in FASTA output' use_indels = 0 working_dirs = args #reference_data = { } # (ref_name, position, change_type) -> string #strain_data = { } # working_dir -> (ref_name, position, change_type) -> string names = ['reference'] + working_dirs substitution_calls = { } # ref_name -> [ [ call ] ] insertion_calls = { } # ref_name -> [ [ call ] ] substitution_evidence = { } insertion_evidence = { } for name, sequence in io.read_sequences(os.path.join(working_dirs[0], 'reference.fa')): substitution_calls[name] = [ list(sequence.upper()) ] insertion_calls[name] = [ [ '-' ] * len(sequence) ] substitution_evidence[name] = [ [ '' ] * len(sequence) ] insertion_evidence[name] = [ [ '' ] * len(sequence) ] for working_dir in working_dirs: for name in substitution_calls: filename = os.path.join(working_dir, grace.filesystem_friendly_name(name) + '-evidence.txt') f = open(filename,'rb') this_substitution_calls = [ ] this_insertion_calls = [ ] this_substitution_evidence = [ ] this_insertion_evidence = [ ] header = f.readline() if header.count('\t') != 5: print >> sys.stderr, 'Old style evidence file. Please re-run nesoni consensus.' return 1 for line in f: fields = line.rstrip('\n').split('\t') this_substitution_calls.append(fields[5]) this_insertion_calls.append(fields[4]) this_substitution_evidence.append(fields[2]) this_insertion_evidence.append(fields[1]) substitution_calls[name].append(this_substitution_calls) insertion_calls[name].append(this_insertion_calls) substitution_evidence[name].append(this_substitution_evidence) insertion_evidence[name].append(this_insertion_evidence) if not use_reference: names.pop(0) for name in substitution_calls: substitution_calls[name].pop(0) insertion_calls[name].pop(0) substitution_evidence[name].pop(0) insertion_evidence[name].pop(0) interesting = find_interesting('substitution', substitution_calls, substitution_evidence) if use_indels: interesting.extend( find_interesting('insertion-before', insertion_calls, insertion_evidence) ) if not use_indels: interesting = [ item for item in interesting if '-' not in item[3] ] interesting.sort() if fasta_output: do_fasta_output(names, interesting) return 0 #strain_reference_having_consensus = { } # working_dir -> ref_name -> string # #for working_dir in working_dirs: # assert working_dir not in strain_data, 'Working directory given twice' # strain_data[working_dir] = { } # # report_file = open(os.path.join(working_dir, 'report.txt'), 'rU') # report_file.readline() # for line in report_file: # ref_name, position, change_type, old, new, evidence = \ # line.rstrip('\n').split('\t') # # if change_type == 'deletion': # change_type = 'substitution' # # if not use_indels and \ # (change_type == 'insertion-before' or new == '-'): # continue # # key = (ref_name, int(position), change_type) # if key in reference_data: # assert reference_data[key] == old # else: # reference_data[key] = old # # strain_data[working_dir][key] = new # report_file.close() # # strain_reference_having_consensus[working_dir] = { } # ref_have_con_filename = os.path.join(working_dir, 'reference_having_consensus.fa') # for name, sequence in io.read_fasta(ref_have_con_filename): # strain_reference_having_consensus[working_dir][name] = sequence # #keys = sorted(reference_data) # ##Fill in any blanks #for working_dir in working_dirs: # for key in keys: # if key in strain_data[working_dir]: continue # # # - Positions in report files start from 1 not 0 # # - Insertions must be bracketed # lacks_consensus = ( # strain_reference_having_consensus[working_dir][key[0]][key[1]-1] == 'N' or # (key[2] == 'insertion-before' and key[1] > 1 and # strain_reference_having_consensus[working_dir][key[0]][key[1]-2] == 'N') # ) # # #If there's no consensus, record it as ambiguous # if lacks_consensus: # strain_data[working_dir][key] = 'N' # else: # strain_data[working_dir][key] = reference_data[key] #all_data_names = ([ 'reference' ] if use_reference else []) + working_dirs #all_data = ([ reference_data ] if use_reference else []) + \ # [ strain_data[working_dir] for working_dir in working_dirs ] #all_data_names = ([ 'reference' ] if use_reference else []) + working_dirs ones = ( 1 << len(names) )-1 total_differences = 0 if make_list: print '\t'.join(['Partition','Sequence','Position in reference','Change type'] + names + names) for i in xrange(1,(1<<len(names))-1,2): set1 = [ ] set2 = [ ] for j in xrange(len(names)): if i & (1<<j): set1.append(j) else: set2.append(j) if make_list: print print ', '.join( names[i] for i in set1 ) + ' vs ' + \ ', '.join( names[i] for i in set2 ) print n = 0 for refname, position, change_type, values, has_ambiguous, evidence in interesting: #Skip if *any* ambiguity if has_ambiguous: continue if any( values[i] != values[set1[0]] for i in set1[1:] ) or \ any( values[i] != values[set2[0]] for i in set2[1:] ): continue if make_list: if change_type == 'substitution' and '-' in values: change_type = 'deletion' print '\t%s\t%d\t%s\t' % (refname,position,change_type) + '\t'.join(values) + '\t' + '\t'.join(evidence) n += 1 total_differences += n if not make_list: print ', '.join( names[i] for i in set1 ) + ' vs ' + \ ', '.join( names[i] for i in set2 ) + \ ': %d differences' %n if not make_list: print print 'Total: %d' % total_differences if make_list: print print 'Ignored' print n_multiway = 0 n_ambiguous = 0 for refname, position, change_type, values, has_ambiguous, evidence in interesting: confusing = False if has_ambiguous: n_ambiguous += 1 confusing = True elif len(set(values)) > 2: n_multiway += 1 confusing = True if make_list and confusing: print '\t%s\t%d\t%s\t' % (refname,position,change_type) + '\t'.join(values) + '\t' + '\t'.join(evidence) if not make_list: print print 'Ambiguities ignored: %d' % n_ambiguous print 'Multi-way changes ignored: %d' % n_multiway assert total_differences + n_ambiguous + n_multiway == len(interesting) return 0
def recombination(args): grace.expect_no_further_options(args) if len(args) != 2: print >> sys.stderr, USAGE raise grace.Help_shown() working_dir, seq_name = args references = dict(io.read_sequences(os.path.join(working_dir, 'reference.fa'))) depth = { } prefixes = { } suffixes = { } for name in references: depth[name] = numpy.zeros(len(references[name]), 'int64') prefixes[name] = [ [] for base in references[name] ] suffixes[name] = [ [] for base in references[name] ] def register_divergence(hit): if not hit.query_forward: hit = hit.reversed() margin = 20 if hit.target_end - hit.target_start < 20: return False depth[hit.target_name][hit.target_start : hit.target_end] += 1 any = False if hit.query_end <= len(hit.query_seq)-margin: # and hit.target_end < len(hit.target_seq): suffixes[hit.target_name][hit.target_end-1].append( hit.query_seq[hit.query_end:] ) any = True if hit.query_start >= margin: # and hit.target_start > 0: prefixes[hit.target_name][hit.target_start].append( hit.query_seq[:hit.query_start] ) any = True return any n = 0 for (read_name, read_seq), hits in shrimp.iter_read_hits(working_dir): # Skip reads containing Ns if 'N' in read_seq: continue for line in hits: register_divergence(alignment_from_shrimp(line, references, read_name, read_seq)) n += 1 #if n > 100000: # break if n%10000 == 0: grace.status('Processing read %s' % grace.pretty_number(n)) grace.status('') def show_items(items): original_length = len(items) cut = 0 while len(items) > 80: cut += 1 items = [ item for item in items if item[0] >= cut ] for item in items: print item[1] if len(items) < original_length: print '(and %d more occurring %d times or less)' % (original_length-len(items), cut-1) def score(items): if not items: return 1.0 return float(sum( item[0] * item[0] for item in items )) / (sum( item[0] for item in items )**2) def summarize_prefixes(seqs, pad): seqs = sorted(seqs, key=lambda seq: seq[::-1]) cut = 100 while True: items = [ ] for (seq, iterator) in itertools.groupby(seqs, key = lambda x: x[-cut:]): ss = list(iterator) anylong = any( item != seq for item in ss ) n = len(ss) items.append( (n, ('%'+str(pad)+'s')%(('...' if anylong else '') + seq) + ' x %d' % n) ) if score(items) >= 1.0/20: break cut -= 1 show_items(items) def summarize_suffixes(seqs, pad): seqs = sorted(seqs) cut = 100 while True: items = [ ] for (seq, iterator) in itertools.groupby(seqs, key = lambda x: x[:cut]): ss = list(iterator) anylong = any( item != seq for item in ss ) n = len(ss) items.append( (n, ('%'+str(pad)+'s')%('%d x '%n) + seq + ('...' if anylong else '')) ) if score(items) >= 1.0/20: break cut -= 1 show_items(items) print 'Position Depth Changed prefixes Changed suffixes' print ' Count % of depth Count % of depth' for i in xrange(len(references[seq_name])): print '%8d %10d %9d %11s %9d %11s' % ( i+1, depth[seq_name][i], len(prefixes[seq_name][i]), '%.3f%%' % (len(prefixes[seq_name][i])*100.0/depth[seq_name][i]) if prefixes[seq_name][i] else '', len(suffixes[seq_name][i]), '%.3f%%' % (len(suffixes[seq_name][i])*100.0/depth[seq_name][i]) if suffixes[seq_name][i] else '') #summarize_suffixes(suffixes[name][i], references[name][i+1:], references[name], suffix_depth[name][i]) print print 'Details' print for i in xrange(len(references[seq_name])): print '%-80s*' % ('Base %d' % (i+1)) print pad_slice(references[seq_name], i-80,i+1+80) summarize_prefixes(prefixes[seq_name][i], 80) summarize_suffixes(suffixes[seq_name][i], 81) print
def default(args): grace.expect_no_further_options(args) if len(args) != 1: print >> sys.stderr, BATCH_HELP % default_nesoni raise grace.Help_shown() options.dirname = args[0]
def reference(args): grace.expect_no_further_options(args) options.references.extend(args)
def reads(args): grace.expect_no_further_options(args) sample.reads.extend(args)
def main(args): mincov, args = grace.get_option_value(args, '--mincov', int, 1) maxdiff, args = grace.get_option_value(args, '--maxdiff', int, 16) minsize, args = grace.get_option_value(args, '--minsize', int, 200) what, args = grace.get_option_value(args, '--what', as_core_or_unique, 'core') is_core = (what == 'core') grace.expect_no_further_options(args) if len(args) < 2: print >> sys.stderr, HELP raise grace.Help_shown() output_dir, working_dirs = args[0], args[1:] assert not path.exists(path.join(output_dir, 'reference.fa')), \ 'Output directory not given' if not path.exists(output_dir): os.mkdir(output_dir) for name, seq in io.read_sequences(path.join(working_dirs[0],'reference.fa')): print name friendly_name = grace.filesystem_friendly_name(name) good = [ True ] * len(seq) for working_dir in working_dirs: if is_core: suffix = '-depth.userplot' else: suffix = '-ambiguous-depth.userplot' data = trivia.read_unstranded_userplot( os.path.join(working_dir, friendly_name+suffix) ) assert len(seq) == len(data) for i in xrange(len(seq)): if good[i]: if is_core: good[i] = data[i] >= mincov else: good[i] = data[i] < mincov #Close holes start = -maxdiff-1 n_holes = 0 for i in xrange(len(seq)): if good[i]: if 0 < i-start <= maxdiff: for j in xrange(start,i): good[j] = True n_holes += 1 start = i+1 print 'Closed', grace.pretty_number(n_holes), 'holes' f = open(path.join(output_dir, '%s-%s.fa' % (friendly_name,what)), 'wb') io.write_fasta(f, name, ''.join([ (seq[i] if good[i] else 'N') for i in xrange(len(seq)) ]) ) f.close() f = open(path.join(output_dir, '%s-%s_masked.fa' % (friendly_name,what)), 'wb') io.write_fasta(f, name, ''.join([ (seq[i] if good[i] else seq[i].lower()) for i in xrange(len(seq)) ]) ) f.close() f_good = open(path.join(output_dir, '%s-%s_parts.fa' % (friendly_name,what)), 'wb') f_nongood = open(path.join(output_dir, '%s-non%s_parts.fa' % (friendly_name,what)), 'wb') start = 0 n_good = [0] n_good_bases = [0] def emit(i): if i-start < minsize: return if good[start]: n_good[0] += 1 n_good_bases[0] += i-start io.write_fasta( f_good if good[start] else f_nongood, '%s:%d..%d' % (name, start+1,i), seq[start:i] ) for i in xrange(1,len(seq)): if good[i] != good[start]: emit(i) start = i emit(len(seq)) f_nongood.close() f_good.close() print grace.pretty_number(sum(good)), 'bases are '+what+', of', grace.pretty_number(len(seq)), 'in reference sequence' print grace.pretty_number(n_good[0]), 'parts at least', grace.pretty_number(minsize), 'bases long with', grace.pretty_number(n_good_bases[0]), 'total bases' print
def main(args): mincov, args = grace.get_option_value(args, '--mincov', int, 1) maxdiff, args = grace.get_option_value(args, '--maxdiff', int, 16) minsize, args = grace.get_option_value(args, '--minsize', int, 200) what, args = grace.get_option_value(args, '--what', as_core_or_unique, 'core') is_core = (what == 'core') grace.expect_no_further_options(args) if len(args) < 2: print >> sys.stderr, HELP raise grace.Help_shown() output_dir, working_dirs = args[0], args[1:] assert not path.exists(path.join(output_dir, 'reference.fa')), \ 'Output directory not given' if not path.exists(output_dir): os.mkdir(output_dir) for name, seq in io.read_sequences( path.join(working_dirs[0], 'reference.fa')): print name friendly_name = grace.filesystem_friendly_name(name) good = [True] * len(seq) for working_dir in working_dirs: if is_core: suffix = '-depth.userplot' else: suffix = '-ambiguous-depth.userplot' data = trivia.read_unstranded_userplot( os.path.join(working_dir, friendly_name + suffix)) assert len(seq) == len(data) for i in xrange(len(seq)): if good[i]: if is_core: good[i] = data[i] >= mincov else: good[i] = data[i] < mincov #Close holes start = -maxdiff - 1 n_holes = 0 for i in xrange(len(seq)): if good[i]: if 0 < i - start <= maxdiff: for j in xrange(start, i): good[j] = True n_holes += 1 start = i + 1 print 'Closed', grace.pretty_number(n_holes), 'holes' f = open(path.join(output_dir, '%s-%s.fa' % (friendly_name, what)), 'wb') io.write_fasta( f, name, ''.join([(seq[i] if good[i] else 'N') for i in xrange(len(seq))])) f.close() f = open( path.join(output_dir, '%s-%s_masked.fa' % (friendly_name, what)), 'wb') io.write_fasta( f, name, ''.join([(seq[i] if good[i] else seq[i].lower()) for i in xrange(len(seq))])) f.close() f_good = open( path.join(output_dir, '%s-%s_parts.fa' % (friendly_name, what)), 'wb') f_nongood = open( path.join(output_dir, '%s-non%s_parts.fa' % (friendly_name, what)), 'wb') start = 0 n_good = [0] n_good_bases = [0] def emit(i): if i - start < minsize: return if good[start]: n_good[0] += 1 n_good_bases[0] += i - start io.write_fasta(f_good if good[start] else f_nongood, '%s:%d..%d' % (name, start + 1, i), seq[start:i]) for i in xrange(1, len(seq)): if good[i] != good[start]: emit(i) start = i emit(len(seq)) f_nongood.close() f_good.close() print grace.pretty_number( sum(good)), 'bases are ' + what + ', of', grace.pretty_number( len(seq)), 'in reference sequence' print grace.pretty_number( n_good[0]), 'parts at least', grace.pretty_number( minsize), 'bases long with', grace.pretty_number( n_good_bases[0]), 'total bases' print