def iterate_table( infile ):
for line in infile:
if line[0] == "#": continue
unaligned_pair = AlignedPairs.UnalignedPair()
unaligned_pair.Read(line)
yield unaligned_pair
def iterate_list(infile, idx1, idx2=None):
fasta1 = IndexedFasta.IndexedFasta(idx1)
if idx2 is None:
fasta2 = fasta1
else:
fasta2 = IndexedFasta.IndexedFasta(idx2)
first = True
for line in infile:
if line[0] == "#":
continue
id1, id2 = line[:-1].split("\t")[:2]
try:
yield AlignedPairs.UnalignedPair(
token1=id1,
sequence1=fasta1.getSequence(id1),
token2=id2,
sequence2=fasta2.getSequence(id2))
except KeyError, msg:
if first:
first = False
continue
raise KeyError(msg)
def iterate_double_fasta ( fn1, fn2 ):
iterator = FastaIterator.iterate_together( fn1, fn2 )
for seq1, seq2 in iterator:
yield AlignedPairs.UnalignedPair(
token1 = seq1.title,
sequence1 = seq1.sequence,
token2 = seq2.title,
sequence2 = seq2.sequence )
def iterate_single_fasta ( fn1 ):
iterator = FastaIterator.FastaIterator( fn1 )
while 1:
seq1, seq2 = iterator.next(), iterator.next()
yield AlignedPairs.UnalignedPair(
token1 = seq1.title,
sequence1 = seq1.sequence,
token2 = seq2.title,
sequence2 = seq2.sequence )
def main( argv = None ):
if argv == None: argv = sys.argv
parser = E.OptionParser( version = "%prog version: $Id: align_pairs.py 2781 2009-09-10 11:33:14Z andreas $", usage = globals()["__doc__"] )
parser.add_option("--skip-statistics", dest="skip_stats", action="store_true",
help="do not compute alignment statistics [%default]." )
parser.add_option("--method", dest="methods", type="choice", action="append",
choices=("dialign", "clustal", "blastz", "nw", "sw", "dba", "dialignlgs" ),
help="alignment method [%default]." )
parser.add_option("--anchor-alignment", dest="anchor_alignment", type="int",
help="anchor alignmet with xxx residues [%default]." )
parser.add_option("--output-format", dest="output_formats", type="choice", action="append",
choices=("fasta", "stats", "psl" ),
help="anchor alignment with xxx residues [%default]." )
parser.add_option("--input-format", dest="input_format", type="choice",
choices=("fasta", "list" ),
help="input format of stdin [%default]." )
parser.add_option("--output-filename-pattern", dest="output_filename_pattern", type="string",
help="output pattern for multiple files [%default]." )
parser.add_option("--filename-sequences1", dest="filename_sequences1", type="string",
help="first indexed input filename with sequences [%default]." )
parser.add_option("--filename-sequences2", dest="filename_sequences2", type="string",
help="second indexed input filename with sequences [%default]." )
parser.add_option("--options-blastz", dest="options_blastz", type="string",
help="command line options for blastz [%default]." )
parser.set_defaults(
skip_stats = False,
methods = [],
output_formats = [],
input_format = "fasta",
output_filename_pattern = None,
filename_sequences1 = None,
filename_sequences2 = None,
anchor_alignment = 0,
options_blastz = "C=2 B=1 T=0 W=6 K=2200" )
(options, args) = E.Start( parser, add_pipe_options = True )
if len(options.methods) == 0:
print USAGE
print "please specify an alignment method."
sys.exit(1)
if len(options.output_formats) == 0:
print USAGE
print "please specify at least one output format."
sys.exit(1)
if len(args) == 2:
iterator = iterate_double_fasta( args[0], args[1] )
elif options.filename_sequences1 and options.filename_sequences2:
if len(args) == 0 or (len(args) == 1 and args[0] == "-"):
infile = options.stdin
elif len(args) == 1:
infile = open( args[0], "r")
iterator = iterate_list( infile, options.filename_sequences1, options.filename_sequences2 )
else:
iterator = iterate_single_fasta( options.stdin )
npairs, ntoken_pairs = 0, 0
ninput, nskipped, nerrors = 0, 0, 0
outfile_table = None
outfile_fasta = None
outfile_psl = None
if "table" in options.output_formats:
outfile_table = getFile( "table ", options )
outfile_table.write( """# CATEGORY: category [intron|exon]
# METHOD: alignment method
# TOKEN: name
# ID: segment id
# TOTAL: number of segments
# LEN: length of segment
# NALIGNED: number of aligned positions
# PALIGNED: percentage of aligned positions
# IDENT: number of identical positions
# TRANSIT: number of transitions
# TRANSVERS: number of transversion
# MATCHES: number of matching positions
# PIDENT: percentage of identical positions
# PTRANSIT: precentage of transitions
# PTRANSVERS: precentage of transversion
# BLOCKSIZES: alignment, length of blocks
# GAPS: gap sizes in sequence 1/2
CATEGORY\tMETHOD\tTOKEN1\tID1\tTOTAL1\tLEN1\tTOKEN2\tID2\tTOTAL2\tLEN2\tNALIGNED\tPALIGNED\tIDENT\tTRANSIT\tTRANSVER\tMATCHES\tPIDENT\tPTRANSVIT\tPTRANVER\tBLOCKSIZES\tGAPSIZES\tGAPSIZES\tTYPE1\tTYPE2\n""")
if "fasta" in options.output_formats:
outfile_fasta = getFile( "fasta", options )
if "psl" in options.output_formats:
outfile_psl = getFile( "psl", options )
## setup alignment objects
for unaligned_pair in iterator:
ninput += 1
for method in options.methods:
pair = AlignedPairs.AlignedPair( unaligned_pair )
pair.mOptionsBlastZ = options.options_blastz
try:
pair.Align( method, anchor = options.anchor_alignment )
except AlignedPairs.AlignmentError, msg:
if options.loglevel >= 1:
options.stdlog.write( "# %s - %s: %s\n" % (msg, unaligned_pair.mToken1, unaligned_pair.mToken2))
if options.loglevel >= 2:
options.stdlog.write( "# input=%s\n" % (str(unaligned_pair)))
nskipped += 1
continue
if outfile_table:
outfile_table.write( str(pair) + "\n" )
if outfile_fasta:
outfile_fasta.write( ">%s\n%s\n>%s\n%s\n" % (pair.mToken1, pair.mAlignedSequence1, pair.mToken2, pair.mAlignedSequence2 ) )
if outfile_psl:
entry = Blat.Match()
entry.mQueryId, entry.mSbjctId = pair.mToken1, pair.mToken2
entry.strand = pair.strand
entry.fromMap( pair.mAlignment )
outfile_psl.write( str(entry) + "\n" )
npairs += 1
def WriteExons(token1, peptide1, cds1, transcript1, token2, peptide2, cds2,
transcript2, peptide_map_a2b):
if param_loglevel >= 3:
for cd in cds1:
print "#", str(cd)
for cd in cds2:
print "#", str(cd)
print "# peptide_map_a2b", str(
alignlib_lite.AlignmentFormatExplicit(peptide_map_a2b))
sys.stdout.flush()
dna_map_a2b = Genomics.AlignmentProtein2CDNA(peptide_map_a2b, cds1, cds2)
if len(cds1) != len(cds2):
if param_loglevel >= 4:
print "" # WARNING: different number of exons!"
seq1 = alignlib_lite.makeSequence(transcript1)
seq2 = alignlib_lite.makeSequence(transcript2)
tmp_map_a2b = alignlib_lite.makeAlignmentVector()
dialign = WrapperDialign.Dialign("-n")
dialignlgs = WrapperDialign.Dialign("-n -it -thr 2 -lmax 30 -smin 8")
dba = WrapperDBA.DBA()
#clustal = WrapperClustal.Clustal()
matrix, gop, gep = global_substitution_matrix
alignator_nw = alignlib_lite.makeAlignatorDPFullDP(
alignlib_lite.ALIGNMENT_GLOBAL, gop, gep, matrix)
alignator_sw = alignlib_lite.makeAlignatorDPFullDP(
alignlib_lite.ALIGNMENT_LOCAL, gop, gep, matrix)
# concatenated alignments for exons:
# 1: only the common parts
ali_common1 = ""
ali_common2 = ""
e1, e2 = 0, 0
while cds1[e1].mGenomeTo <= dna_map_a2b.getRowFrom():
e1 += 1
while cds2[e2].mGenomeTo <= dna_map_a2b.getColFrom():
e2 += 1
nskipped, nerrors = 0, 0
if param_loglevel >= 5:
nmapped = 0
for x in range(dna_map_a2b.getRowFrom(), dna_map_a2b.getRowTo() + 1):
if dna_map_a2b.mapRowToCol(x) >= 0:
nmapped += 1
print "# nmapped=", nmapped
print str(alignlib_lite.AlignmentFormatEmissions(dna_map_a2b))
# declare alignments used
map_intron_a2b = alignlib_lite.makeAlignmentVector()
result = Exons.CompareGeneStructures(cds1,
cds2,
map_cmp2ref=peptide_map_a2b)
if param_loglevel >= 2:
print result.Pretty("#")
nskipped_exons, nskipped_introns = 0, 0
last_e1, last_e2 = None, None
for link in result.mEquivalences:
if link.mCoverage <= param_min_exon_coverage:
nskipped_exons += 1
continue
e1, e2 = link.mId1, link.mId2
c1 = cds1[e1]
c2 = cds2[e2]
exon_fragment1 = transcript1[c1.mGenomeFrom:c1.mGenomeTo]
exon_fragment2 = transcript2[c2.mGenomeFrom:c2.mGenomeTo]
#######################################################################
# write unaligned exons
if param_write_exons:
pair = AlignedPairs.UnalignedPair()
pair.mCategory = "exon"
pair.mToken1 = token1
pair.mId1 = e1 + 1
pair.mNum1 = len(cds1)
pair.mLen1 = len(exon_fragment1)
pair.mSequence1 = exon_fragment1
pair.mToken2 = token2
pair.mId2 = e2 + 1
pair.mNum2 = len(cds2)
pair.mLen2 = len(exon_fragment2)
pair.mSequence2 = exon_fragment2
pair.mFrom1, pair.mTo1 = c1.mGenomeFrom, c1.mGenomeTo,
pair.mFrom2, pair.mTo2 = c2.mGenomeFrom, c2.mGenomeTo,
print str(pair)
sys.stdout.flush()
#######################################################################
# build alignment for overlap of both exons
# tmp_map_a2b.clear()
# alignlib_lite.copyAlignment( tmp_map_a2b, dna_map_a2b,
# c1.mGenomeFrom + 1, c1.mGenomeTo )
# if param_loglevel >= 5:
# print "# alignment: %i-%i" % (c1.mGenomeFrom + 1, c1.mGenomeTo)
# for x in alignlib_lite.writeAlignmentTable( tmp_map_a2b ).split("\n"):
# print "#", x
# if tmp_map_a2b.getLength() == 0:
# if param_loglevel >= 1:
# print "# WARNING: empty alignment between exon %i (from %i to %i) and exon %i" % \
## (e1,c1.mGenomeFrom + 1, c1.mGenomeTo, e2)
# print "## peptide_map_a2b", peptide_map_a2b.getRowFrom(), peptide_map_a2b.getRowTo(),\
## peptide_map_a2b.getColFrom(), peptide_map_a2b.getColTo(), \
# Alignlib.writeAlignmentCompressed(peptide_map_a2b)
# print "## dna_map_a2b", dna_map_a2b.getRowFrom(), dna_map_a2b.getRowTo(),\
## dna_map_a2b.getColFrom(), dna_map_a2b.getColTo(), \
# Alignlib.writeAlignmentCompressed(dna_map_a2b)
# for cd in cds1: print "##", str(cd)
# for cd in cds2: print "##", str(cd)
## nerrors += 1
# continue
## data = map(lambda x: x.split("\t"), alignlib_lite.writePairAlignment( seq1, seq2, tmp_map_a2b ).split("\n"))
# if "caligned" in param_write_exons :
# print "exon\tcaligned\t%s\t%i\t%s\t%i\t%s\t%s\t%s\t%s\t%s\t%s" % ( token1, e1,
## token2, e2,
## data[0][0], data[0][2],
## data[1][0], data[1][2],
# data[0][1], data[1][1] )
## ali_common1 += data[0][1]
## ali_common2 += data[1][1]
#######################################################################
# write alignment of introns for orthologous introns
# orthologous introns are between orthologous exons
if param_write_introns:
if last_e1 is not None:
if e1 - last_e1 != 1 or e2 - last_e2 != 1:
nskipped_introns += 1
else:
pair = AlignedPairs.UnalignedPair()
intron_from1 = cds1[e1 - 1].mGenomeTo
intron_to1 = cds1[e1].mGenomeFrom
intron_from2 = cds2[e2 - 1].mGenomeTo
intron_to2 = cds2[e2].mGenomeFrom
intron_fragment1 = transcript1[intron_from1:intron_to1]
intron_fragment2 = transcript2[intron_from2:intron_to2]
if len(intron_fragment1) == 0 or len(
intron_fragment2) == 0:
print "## ERROR: empty intron fragments: %i-%i out of %i and %i-%i out of %i." %\
(intron_from1, intron_to1, len(transcript1),
intron_from2, intron_to2, len(transcript2))
continue
pair.mCategory = "intron"
pair.mToken1 = token1
pair.mId1 = e1 + 1
pair.mNum1 = len(cds1) - 1
pair.mLen1 = len(intron_fragment1)
pair.mFrom1 = intron_from1
pair.mTo1 = intron_to1
pair.mSequence1 = intron_fragment1
pair.mToken2 = token2
pair.mId2 = e2 + 1
pair.mNum1 = len(cds2) - 1
pair.mLen2 = len(intron_fragment2)
pair.mFrom2 = intron_from2
pair.mTo2 = intron_to2
pair.mSequence2 = intron_fragment2
if (param_min_intron_length and len(intron_fragment1) < param_min_intron_length) or \
(param_min_intron_length and len(intron_fragment2) < param_min_intron_length) or \
(param_max_intron_length and len(intron_fragment1) > param_max_intron_length) or \
(param_max_intron_length and len(intron_fragment2) > param_max_intron_length):
if param_loglevel >= 1:
print "# skipped: fragment lengths out of bounds for: %s\t%s\t%s\t%s\t%i\t%i" %\
(token1, e1, token2, e2,
len(intron_fragment1),
len(intron_fragment2))
sys.stdout.flush()
nskipped += 1
print str(pair)
# else:
## anchored_from1 = intron_from1 - param_extend_introns
## anchored_to1 = intron_to1 + param_extend_introns
## anchored_from2 = intron_from2 - param_extend_introns
## anchored_to2 = intron_to2 + param_extend_introns
## anchored_fragment1 = transcript1[anchored_from1:anchored_to1]
## anchored_fragment2 = transcript2[anchored_from2:anchored_to2]
# for method in param_write_introns:
# if param_loglevel >= 2:
# print "## aligning with method %s" % method
# sys.stdout.flush
# map_intron_a2b.clear()
# if method == "unaligned":
## from1, to1, ali1, from2, to2, ali2 = 0, 0, intron_fragment1, 0, 0, intron_fragment2
# elif method in ("dialigned", "dbaligned", "clusaligned", "dialignedlgs"):
## tmp_intron_a2b = alignlib_lite.makeAlignmentVector()
# if param_loglevel >= 1:
# print "# aligning with method %s two fragments of length %i and %i" % (method,
# len(anchored_fragment1),
# len(anchored_fragment2))
# sys.stdout.flush()
# if method == "dialigned":
## result = dialign.Align( anchored_fragment1, anchored_fragment2, tmp_intron_a2b )
# elif method == "dialignedlgs":
## result = dialignlgs.Align( anchored_fragment1, anchored_fragment2, tmp_intron_a2b )
# elif method == "dbaligned":
## result = dba.Align( anchored_fragment1, anchored_fragment2, tmp_intron_a2b )
# elif method == "clusaligned":
## result = clustal.Align( anchored_fragment1, anchored_fragment2, tmp_intron_a2b )
# if not result or result.getLength() == 0:
# if param_loglevel >= 1:
# print "# Error: empty intron alignment"
# sys.stdout.flush()
## nerrors += 1
# continue
## tmp_intron_a2b.moveAlignment( anchored_from1, anchored_from2 )
# alignlib_lite.copyAlignment( map_intron_a2b, tmp_intron_a2b,
## intron_from1 + 1, intron_to1,
# intron_from2 + 1, intron_to2 )
# elif method == "nwaligned":
## seq1.useSegment( cds1[e1-1].mGenomeTo + 1, cds1[e1].mGenomeFrom )
## seq2.useSegment( cds2[e2-1].mGenomeTo + 1, cds2[e2].mGenomeFrom )
## alignator_nw.Align( seq1, seq2, map_intron_a2b )
# seq1.useFullLength()
# seq2.useFullLength()
# elif method == "swaligned":
## seq1.useSegment( cds1[e1-1].mGenomeTo + 1, cds1[e1].mGenomeFrom )
## seq2.useSegment( cds2[e2-1].mGenomeTo + 1, cds2[e2].mGenomeFrom )
## alignlib_lite.performIterativeAlignment( map_intron_a2b, seq1, seq2, alignator_sw, param_min_score_sw )
# seq1.useFullLength()
# seq2.useFullLength()
# else:
## raise "unknown method %s" % method
# if map_intron_a2b.getLength() > 0:
# if param_compress:
## from1, to1 = map_intron_a2b.getRowFrom(), map_intron_a2b.getRowTo()
## from2, to2 = map_intron_a2b.getColFrom(), map_intron_a2b.getColTo()
## ali1, ali2 = Alignlib.writeAlignmentCompressed( map_intron_a2b )
# else:
# data = map(lambda x: x.split("\t"),
# alignlib_lite.writePairAlignment( seq1, seq2, map_intron_a2b ).split("\n"))
# if len(data) < 2:
## data=[ ( 0, "", 0), (0, "", 0)]
## from1, ali1, to1 = data[0]
## from2, ali2, to2 = data[1]
# print string.join(map(str, ("intron",
# method,
## token1, e1, len(cds1) - 1, len(intron_fragment1),
## token2, e2, len(cds2) - 1, len(intron_fragment2),
# map_intron_a2b.getNumGaps(),
# map_intron_a2b.getLength(),
## map_intron_a2b.getLength() - map_intron_a2b.getNumGaps(),
## from1, to1, ali1,
## from2, to2, ali2,
## intron_from1, intron_to1,
# intron_from2, intron_to2)), "\t")
# sys.stdout.flush()
last_e1, last_e2 = e1, e2
##########################################################################
# write concatenated exons
# for method in param_write_exons:
# if method == "common":
# print "exon\tcommon\t%s\t%i\t%s\t%i\t%s\t%s\t%s\t%s\t%s\t%s" % ( token1, 0,
## token2, 0,
## 0, 0,
## 0, 0,
# ali_common1, ali_common2 )
# elif method == "exons":
# Write full alignment without gaps.
# This will not care about exon boundaries and gaps.
# data = map(lambda x: x.split("\t"),
# alignlib_lite.writePairAlignment( seq1, seq2, dna_map_a2b ).split("\n"))
# try:
## from1, s1, to1, from2, s2, to2 = data[0] + data[1]
# except ValueError:
## from1, to1, from2, to2 = 0, 0, 0, 0
## s1, s2 = "", ""
## nerrors += 1
# except IndexError:
## from1, to1, from2, to2 = 0, 0, 0, 0
## s1, s2 = "", ""
## nerrors += 1
# if from1:
# if len(s1) != len(s2):
# print "# WARNING: alignment of different lengths: %i and %i" % (len(s1), len(s2))
## nerrors += 1
## from1, to1, from2, to2 = 0, 0, 0, 0
## s1, s2 = "", ""
# else:
## a1, a2 = [], []
# for x in range( min(len(s1), len(s2)) ):
# if s1[x] != "-" and s2[x] != "-":
## a1.append( s1[x] )
## a2.append( s2[x] )
## s1 = string.join(a1, "")
## s2 = string.join(a2, "")
# print "exon\texons\t%s\t%i\t%s\t%i\t%s\t%s\t%s\t%s\t%s\t%s" % ( (token1, 0,
## token2, 0,
## from1, to1,
## from2, to2,
# s1, s2 ) )
# elif method == "full":
# write full alignment (do not care about exon boundaries)
# data = map(lambda x: x.split("\t"),
# alignlib_lite.writePairAlignment( seq1, seq2, dna_map_a2b ).split("\n"))
## if len(data) < 2: data=[ ( 0, "", 0), (0, "", 0)]
# print "exon\tfull\t%s\t%i\t%s\t%i\t%s\t%s\t%s\t%s\t%s\t%s" % ( token1, 0,
## token2, 0,
## data[0][0], data[0][2],
## data[1][0], data[1][2],
# data[0][1], data[1][1] )
if param_loglevel >= 3:
print "# skipped_exons=%i, skipped_introns=%i" % (nskipped_exons,
nskipped_introns)
return nerrors, nskipped