def reconstruct_indelfo_from_indel_list(indel_list, line, iseq, debug=False): # old-style files
if 'reversed_seq' in indel_list: # handle super-old files
print '%s encountered file with super old, unhandled indel format, proceeding, but indel info may be inconsistent' % (utils.color('red', 'error'))
return
line['indelfos'][iseq] = get_empty_indel()
if len(indel_list) == 0:
return
ifo_positions = [ifo['pos'] for ifo in indel_list]
if len(ifo_positions) != len(set(ifo_positions)):
print '%s two indels at the same position, everything will be kinda messed up' % utils.color('red', 'error')
ifos_by_pos = {ifo['pos'] : ifo for ifo in indel_list}
qr_gap_seq, gl_gap_seq = [], []
iqr, igl, iindel = 0, 0, 0
if debug:
print len(line['input_seqs'][iseq]), line['input_seqs'][iseq]
print len(line['naive_seq']), line['naive_seq']
while iqr < len(line['input_seqs'][iseq]):
if debug:
print ' %3d %3d' % (iqr, igl),
if iindel in ifos_by_pos:
ifo = ifos_by_pos[iindel]
if ifo['type'] == 'insertion':
if ifo['seqstr'] != line['input_seqs'][iseq][iqr : iqr + ifo['len']]:
print '%s indel info seqstr doesn\'t match input seq str:' % utils.color('red', 'error')
utils.color_mutants(ifo['seqstr'], line['input_seqs'][iseq][iqr : iqr + ifo['len']], align=True, print_result=True, extra_str=' ')
qr_gap_seq += ifo['seqstr'].split()
gl_gap_seq += [ifo['len'] * utils.gap_chars[0]]
if debug:
print ' %s %s' % (ifo['seqstr'].split(), [ifo['len'] * utils.gap_chars[0]])
iqr += ifo['len']
else:
if ifo['seqstr'] != line['naive_seq'][igl : igl + ifo['len']]:
print '%s indel info seqstr doesn\'t match naive seq str:' % utils.color('red', 'error')
utils.color_mutants(ifo['seqstr'], line['naive_seq'][igl : igl + ifo['len']], align=True, print_result=True, extra_str=' ')
qr_gap_seq += [ifo['len'] * utils.gap_chars[0]]
gl_gap_seq += ifo['seqstr'].split()
if debug:
print ' %s %s' % ([ifo['len'] * utils.gap_chars[0]], ifo['seqstr'].split())
igl += ifo['len']
del ifos_by_pos[iindel]
iindel += ifo['len']
else:
qr_gap_seq += [line['input_seqs'][iseq][iqr]]
gl_gap_seq += [line['naive_seq'][igl]]
if debug:
print ' %s %s' % (line['input_seqs'][iseq][iqr], line['naive_seq'][igl])
iqr += 1
igl += 1
iindel += 1
line['indelfos'][iseq]['qr_gap_seq'] = ''.join(qr_gap_seq)
line['indelfos'][iseq]['gl_gap_seq'] = ''.join(gl_gap_seq)
line['indelfos'][iseq]['indels'] = indel_list
line['indelfos'][iseq]['reversed_seq'] = line['indel_reversed_seqs'][iseq]
line['indelfos'][iseq]['genes'] = {r : line[r + '_gene'] for r in utils.regions}
if debug:
print ' reconstructed indelfo'
print get_dbg_str(line['indelfos'][iseq])
def check_single_ifo(old_ifo, new_ifo):
if debug:
print ' len %d pos %d seqstr %s' % (
old_ifo['len'], old_ifo['pos'], old_ifo['seqstr']),
if new_ifo != old_ifo:
if debug:
print ' %s' % utils.color('red', 'nope')
new_seqstr, old_seqstr = utils.color_mutants(
old_ifo['seqstr'],
new_ifo['seqstr'],
return_ref=True,
align=True) #len(old_ifo['seqstr']) != len(new_ifo['seqstr']))
if print_on_err:
print ' pos %d --> %s len %d --> %s seqstr %s --> %s' % (
old_ifo['pos'],
utils.color(
None if new_ifo['pos'] == old_ifo['pos'] else 'red',
'%d' % new_ifo['pos']), old_ifo['len'],
utils.color(
None if new_ifo['len'] == old_ifo['len'] else 'red',
'%d' % new_ifo['len']), old_seqstr, new_seqstr)
return False
else:
if debug:
print ' %s' % utils.color('green', 'ok')
return True
def add_new_allele(glfo, newfo, remove_template_genes, debug=False):
"""
Add a new allele to <glfo>, specified by <newfo> which is of the
form: {'template-gene' : 'IGHV3-71*01', 'gene' : 'IGHV3-71*01+C35T.T47G', 'seq' : 'ACTG yadda yadda CGGGT'}
If <remove_template_genes>, we also remove 'template-gene' from <glfo>.
"""
template_gene = newfo['template-gene']
region = utils.get_region(template_gene)
if template_gene not in glfo['seqs'][region]:
raise Exception('unknown template gene %s' % template_gene)
new_gene = newfo['gene']
if region == 'v':
glfo['cyst-positions'][new_gene] = glfo['cyst-positions'][template_gene]
elif region == 'j':
glfo['tryp-positions'][new_gene] = glfo['tryp-positions'][template_gene]
glfo['seqs'][region][new_gene] = newfo['seq']
if debug:
print ' adding new allele to glfo:'
print ' template %s %s' % (glfo['seqs'][region][template_gene], utils.color_gene(template_gene))
print ' new %s %s' % (utils.color_mutants(glfo['seqs'][region][template_gene], newfo['seq']), utils.color_gene(new_gene))
if remove_template_genes:
remove_gene(glfo, template_gene, debug=True)
def print_match(self, region, gene, query_seq, score, glbounds, qrbounds, codon_pos, warnings, skipping=False):
if self.debug < 2:
return
out_str_list = []
buff_str = (20 - len(gene)) * ' '
tmp_val = score
if self.args.apply_choice_probs_in_sw and self.get_choice_prob(region, gene) != 0.0:
tmp_val = score / self.get_choice_prob(region, gene)
if self.args.apply_choice_probs_in_sw:
out_str_list.append('%8s%s%s%9.1e * %3.0f = %-6.1f' % (' ', utils.color_gene(gene), buff_str, self.get_choice_prob(region, gene), tmp_val, score))
else:
out_str_list.append('%8s%s%s%9s%3s %6.0f ' % (' ', utils.color_gene(gene), '', '', buff_str, score))
out_str_list.append('%4d%4d %s\n' % (glbounds[0], glbounds[1], self.germline_seqs[region][gene][glbounds[0]:glbounds[1]]))
out_str_list.append('%46s %4d%4d' % ('', qrbounds[0], qrbounds[1]))
out_str_list.append(' %s ' % (utils.color_mutants(self.germline_seqs[region][gene][glbounds[0]:glbounds[1]], query_seq[qrbounds[0]:qrbounds[1]])))
if region != 'd':
out_str_list.append('(%s %d)' % (utils.conserved_codon_names[region], codon_pos))
if warnings[gene] != '':
out_str_list.append('WARNING ' + warnings[gene])
if skipping:
out_str_list.append('skipping!')
if self.args.outfname is None:
print ''.join(out_str_list)
else:
out_str_list.append('\n')
self.outfile.write(''.join(out_str_list))
def add_new_allele(glfo, newfo, remove_template_genes=False, debug=False):
"""
Add a new allele to <glfo>, specified by <newfo> which is of the
form: {'gene' : 'IGHV3-71*01+C35T.T47G', 'seq' : 'ACTG yadda yadda CGGGT', 'template-gene' : 'IGHV3-71*01'}
If <remove_template_genes>, we also remove 'template-gene' from <glfo>.
"""
template_gene = newfo["template-gene"]
region = utils.get_region(template_gene)
if template_gene not in glfo["seqs"][region]:
raise Exception("unknown template gene %s" % template_gene)
new_gene = newfo["gene"]
if region == "v":
glfo["cyst-positions"][new_gene] = glfo["cyst-positions"][template_gene]
elif region == "j":
glfo["tryp-positions"][new_gene] = glfo["tryp-positions"][template_gene]
glfo["seqs"][region][new_gene] = newfo["seq"]
if debug:
print " adding new allele to glfo:"
print " template %s %s" % (glfo["seqs"][region][template_gene], utils.color_gene(template_gene))
print " new %s %s" % (
utils.color_mutants(glfo["seqs"][region][template_gene], newfo["seq"]),
utils.color_gene(new_gene),
)
if remove_template_genes:
remove_gene(glfo, template_gene, debug=True)
def add_new_allele(glfo, newfo, remove_template_genes=False, debug=False):
"""
Add a new allele to <glfo>, specified by <newfo> which is of the
form: {'gene' : 'IGHV3-71*01+C35T.T47G', 'seq' : 'ACTG yadda yadda CGGGT', 'template-gene' : 'IGHV3-71*01'}
If <remove_template_genes>, we also remove 'template-gene' from <glfo>.
"""
template_gene = newfo['template-gene']
region = utils.get_region(template_gene)
if template_gene not in glfo['seqs'][region]:
raise Exception('unknown template gene %s' % template_gene)
new_gene = newfo['gene']
if region == 'v':
glfo['cyst-positions'][new_gene] = glfo['cyst-positions'][template_gene]
elif region == 'j':
glfo['tryp-positions'][new_gene] = glfo['tryp-positions'][template_gene]
glfo['seqs'][region][new_gene] = newfo['seq']
if debug:
print ' adding new allele to glfo:'
print ' template %s %s' % (glfo['seqs'][region][template_gene], utils.color_gene(template_gene))
print ' new %s %s' % (utils.color_mutants(glfo['seqs'][region][template_gene], newfo['seq']), utils.color_gene(new_gene))
if remove_template_genes:
remove_gene(glfo, template_gene, debug=True)
def add_new_allele(self, gene, fitfo, n_candidate_snps, debug=False):
# figure out what the new nukes are
old_seq = self.glfo['seqs'][utils.get_region(gene)][gene]
new_seq = old_seq
mutfo = {}
for pos in sorted(fitfo['candidates'][n_candidate_snps]):
obs_counts = {nuke : self.counts[gene][pos][n_candidate_snps][nuke] for nuke in utils.nukes} # NOTE it's super important to only use the counts from sequences with <n_candidate_snps> total mutations
sorted_obs_counts = sorted(obs_counts.items(), key=operator.itemgetter(1), reverse=True)
original_nuke = self.mfreqer.counts[gene][pos]['gl_nuke']
new_nuke = None
for nuke, _ in sorted_obs_counts: # take the most common one that isn't the existing gl nuke
if nuke != original_nuke:
new_nuke = nuke
break
print ' %3d (%s --> %s)' % (pos, original_nuke, new_nuke),
assert old_seq[pos] == original_nuke
mutfo[pos] = {'original' : original_nuke, 'new' : new_nuke}
new_seq = new_seq[:pos] + new_nuke + new_seq[pos+1:]
new_name, mutfo = glutils.get_new_allele_name_and_change_mutfo(gene, mutfo)
print ''
print ' %s %s' % (old_seq, utils.color_gene(gene))
print ' %s %s' % (utils.color_mutants(old_seq, new_seq), utils.color_gene(new_name))
# and add it to the set of new alleles for this gene
self.new_allele_info.append({
'template-gene' : gene,
'gene' : new_name,
'seq' : new_seq,
'aligned-seq' : None
})
def trim_and_remove_genes(region, gene, seq, glfo, template_glfo, debug=False):
nearest_template_gene = glutils.find_nearest_gene_using_names(
template_glfo, gene)
nearest_template_seq = template_glfo['seqs'][region][nearest_template_gene]
# extra_bases = glfo['cyst-positions'][gene] - template_glfo['cyst-positions'][nearest_template_gene] # not right if there's some internal gaps in the alignment
aligned_nearest_template_seq, aligned_seq = utils.align_seqs(
nearest_template_seq, seq)
if debug:
print ' %s' % utils.color_gene(gene)
utils.color_mutants(aligned_nearest_template_seq,
aligned_seq,
print_result=True,
ref_label='template ',
extra_str=' ')
if aligned_seq[0] not in utils.gap_chars and aligned_nearest_template_seq[
0] not in utils.gap_chars:
if debug:
print ' ok'
elif aligned_seq[0] in utils.gap_chars:
if debug:
print ' %s, removing' % utils.color('red', 'too small')
glutils.remove_gene(glfo, gene)
else:
if debug:
print ' extra bases %s' % utils.color_gene(gene)
extra_bases = len(aligned_nearest_template_seq) - len(
aligned_nearest_template_seq.lstrip('-'))
seq = seq[extra_bases:]
if debug:
print ' removed %d bases' % extra_bases
if seq in glfo['seqs'][region].values():
print ' trimmed seq already in glfo under name %s, so removing it' % ' '.join(
[
utils.color_gene(g)
for g, s in glfo['seqs'][region].items() if s == seq
])
glutils.remove_gene(glfo, gene, debug=True)
return
glfo['seqs'][region][gene] = seq
glfo['cyst-positions'][gene] -= extra_bases
# utils.color_mutants(nearest_template_seq, seq, print_result=True, ref_label='template ', align=True, extra_str=' ')
assert utils.codon_unmutated('cyst',
glfo['seqs'][region][gene],
glfo['cyst-positions'][gene],
debug=True)
def print_match(self, region, gene, query_seq, score, glbounds, qrbounds, codon_pos, warnings, skipping=False):
out_str_list = []
buff_str = (20 - len(gene)) * ' '
out_str_list.append('%8s%s%s%9s%3s %6.0f ' % (' ', utils.color_gene(gene), '', '', buff_str, score))
out_str_list.append('%4d%4d %s\n' % (glbounds[0], glbounds[1], self.glfo['seqs'][region][gene][glbounds[0]:glbounds[1]]))
out_str_list.append('%46s %4d%4d' % ('', qrbounds[0], qrbounds[1]))
out_str_list.append(' %s ' % (utils.color_mutants(self.glfo['seqs'][region][gene][glbounds[0]:glbounds[1]], query_seq[qrbounds[0]:qrbounds[1]])))
if region != 'd':
out_str_list.append('(%s %d)' % (utils.conserved_codons[region], codon_pos))
if warnings[gene] != '':
out_str_list.append('WARNING ' + warnings[gene])
if skipping:
out_str_list.append('skipping!')
print ''.join(out_str_list)
def print_lines(nseq_info, ref_seq, namestr, namecolor):
assert nseq_info == sorted(nseq_info,
key=operator.itemgetter(1),
reverse=True)
total_prob = 0.
for naive_seq, prob in nseq_info:
print ' %s %5.2f %s' % (utils.color_mutants(
naive_seq if ref_seq is None else ref_seq,
naive_seq), prob, utils.color(namecolor, namestr))
if ref_seq is None:
ref_seq = naive_seq
if 1. - total_prob < args.prob_to_ignore:
break
total_prob += prob
return ref_seq
def print_match(self,
region,
gene,
query_seq,
score,
glbounds,
qrbounds,
codon_pos,
warnings,
skipping=False):
if self.debug < 2:
return
out_str_list = []
buff_str = (20 - len(gene)) * ' '
tmp_val = score
if self.args.apply_choice_probs_in_sw and self.get_choice_prob(
region, gene) != 0.0:
tmp_val = score / self.get_choice_prob(region, gene)
if self.args.apply_choice_probs_in_sw:
out_str_list.append(
'%8s%s%s%9.1e * %3.0f = %-6.1f' %
(' ', utils.color_gene(gene), buff_str,
self.get_choice_prob(region, gene), tmp_val, score))
else:
out_str_list.append(
'%8s%s%s%9s%3s %6.0f ' %
(' ', utils.color_gene(gene), '', '', buff_str, score))
out_str_list.append(
'%4d%4d %s\n' %
(glbounds[0], glbounds[1],
self.germline_seqs[region][gene][glbounds[0]:glbounds[1]]))
out_str_list.append('%46s %4d%4d' % ('', qrbounds[0], qrbounds[1]))
out_str_list.append(' %s ' % (utils.color_mutants(
self.germline_seqs[region][gene][glbounds[0]:glbounds[1]],
query_seq[qrbounds[0]:qrbounds[1]])))
if region != 'd':
out_str_list.append(
'(%s %d)' % (utils.conserved_codon_names[region], codon_pos))
if warnings[gene] != '':
out_str_list.append('WARNING ' + warnings[gene])
if skipping:
out_str_list.append('skipping!')
if self.args.outfname is None:
print ''.join(out_str_list)
else:
out_str_list.append('\n')
self.outfile.write(''.join(out_str_list))
seq = glfo['seqs'][args.region][gene]
pos = codon_positions[gene]
if pos < ref_pos: # align the codon position in the case that this seq is shorter up to the codon
seq = (ref_pos - pos) * '-' + seq
pos += (ref_pos - pos)
right_pad_str = '' # i think i don't need this any more since i have the align option in color_mutants
# if len(seq) < max_seq_len:
# right_pad_str = (max_seq_len - len(seq)) * ' '
emph_positions = None if args.region == 'd' else [
pos + i for i in range(3)
]
colored_seq, isnps = utils.color_mutants(ref_seq,
seq,
return_isnps=True,
emphasis_positions=emph_positions,
align=True)
seqstrs[igene] += '%s%s' % (colored_seq, right_pad_str)
if len(isnps) > 0:
snpstrs[igene] = '%2d (%s)' % (len(isnps), ' '.join(
[str(i) for i in isnps]))
# ----------------------------------------------------------------------------------------
def print_str(gene, seqstr, snpstr):
return '%s %s %s %s' % (
utils.color_gene(gene, width=gene_str_width), seqstr,
utils.color_gene(gene, width=gene_str_width), snpstr)
def parse_query_text(self, unique_id, query_info):
if len(query_info
) == 0: # one for the query sequence, then one for v, d, and j
print 'no info for', unique_id
return {}
elif len(query_info) < 4:
regions_ok = ''
for info in query_info:
for region in utils.regions:
if 'IGH' + region.upper() in info:
regions_ok += region
for region in utils.regions:
if region not in regions_ok:
print ' ERROR no %s matches' % region
return {}
assert False # shouldn't get here
elif len(query_info) != 4:
print 'info for', unique_id, 'all messed up'
for info in query_info:
print info
sys.exit()
full_qr_seq = query_info[0].replace('>', '').replace(
unique_id, '') # strip off the unique id
full_qr_seq = ''.join(full_qr_seq.split()).upper(
) # strip off white space and uppercase it
assert full_qr_seq == self.seqinfo[unique_id]['seq']
line = {}
line['unique_id'] = unique_id
line['seq'] = full_qr_seq
for ireg in range(len(utils.regions)):
region = utils.regions[ireg]
info = query_info[ireg + 1].splitlines()
while unique_id not in info[
0]: # remove the line marking cdr3 and framework regions
info.pop(0)
if len(info) <= 1:
print info
assert len(info) > 1
assert len(info[0].split()) == 2
qr_seq = info[0].split()[1].upper(
) # this line should be '<unique_id> .............<query_seq>'
imatch = 1 # which match to take
match_name = str(info[imatch].split()[2])
# if 'IGHV3-69' in match_name: # it's not right anyway
# line['failed'] = True
# return line
# while unacceptable_match(match_name, self.germline_seqs):
# imatch += 1
# match_name = str(info[imatch].split()[2])
# print ' new match name: %s' % match_name
gl_seq = info[imatch].split()[4].upper()
if qr_seq.replace('.', '') not in self.seqinfo[unique_id]['seq']:
if self.args.debug:
print ' qr_seq not foundin seqinfo'
line['failed'] = True
return line
if self.args.debug:
print ' ', region, match_name
print ' gl', gl_seq
print ' ', qr_seq
# replace the dots (gaps) in the gl match
new_qr_seq, new_gl_seq = [], []
for inuke in range(min(len(qr_seq), len(gl_seq))):
if gl_seq[inuke] == '.':
pass
else:
new_qr_seq.append(
qr_seq[inuke]
) # this should only be out of range if the v match extends through the whole query sequence, i.e. friggin never
new_gl_seq.append(gl_seq[inuke])
for inuke in range(len(gl_seq), len(qr_seq)):
new_qr_seq.append(qr_seq[inuke])
for inuke in range(len(qr_seq), len(gl_seq)):
new_gl_seq.append(gl_seq[inuke])
qr_seq = ''.join(new_qr_seq)
gl_seq = ''.join(new_gl_seq)
# work out the erosions
qr_ldots = qr_seq.rfind(
'.') + 1 # first strip off any dots on the left of query seq
qr_seq = qr_seq[qr_ldots:]
gl_seq = gl_seq[qr_ldots:]
gl_ldots = gl_seq.rfind(
'.') + 1 # then remove dots on the left of the germline seq
qr_seq = qr_seq[gl_ldots:]
gl_seq = gl_seq[gl_ldots:]
del_5p = qr_ldots + gl_ldots
jf_insertion = ''
if region == 'j':
jf_insertion = qr_seq[len(gl_seq):]
qr_seq = qr_seq[:len(
gl_seq
)] # then strip the right-hand portion of the query sequence that isn't aligned to the germline
del_3p = len(gl_seq) - len(
qr_seq
) # then do the same for the germline overhanging on the right of the query
gl_seq = gl_seq[:len(qr_seq)]
assert len(gl_seq) == len(qr_seq)
new_gl_seq = []
for inuke in range(len(gl_seq)): # replace dashes (matched bases)
assert gl_seq[inuke] != '.' # hoping there's no gaps in here
if gl_seq[inuke] == '-':
new_gl_seq.append(qr_seq[inuke])
else:
new_gl_seq.append(gl_seq[inuke])
gl_seq = ''.join(new_gl_seq)
if match_name not in self.germline_seqs[region]:
print ' ERROR couldn\'t find %s in germlines' % match_name
line['failed'] = True
return line
if self.germline_seqs[region][match_name].find(
gl_seq
) != del_5p: # why the *@*!! can't they make this consistent?
if self.germline_seqs[region][match_name].find(gl_seq) < 0:
print 'whooooaa'
print self.germline_seqs[region][match_name]
print gl_seq
sys.exit()
del_5p += self.germline_seqs[region][match_name].find(gl_seq)
try:
assert del_5p + len(gl_seq) + del_3p + len(
jf_insertion) == len(
self.germline_seqs[region][match_name])
except:
print ' ERROR lengths failed for %s' % unique_id
# print del_5p, len(gl_seq), del_3p, del_5p + len(gl_seq) + del_3p , len(self.germline_seqs[region][match_name])
# print gl_seq
# print self.germline_seqs[region][match_name]
line['failed'] = True
return line
# assert False
if self.args.debug:
utils.color_mutants(gl_seq,
qr_seq,
ref_label='gl ',
extra_str=' ',
print_result=True,
post_str=' del: %d %d' %
(del_5p, del_3p))
# try:
# match_name = joinparser.figure_out_which_damn_gene(self.germline_seqs, match_name, gl_seq, debug=self.args.debug)
# except:
# print 'ERROR couldn\'t figure out the gene for %s' % match_name
# return {}
line[region + '_gene'] = match_name
line[region + '_qr_seq'] = qr_seq
line[region + '_gl_seq'] = gl_seq
line[region + '_5p_del'] = del_5p
line[region + '_3p_del'] = del_3p
if region == 'j':
line['jf_insertion'] = jf_insertion
return line
def make_single_tree(self, partitions, annotations, uid_set, get_fasttrees=False, n_max_cons_seqs=10, debug=False):
# NOTE don't call this externally -- if you want a single tree, call make_trees() with <i_only_cluster> set
def getline(uidstr, uid_set=None):
if uidstr in annotations: # if we have this exact annotation
return annotations[uidstr]
else:
if uid_set is None:
uid_set = set(uidstr.split(':')) # should only get called if it's a singleton
# note that for internal nodes in a fasttree-derived subtree, the uids will be out of order compared the the annotation keys
for line in annotations.values(): # we may actually have the annotation for every subcluster (e.g. if --calculate-alternative-annotations was set), but in case we don't, this is fine
if len(uid_set & set(line['unique_ids'])) > 0: # just take the first one with any overlap. Yeah, it's not necessarily the best, but its naive sequence probably isn't that different, and for just getting the fasttree it reeeeeeaaaallly doesn't matter
return line
raise Exception('couldn\'t find uid %s in annotations' % uid)
def getseq(uid):
line = getline(uid)
return line['seqs'][line['unique_ids'].index(uid)]
def lget(uid_list):
return ':'.join(uid_list)
# check for repeated uids (was only from seed uid, which shouldn't happen any more, but the code below throws an infinite loop if we do, so may as well be careful)
for partition in partitions:
if sum(len(c) for c in partition) > len(set(u for c in partition for u in c)):
repeated_uids = [u for u, count in collections.Counter([u for c in partition for u in c]).items() if count > 1]
raise Exception('found %d uid%s in more than one cluster (%s)' % (len(repeated_uids), utils.plural(len(repeated_uids)), ', '.join(repeated_uids)))
default_edge_length = 999999 # it's nice to have the edges all set to something that's numeric (so the trees print), but also obvious wrong, if we forget to set somebody
assert len(partitions[-1]) == 1
root_label = lget(partitions[-1][0]) # we want the order of the uids in the label to correspond to the order in self.partitions
tns = dendropy.TaxonNamespace([root_label])
root_node = dendropy.Node(taxon=tns.get_taxon(root_label))
root_node.uids = uid_set # each node keeps track of the uids of its children
dtree = dendropy.Tree(taxon_namespace=tns, seed_node=root_node)
if debug:
print ' starting tree with %d leaves' % len(uid_set)
for ipart in reversed(range(len(partitions) - 1)): # dendropy seems to only have fcns to build a tree from the root downward, so we loop starting with the last partition (- 1 is because the last partition is guaranteed to be just one cluster)
for lnode in dtree.leaf_node_iter(): # look for leaf nodes that contain uids from two clusters in this partition, and add those as children
tclusts = [c for c in partitions[ipart] if len(set(c) & lnode.uids) > 0]
if len(tclusts) < 2:
continue
for tclust in tclusts:
ttaxon = dendropy.Taxon(lget(tclust))
tns.add_taxon(ttaxon)
child = lnode.new_child(taxon=ttaxon, edge_length=default_edge_length)
child.uids = set(tclust)
if debug:
print ' ipart %d' % ipart
print ' split node: %d --> %s %s --> %s' % (len(lnode.uids), ' '.join([str(len(tc)) for tc in tclusts]), lnode.taxon.label, ' '.join([c.taxon.label for c in lnode.child_node_iter()]))
# split existing leaves, which are probably not singletons (they're probably from the initial naive sequence collapse step) into subtrees such that each leaf is a singleton
for lnode in dtree.leaf_node_iter():
if len(lnode.uids) == 1:
continue
if get_fasttrees and len(lnode.uids) > 2:
seqfos = [{'name' : uid, 'seq' : getseq(uid)} for uid in lnode.taxon.label.split(':')] # may as well add them in the right order, although I don't think it matters
subtree = treeutils.get_fasttree_tree(seqfos, getline(lnode.taxon.label, uid_set=lnode.uids)['naive_seq'], suppress_internal_node_taxa=True) # note that the fasttree distances get ignored below (no idea if they'd be better than what we set down there, but they probably wouldn't be consistent, so I'd rather ignore them)
for tmpnode in subtree.postorder_node_iter():
if tmpnode.is_leaf():
tmpnode.uids = set([tmpnode.taxon.label])
else:
tmpnode.uids = set([uid for c in tmpnode.child_node_iter() for uid in c.uids])
ttaxon = dendropy.Taxon(lget(tmpnode.uids))
subtree.taxon_namespace.add_taxon(ttaxon)
tmpnode.taxon = ttaxon # ...and use the string of leaf nodes, even though they'll be in the wrong order (I think these get ignored when I call label_nodes() below, but it's still tidier to have them right in the meantime, and anyway since I'm suppressing internal taxa I think I need to set them to something)
if debug:
print ' adding subtree with %d leaves from fastree at leaf node %s' % (len(seqfos), lnode.taxon.label)
print utils.pad_lines(treeutils.get_ascii_tree(dendro_tree=subtree))
dtree.taxon_namespace.add_taxa(subtree.taxon_namespace)
lnode.add_child(subtree.seed_node)
assert len(lnode.child_edges()) == 1 # we're iterating over leaves, so this should always be true
lnode.child_edges()[0].collapse()
else: # just add a star subtree
for uid in lnode.taxon.label.split(':'): # may as well add them in the right order, although I don't think it matters
ttaxon = dendropy.Taxon(uid)
tns.add_taxon(ttaxon)
child = lnode.new_child(taxon=ttaxon, edge_length=default_edge_length)
child.uids = set([uid])
if debug:
print ' added %d singleton children for %s' % (len(lnode.uids), lnode.taxon.label)
# in order to set edge lengths, we need node sequences, so first set leaf node seqs
for lnode in dtree.leaf_node_iter():
assert len(lnode.uids) == 1
lnode.seq = getseq(lnode.taxon.label)
lnode.n_descendent_leaves = 1 # keep track of how many leaf nodes contributed to each node's consensus sequence (these are leaves, so it's trivally 1). This is less accurate than keeping track of all the sequences, but also faster
# then set internal node seqs as the consensus of their children, and set the distance as hamming distance to child seqs
if debug:
print ' adding edge lengths either from fasttree %s or cons seq %s' % (utils.color('blue', 'x'), utils.color('red', 'x'))
min_edge_length = None # setting this is nice for better debug viewing
for node in dtree.postorder_internal_node_iter(): # includes root node
child_cons_seq_counts = [c.n_descendent_leaves for c in node.child_node_iter()]
total_descendent_leaves = sum(child_cons_seq_counts)
if total_descendent_leaves > n_max_cons_seqs: # if there's tons of descendent leaves, we don't want to pass them all to the consensus fcn since it's slow, so we choose them in proportion to their actual proportions, but scaled down to <n_max_cons_seqs>
child_cons_seq_counts = [int(n_max_cons_seqs * csc / float(total_descendent_leaves)) for csc in child_cons_seq_counts]
child_cons_seq_counts = [max(1, csc) for csc in child_cons_seq_counts] # don't eliminate any sequences entirely (this makes the proportions less accurate (in some cases), but is the easy way to handle the case where there's a ton of singleton children
if debug:
print ' %s' % utils.color('green', node.taxon.label)
csc_str = ' (reduced: %s)' % ' '.join([str(csc) for csc in child_cons_seq_counts]) if total_descendent_leaves > n_max_cons_seqs else ''
print ' desc leaves per child: %s%s' % (' '.join(str(c.n_descendent_leaves) for c in node.child_node_iter()), csc_str)
child_seqfos = [{'name' : cn.taxon.label + '-leaf-' + str(il), 'seq' : cn.seq} for cn, count in zip(node.child_node_iter(), child_cons_seq_counts) for il in range(count)]
node.seq = utils.cons_seq(0.01, aligned_seqfos=child_seqfos, tie_resolver_seq=getline(root_label)['naive_seq']) #, debug=debug) # the consensus has an N at every position where the constituent sequences gave a tie. But Ns screw up the distances (especially because once we *get* an N, we can't get rid of it and it's propagated all the way up the tree), and in almost all cases the correct choice should be the naive base, so we use that
node.n_descendent_leaves = total_descendent_leaves
for edge in node.child_edge_iter():
from_fasttree = False
if edge.length == default_edge_length: # otherwise it was set by fasttree, and it's probably better than what we'd get from this (it'd be nice to skip the cons seq stuff for the whole fasttree subtree, but then we don't have the cons seqs we need for later)
edge.length = utils.hamming_distance(edge.head_node.seq, node.seq) / float(len(node.seq))
else:
from_fasttree = True
if min_edge_length is not None:
edge.length = max(min_edge_length, edge.length)
if debug:
print ' %6.3f %s %s' % (edge.length, utils.color('blue' if from_fasttree else 'red', 'x'), edge.head_node.taxon.label)
if debug:
print ' naive seq %s' % getline(root_label)['naive_seq'] # NOTE might be worthwhile to add an edge connecting seed node and the actual naive sequence (i.e. for cases where our approximate naive is off)
print ' root cons seq %s' % utils.color_mutants(getline(root_label)['naive_seq'], dtree.seed_node.seq)
for node in dtree.preorder_node_iter():
del node.uids
del node.seq
del node.n_descendent_leaves
treeutils.label_nodes(dtree, ignore_existing_internal_node_labels=True, ignore_existing_internal_taxon_labels=True, debug=debug)
dtree.update_bipartitions() # probably don't really need this
if debug:
print treeutils.utils.pad_lines(treeutils.get_ascii_tree(dendro_tree=dtree, width=250))
return dtree
def parse_query_text(self, unique_id, query_info):
if len(query_info) == 0: # one for the query sequence, then one for v, d, and j
print 'no info for',unique_id
return {}
elif len(query_info) < 4:
regions_ok = ''
for info in query_info:
for region in utils.regions:
if 'IGH' + region.upper() in info:
regions_ok += region
for region in utils.regions:
if region not in regions_ok:
print ' ERROR no %s matches' % region
return {}
assert False # shouldn't get here
elif len(query_info) != 4:
print 'info for', unique_id, 'all messed up'
for info in query_info:
print info
sys.exit()
full_qr_seq = query_info[0].replace('>', '').replace(unique_id, '') # strip off the unique id
full_qr_seq = ''.join(full_qr_seq.split()).upper() # strip off white space and uppercase it
assert full_qr_seq == self.seqinfo[unique_id]['seq']
line = {}
line['unique_id'] = unique_id
line['seq'] = full_qr_seq
for ireg in range(len(utils.regions)):
region = utils.regions[ireg]
info = query_info[ireg + 1].splitlines()
while unique_id not in info[0]: # remove the line marking cdr3 and framework regions
info.pop(0)
if len(info) <= 1:
print info
assert len(info) > 1
assert len(info[0].split()) == 2
qr_seq = info[0].split()[1].upper() # this line should be '<unique_id> .............<query_seq>'
true_gene = self.seqinfo[unique_id][region + '_gene']
imatch = 1 # which match to take
match_name = str(info[imatch].split()[2])
while match_name in just_always_friggin_skip and len(info) > imatch+1 and len(info[imatch+1].split()) > 2:
imatch += 1
old_one = match_name
match_name = str(info[imatch].split()[2])
if self.args.debug:
print ' %s: taking next match: %s --> %s)' % (unique_id, utils.color_gene(old_one), utils.color_gene(match_name))
infer_gene = match_name
for gset in equivalent_genes:
if match_name in gset and true_gene in gset and match_name != true_gene: # if the true gene and the inferred gene are in the same equivalence set, treat it as correct, i.e. just pretend it inferred the right name
if self.args.debug:
print ' %s: replacing name %s with true name %s' % (unique_id, match_name, true_gene)
infer_gene = true_gene
# ----------------------------------------------------------------------------------------
# skipping bullshit
def skip_gene(gene):
print ' %s in list of genes to skip' % utils.color_gene(gene)
if gene not in genes_actually_skipped:
genes_actually_skipped[gene] = 0
genes_actually_skipped[gene] += 1
line['skip_gene'] = True
if infer_gene not in self.germline_seqs[region]:
print ' couldn\'t find %s in germlines (skipping)' % infer_gene
skip_gene(infer_gene)
return line
if infer_gene in just_always_friggin_skip:
skip_gene(infer_gene)
return line
if true_gene in just_always_friggin_skip:
skip_gene(true)
return line
if not self.args.dont_skip_or15_genes and '/OR1' in true_gene:
skip_gene(true_gene)
return line
if self.args.skip_missing_genes:
if infer_gene in genes_to_skip:
skip_gene(infer_gene)
return line
if true_gene in genes_to_skip:
skip_gene(true_gene)
return line
gl_seq = info[imatch].split()[4].upper()
if qr_seq.replace('.', '') not in self.seqinfo[unique_id]['seq']:
# if self.args.debug:
print ' qr_seq not found in seqinfo'
line['failed'] = True
return line
if self.args.debug:
if utils.are_alleles(infer_gene, true_gene):
regionstr = utils.color('bold', utils.color('blue', region))
truestr = '(originally %s)' % match_name
else:
regionstr = utils.color('bold', utils.color('red', region))
truestr = '(true: %s)' % utils.color_gene(true_gene).replace(region, '')
print ' %s %s %s' % (regionstr, utils.color_gene(infer_gene).replace(region, ''), truestr)
print ' gl', gl_seq
print ' ', qr_seq
# replace the dots (gaps) in the gl match
new_qr_seq, new_gl_seq = [], []
for inuke in range(min(len(qr_seq), len(gl_seq))):
if gl_seq[inuke] == '.':
pass
else:
new_qr_seq.append(qr_seq[inuke]) # this should only be out of range if the v match extends through the whole query sequence, i.e. friggin never
new_gl_seq.append(gl_seq[inuke])
for inuke in range(len(gl_seq), len(qr_seq)):
new_qr_seq.append(qr_seq[inuke])
for inuke in range(len(qr_seq), len(gl_seq)):
new_gl_seq.append(gl_seq[inuke])
qr_seq = ''.join(new_qr_seq)
gl_seq = ''.join(new_gl_seq)
# work out the erosions
qr_ldots = qr_seq.rfind('.') + 1 # first strip off any dots on the left of query seq
qr_seq = qr_seq[qr_ldots : ]
gl_seq = gl_seq[qr_ldots : ]
gl_ldots = gl_seq.rfind('.') + 1 # then remove dots on the left of the germline seq
qr_seq = qr_seq[gl_ldots : ]
gl_seq = gl_seq[gl_ldots : ]
del_5p = qr_ldots + gl_ldots
jf_insertion = ''
if region == 'j':
jf_insertion = qr_seq[len(gl_seq) : ]
qr_seq = qr_seq[ : len(gl_seq)] # then strip the right-hand portion of the query sequence that isn't aligned to the germline
del_3p = len(gl_seq) - len(qr_seq) # then do the same for the germline overhanging on the right of the query
gl_seq = gl_seq[ : len(qr_seq)]
assert len(gl_seq) == len(qr_seq)
new_gl_seq = []
for inuke in range(len(gl_seq)): # replace dashes (matched bases)
assert gl_seq[inuke] != '.' # hoping there's no gaps in here
if gl_seq[inuke] == '-':
new_gl_seq.append(qr_seq[inuke])
else:
new_gl_seq.append(gl_seq[inuke])
gl_seq = ''.join(new_gl_seq)
if self.germline_seqs[region][infer_gene].find(gl_seq) != del_5p: # why the *@*!! can't they make this consistent?
if self.germline_seqs[region][infer_gene].find(gl_seq) < 0:
print 'whooooaa'
print self.germline_seqs[region][infer_gene]
print gl_seq
line['failed'] = True
return line
del_5p += self.germline_seqs[region][infer_gene].find(gl_seq)
try:
assert del_5p + len(gl_seq) + del_3p + len(jf_insertion) == len(self.germline_seqs[region][infer_gene])
except:
print ' ERROR lengths failed for %s' % unique_id
# print del_5p, len(gl_seq), del_3p, del_5p + len(gl_seq) + del_3p , len(self.germline_seqs[region][infer_gene])
# print gl_seq
# print self.germline_seqs[region][infer_gene]
line['failed'] = True
return line
# assert False
if self.args.debug:
utils.color_mutants(gl_seq, qr_seq, ref_label='gl ', extra_str=' ', print_result=True, post_str=' del: %d %d' % (del_5p, del_3p))
# try:
# infer_gene = joinparser.figure_out_which_damn_gene(self.germline_seqs, infer_gene, gl_seq, debug=self.args.debug)
# except:
# print 'ERROR couldn\'t figure out the gene for %s' % infer_gene
# return {}
line[region + '_gene'] = infer_gene
line[region + '_qr_seq'] = qr_seq
line[region + '_gl_seq'] = gl_seq
line[region + '_5p_del'] = del_5p
line[region + '_3p_del'] = del_3p
if region == 'j':
line['jf_insertion'] = jf_insertion
return line
def read_sequence_file(infname, is_data, n_max_queries=-1, args=None, simglfo=None, quiet=False, more_input_info=None):
# NOTE renamed this from get_seqfile_info() since I'm changing the return values, but I don't want to update the calls everywhere (e.g. in compareutils)
yaml_glfo = None
suffix = utils.getsuffix(infname)
if suffix in delimit_info:
seqfile = open(infname) # closes on function exit. no, this isn't the best way to do this
reader = csv.DictReader(seqfile, delimiter=delimit_info[suffix])
elif suffix in ['.fa', '.fasta', '.fq', '.fastq', '.fastx']:
add_info = args is not None and args.name_column is not None and 'fasta-info-index' in args.name_column
reader = utils.read_fastx(infname, name_key='unique_ids', seq_key='input_seqs', add_info=add_info, sanitize_uids=True, n_max_queries=n_max_queries, # NOTE don't use istarstop kw arg here, 'cause it f***s with the istartstop treatment in the loop below
queries=(args.queries if (args is not None and not args.abbreviate) else None)) # NOTE also can't filter on args.queries here if we're also translating
elif suffix == '.yaml':
yaml_glfo, reader, _ = utils.read_yaml_output(infname, n_max_queries=n_max_queries, synth_single_seqs=True, dont_add_implicit_info=True) # not really sure that long term I want to synthesize single seq lines, but for backwards compatibility it's nice a.t.m.
if not is_data:
simglfo = yaml_glfo # doesn't replace the contents, of course, which is why we return it
else:
raise Exception('unhandled file extension \'%s\' on file \'%s\'' % (suffix, infname))
input_info = OrderedDict()
reco_info = None
if not is_data:
reco_info = OrderedDict()
n_duplicate_uids = 0
printed_simu_mismatch_warning = False
n_queries_added = 0
found_seed = False
potential_names, used_names = None, None # for abbreviating
iname = None # line number -- used as sequence id if there isn't a name column in the file
iline = -1
for line in reader:
iline += 1
if args is not None:
if args.istartstop is not None:
if iline < args.istartstop[0]:
continue
if iline >= args.istartstop[1]:
break
if args.name_column is not None:
if 'infostrs' in line and args.name_column.split('-')[:3] == ['fasta', 'info', 'index']:
assert len(args.name_column.split('-')) == 4
line['unique_ids'] = line['infostrs'][int(args.name_column.split('-')[3])]
else:
line['unique_ids'] = line[args.name_column]
del line[args.name_column]
if args.seq_column is not None:
line['input_seqs'] = line[args.seq_column]
if args.seq_column != 'seqs': # stupid god damn weird backwards compatibility edge case bullshit
del line[args.seq_column]
if iname is None and 'unique_ids' not in line and 'unique_id' not in line:
print ' %s: couldn\'t find a name (unique id) column, so using line number as the sequence label (you can set the name column with --name-column)' % (utils.color('yellow', 'warning'))
iname = 0
if iname is not None:
line['unique_ids'] = '%09d' % iname
iname += 1
if 'input_seqs' not in line and 'seq' not in line:
raise Exception('couldn\'t find a sequence column in %s (you can set this with --seq-column)' % infname)
if suffix != '.yaml':
utils.process_input_line(line)
if len(line['unique_ids']) > 1:
raise Exception('can\'t yet handle multi-seq csv input files')
uid = line['unique_ids'][0]
if uid in input_info:
new_uid = uid
iid = 2
while new_uid in input_info:
new_uid = uid + '-' + str(iid)
iid += 1
if n_duplicate_uids == 0:
print ' %s duplicate uid(s) in input file, so renaming by appending integer string, e.g. \'%s\' --> \'%s\'' % (utils.color('yellow', 'warning'), uid, new_uid)
n_duplicate_uids += 1
uid = new_uid # if you decide you want to change it also in <reco_info>, don't forget to also modify the tree (and maybe other stuff, hence why I don't want to do it)
inseq = line['input_seqs'][0]
# # it would be nice to check here for forbidden characters (in addition to in the .fa code above), but it's hard because we won't have read the csv properly above if it has them
# if any(fc in uid for fc in utils.forbidden_characters):
# raise Exception('found a forbidden character (one of %s) in sequence id \'%s\'' % (' '.join(["'" + fc + "'" for fc in utils.forbidden_characters]), uid))
if args is not None:
if args.abbreviate: # note that this changes <uid>, but doesn't modify <line>
uid, potential_names, used_names = utils.choose_new_uid(potential_names, used_names)
if args.queries is not None and uid not in args.queries:
continue
if args.reco_ids is not None and line['reco_id'] not in args.reco_ids:
continue
if args.seed_unique_id is not None and uid == args.seed_unique_id:
found_seed = True
if uid in input_info:
raise Exception('found uid \'%s\' twice in input file %s' % (uid, infname))
if any(c not in utils.alphabet for c in inseq): # NOTE should really be integrated with sanitize_seqs arg in utils.read_fastx()
unexpected_chars = set([ch for ch in inseq if ch not in utils.alphabet])
raise Exception('unexpected character%s %s (not among %s) in input sequence with id %s:\n %s' % (utils.plural(len(unexpected_chars)), ', '.join([('\'%s\'' % ch) for ch in unexpected_chars]), utils.alphabet, uid, inseq))
# da business
input_info[uid] = {'unique_ids' : [uid, ], 'seqs' : [inseq, ]}
if not is_data:
if 'v_gene' not in line:
raise Exception('simulation info not found in %s' % infname)
reco_info[uid] = line # this used to be deepcopy'd, but it's really slow and i'm really pretty sure it's not necessary
if uid != line['unique_ids'][0] and not printed_simu_mismatch_warning:
print ' note: uid in simulation info %s doesn\'t match input file uid %s (latter was probably changed above). Simulation info will be internally consistent, but the key indexing that info in <reco_info> will be different, since it corresponds to the newly chosen uid above.' % (uid, line['unique_ids'][0])
printed_simu_mismatch_warning = True
if simglfo is not None:
utils.add_implicit_info(simglfo, reco_info[uid])
for line_key in utils.input_metafile_keys.values():
if line_key in reco_info[uid]: # this is kind of weird to copy from sim info to input info, but it makes sense because affinity is really meta info (the only other place affinity could come from is --input-metafname below). Where i'm defining meta info more or less as any input info besides name and sequence (i think the distinction is only really important because we want to support fastas, which can't [shouldn't!] handle anything else))
input_info[uid][line_key] = copy.deepcopy(reco_info[uid][line_key]) # note that the args.input_metafname stuff below should print a warning if you've also specified that (which you shouldn't, if it's simulation)
n_queries_added += 1
if n_max_queries > 0 and n_queries_added >= n_max_queries:
if not quiet: # just adding <quiet>, and too lazy to decide what other print statements it should effect, this is the only one I care about right now
print ' --n-max-queries: stopped after reading %d queries from input file' % len(input_info)
break
if n_duplicate_uids > 0:
print ' %s renamed %d duplicate uids from %s' % (utils.color('yellow', 'warning'), n_duplicate_uids, infname)
if more_input_info is not None: # if you use this on simulation, the extra queries that aren't in <reco_info> may end up breaking something down the line (but I don't imagine this really getting used on simulation)
if len(set(more_input_info) & set(input_info)) > 0: # check for sequences in both places
common_uids = set(more_input_info) & set(input_info)
print ' note: found %d queries in both --infname and --queries-to-include-fname: %s' % (len(common_uids), ' '.join(common_uids)) # not necessarily a problem, but you probably *shouldn't* have sequences floating around in two different files
differing_seqs = [q for q in common_uids if more_input_info[q]['seqs'][0] != input_info[q]['seqs'][0]]
if len(differing_seqs) > 0: # if they have different sequences, though, that's a problem
for q in differing_seqs:
print q
utils.color_mutants(input_info[q]['seqs'][0], more_input_info[q]['seqs'][0], align_if_necessary=True, print_result=True, ref_label=' --infname ', seq_label=' --queries-to-include-fname ')
raise Exception('inconsistent sequences for %d of the queries in both --infname and --queries-to-include-fname (see preceding lines)' % len(differing_seqs))
if args is not None and args.seed_unique_id is not None and args.seed_unique_id in more_input_info:
found_seed = True
input_info.update(more_input_info)
if args is not None and args.input_metafname is not None:
read_input_metafo(args.input_metafname, input_info.values(), debug=True)
post_process(input_info, reco_info, args, infname, found_seed, is_data, iline)
if len(input_info) == 0:
raise Exception('didn\'t read any sequences from %s' % infname)
return input_info, reco_info, yaml_glfo
def parse_query_text(self, unique_id, query_info):
if len(query_info
) == 0: # one for the query sequence, then one for v, d, and j
print 'no info for', unique_id
return {}
elif len(query_info) < 4:
regions_ok = ''
for info in query_info:
for region in utils.regions:
if 'IGH' + region.upper() in info:
regions_ok += region
for region in utils.regions:
if region not in regions_ok:
print ' ERROR no %s matches' % region
return {}
assert False # shouldn't get here
elif len(query_info) != 4:
print 'info for', unique_id, 'all messed up'
for info in query_info:
print info
sys.exit()
full_qr_seq = query_info[0].replace('>', '').replace(
unique_id, '') # strip off the unique id
full_qr_seq = ''.join(full_qr_seq.split()).upper(
) # strip off white space and uppercase it
assert full_qr_seq == self.seqinfo[unique_id]['seq']
line = {}
line['unique_id'] = unique_id
line['seq'] = full_qr_seq
for ireg in range(len(utils.regions)):
region = utils.regions[ireg]
info = query_info[ireg + 1].splitlines()
while unique_id not in info[
0]: # remove the line marking cdr3 and framework regions
info.pop(0)
if len(info) <= 1:
print info
assert len(info) > 1
assert len(info[0].split()) == 2
qr_seq = info[0].split()[1].upper(
) # this line should be '<unique_id> .............<query_seq>'
true_gene = self.seqinfo[unique_id][region + '_gene']
imatch = 1 # which match to take
match_name = str(info[imatch].split()[2])
while match_name in just_always_friggin_skip and len(
info) > imatch + 1 and len(info[imatch + 1].split()) > 2:
imatch += 1
old_one = match_name
match_name = str(info[imatch].split()[2])
if self.args.debug:
print ' %s: taking next match: %s --> %s)' % (
unique_id, utils.color_gene(old_one),
utils.color_gene(match_name))
infer_gene = match_name
for gset in equivalent_genes:
if match_name in gset and true_gene in gset and match_name != true_gene: # if the true gene and the inferred gene are in the same equivalence set, treat it as correct, i.e. just pretend it inferred the right name
if self.args.debug:
print ' %s: replacing name %s with true name %s' % (
unique_id, match_name, true_gene)
infer_gene = true_gene
# ----------------------------------------------------------------------------------------
# skipping bullshit
def skip_gene(gene):
print ' %s in list of genes to skip' % utils.color_gene(
gene)
if gene not in genes_actually_skipped:
genes_actually_skipped[gene] = 0
genes_actually_skipped[gene] += 1
line['skip_gene'] = True
if infer_gene not in self.germline_seqs[region]:
print ' couldn\'t find %s in germlines (skipping)' % infer_gene
skip_gene(infer_gene)
return line
if infer_gene in just_always_friggin_skip:
skip_gene(infer_gene)
return line
if true_gene in just_always_friggin_skip:
skip_gene(true)
return line
if not self.args.dont_skip_or15_genes and '/OR1' in true_gene:
skip_gene(true_gene)
return line
if self.args.skip_missing_genes:
if infer_gene in genes_to_skip:
skip_gene(infer_gene)
return line
if true_gene in genes_to_skip:
skip_gene(true_gene)
return line
gl_seq = info[imatch].split()[4].upper()
if qr_seq.replace('.', '') not in self.seqinfo[unique_id]['seq']:
# if self.args.debug:
print ' qr_seq not found in seqinfo'
line['failed'] = True
return line
if self.args.debug:
if utils.are_alleles(infer_gene, true_gene):
regionstr = utils.color('bold',
utils.color('blue', region))
truestr = '(originally %s)' % match_name
else:
regionstr = utils.color('bold', utils.color('red', region))
truestr = '(true: %s)' % utils.color_gene(
true_gene).replace(region, '')
print ' %s %s %s' % (regionstr,
utils.color_gene(infer_gene).replace(
region, ''), truestr)
print ' gl', gl_seq
print ' ', qr_seq
# replace the dots (gaps) in the gl match
new_qr_seq, new_gl_seq = [], []
for inuke in range(min(len(qr_seq), len(gl_seq))):
if gl_seq[inuke] == '.':
pass
else:
new_qr_seq.append(
qr_seq[inuke]
) # this should only be out of range if the v match extends through the whole query sequence, i.e. friggin never
new_gl_seq.append(gl_seq[inuke])
for inuke in range(len(gl_seq), len(qr_seq)):
new_qr_seq.append(qr_seq[inuke])
for inuke in range(len(qr_seq), len(gl_seq)):
new_gl_seq.append(gl_seq[inuke])
qr_seq = ''.join(new_qr_seq)
gl_seq = ''.join(new_gl_seq)
# work out the erosions
qr_ldots = qr_seq.rfind(
'.') + 1 # first strip off any dots on the left of query seq
qr_seq = qr_seq[qr_ldots:]
gl_seq = gl_seq[qr_ldots:]
gl_ldots = gl_seq.rfind(
'.') + 1 # then remove dots on the left of the germline seq
qr_seq = qr_seq[gl_ldots:]
gl_seq = gl_seq[gl_ldots:]
del_5p = qr_ldots + gl_ldots
jf_insertion = ''
if region == 'j':
jf_insertion = qr_seq[len(gl_seq):]
qr_seq = qr_seq[:len(
gl_seq
)] # then strip the right-hand portion of the query sequence that isn't aligned to the germline
del_3p = len(gl_seq) - len(
qr_seq
) # then do the same for the germline overhanging on the right of the query
gl_seq = gl_seq[:len(qr_seq)]
assert len(gl_seq) == len(qr_seq)
new_gl_seq = []
for inuke in range(len(gl_seq)): # replace dashes (matched bases)
assert gl_seq[inuke] != '.' # hoping there's no gaps in here
if gl_seq[inuke] == '-':
new_gl_seq.append(qr_seq[inuke])
else:
new_gl_seq.append(gl_seq[inuke])
gl_seq = ''.join(new_gl_seq)
if self.germline_seqs[region][infer_gene].find(
gl_seq
) != del_5p: # why the *@*!! can't they make this consistent?
if self.germline_seqs[region][infer_gene].find(gl_seq) < 0:
print 'whooooaa'
print self.germline_seqs[region][infer_gene]
print gl_seq
line['failed'] = True
return line
del_5p += self.germline_seqs[region][infer_gene].find(gl_seq)
try:
assert del_5p + len(gl_seq) + del_3p + len(
jf_insertion) == len(
self.germline_seqs[region][infer_gene])
except:
print ' ERROR lengths failed for %s' % unique_id
# print del_5p, len(gl_seq), del_3p, del_5p + len(gl_seq) + del_3p , len(self.germline_seqs[region][infer_gene])
# print gl_seq
# print self.germline_seqs[region][infer_gene]
line['failed'] = True
return line
# assert False
if self.args.debug:
utils.color_mutants(gl_seq,
qr_seq,
ref_label='gl ',
extra_str=' ',
print_result=True,
post_str=' del: %d %d' %
(del_5p, del_3p))
# try:
# infer_gene = joinparser.figure_out_which_damn_gene(self.germline_seqs, infer_gene, gl_seq, debug=self.args.debug)
# except:
# print 'ERROR couldn\'t figure out the gene for %s' % infer_gene
# return {}
line[region + '_gene'] = infer_gene
line[region + '_qr_seq'] = qr_seq
line[region + '_gl_seq'] = gl_seq
line[region + '_5p_del'] = del_5p
line[region + '_3p_del'] = del_3p
if region == 'j':
line['jf_insertion'] = jf_insertion
return line
# print '%s %3d' % (utils.color_gene(g, width=20), len(s) - glfo['cyst-positions'][g])
# sys.exit()
# base = '4-59'
# a1, a2 = '12', '01'
# gene1, gene2 = 'IGHV' + base + '*' + a1, 'IGHV' + base + '*' + a2
genes = ['IG' + args.chain.upper() + args.region.upper() + args.base + '*' + al for al in args.alleles]
if args.other_genes is not None:
genes += args.other_genes
codon_positions = glfo[utils.conserved_codons[args.chain][args.region] + '-positions'] if args.region != 'd' else None
def print_str(gene, seq):
return '%s %s' % (utils.color_gene(gene, width=15), seq)
ref_gene = genes[0]
ref_seq = glfo['seqs'][args.region][ref_gene]
print print_str(ref_gene, utils.color_mutants(ref_seq, ref_seq, emphasis_positions=None if args.region == 'd' else [codon_positions[ref_gene] + i for i in range(3)])), ' (reference)'
for igene in range(1, len(genes)):
gene = genes[igene]
seq = glfo['seqs'][args.region][gene]
min_length = min(len(seq), len(ref_seq))
colored_seq = utils.color_mutants(ref_seq[:min_length], seq[:min_length], print_isnps=True, emphasis_positions=None if args.region == 'd' else [codon_positions[gene] + i for i in range(3)])
print print_str(gene, colored_seq)
if min_length < len(ref_seq) and igene == 0:
print 'extra for %s: %s' % (utils.color_gene(ref_gene), ref_seq[min_length:])
if min_length < len(seq):
print 'extra for %s: %s' % (utils.color_gene(gene), seq[min_length:])
sorted_clusters = sorted(
[c for c in partitions[if2] if keyfcn(c) is not None],
key=keyfcn
) # make a list of the clusters in the other partition that's sorted by how similar their naive sequence are
nearest_cluster_lists[label1][label2].append(sorted_clusters)
extra_str = ''
inner_loop_str = ''
if len(sorted_clusters) == 0:
# extra_str = utils.color('yellow', '-', width=3)
inner_loop_str = utils.color('yellow', '- -', width=8)
size_index_str = '%s %3d' % (utils.color(
'blue',
'%4d' % len(cluster1)), partitions[if1].index(cluster1))
print ' %-3s%s %8s %-30s%3s' % (
extra_str, size_index_str, inner_loop_str,
cdr3_translation(info1), extra_str)
for nclust in sorted_clusters:
nclust_naive_cdr3 = cdr3_translation(
annotations[if2][getkey(nclust)])
hdist = naive_hdist_or_none(info1,
annotations[if2][getkey(nclust)])
print ' %s %4d %2s %-30s' % (
utils.color('blue', '%4d' % len(nclust)),
partitions[if2].index(nclust),
'%d' % hdist if hdist > 0 else '',
utils.color_mutants(cdr3_translation(info1),
nclust_naive_cdr3,
amino_acid=True))
def parse_query_text(self, unique_id, query_info):
if len(query_info) == 0: # one for the query sequence, then one for v, d, and j
print 'no info for',unique_id
return {}
elif len(query_info) < 4:
regions_ok = ''
for info in query_info:
for region in utils.regions:
if 'IGH' + region.upper() in info:
regions_ok += region
for region in utils.regions:
if region not in regions_ok:
print ' ERROR no %s matches' % region
return {}
assert False # shouldn't get here
elif len(query_info) != 4:
print 'info for', unique_id, 'all messed up'
for info in query_info:
print info
sys.exit()
full_qr_seq = query_info[0].replace('>', '').replace(unique_id, '') # strip off the unique id
full_qr_seq = ''.join(full_qr_seq.split()).upper() # strip off white space and uppercase it
assert full_qr_seq == self.seqinfo[unique_id]['seq']
line = {}
line['unique_id'] = unique_id
line['seq'] = full_qr_seq
for ireg in range(len(utils.regions)):
region = utils.regions[ireg]
info = query_info[ireg + 1].splitlines()
while unique_id not in info[0]: # remove the line marking cdr3 and framework regions
info.pop(0)
if len(info) <= 1:
print info
assert len(info) > 1
assert len(info[0].split()) == 2
qr_seq = info[0].split()[1].upper() # this line should be '<unique_id> .............<query_seq>'
imatch = 1 # which match to take
match_name = str(info[imatch].split()[2])
# if 'IGHV3-69' in match_name: # it's not right anyway
# line['failed'] = True
# return line
# while unacceptable_match(match_name, self.germline_seqs):
# imatch += 1
# match_name = str(info[imatch].split()[2])
# print ' new match name: %s' % match_name
gl_seq = info[imatch].split()[4].upper()
if qr_seq.replace('.', '') not in self.seqinfo[unique_id]['seq']:
if self.args.debug:
print ' qr_seq not foundin seqinfo'
line['failed'] = True
return line
if self.args.debug:
print ' ', region, match_name
print ' gl', gl_seq
print ' ', qr_seq
# replace the dots (gaps) in the gl match
new_qr_seq, new_gl_seq = [], []
for inuke in range(min(len(qr_seq), len(gl_seq))):
if gl_seq[inuke] == '.':
pass
else:
new_qr_seq.append(qr_seq[inuke]) # this should only be out of range if the v match extends through the whole query sequence, i.e. friggin never
new_gl_seq.append(gl_seq[inuke])
for inuke in range(len(gl_seq), len(qr_seq)):
new_qr_seq.append(qr_seq[inuke])
for inuke in range(len(qr_seq), len(gl_seq)):
new_gl_seq.append(gl_seq[inuke])
qr_seq = ''.join(new_qr_seq)
gl_seq = ''.join(new_gl_seq)
# work out the erosions
qr_ldots = qr_seq.rfind('.') + 1 # first strip off any dots on the left of query seq
qr_seq = qr_seq[qr_ldots : ]
gl_seq = gl_seq[qr_ldots : ]
gl_ldots = gl_seq.rfind('.') + 1 # then remove dots on the left of the germline seq
qr_seq = qr_seq[gl_ldots : ]
gl_seq = gl_seq[gl_ldots : ]
del_5p = qr_ldots + gl_ldots
jf_insertion = ''
if region == 'j':
jf_insertion = qr_seq[len(gl_seq) : ]
qr_seq = qr_seq[ : len(gl_seq)] # then strip the right-hand portion of the query sequence that isn't aligned to the germline
del_3p = len(gl_seq) - len(qr_seq) # then do the same for the germline overhanging on the right of the query
gl_seq = gl_seq[ : len(qr_seq)]
assert len(gl_seq) == len(qr_seq)
new_gl_seq = []
for inuke in range(len(gl_seq)): # replace dashes (matched bases)
assert gl_seq[inuke] != '.' # hoping there's no gaps in here
if gl_seq[inuke] == '-':
new_gl_seq.append(qr_seq[inuke])
else:
new_gl_seq.append(gl_seq[inuke])
gl_seq = ''.join(new_gl_seq)
if match_name not in self.germline_seqs[region]:
print ' ERROR couldn\'t find %s in germlines' % match_name
line['failed'] = True
return line
if self.germline_seqs[region][match_name].find(gl_seq) != del_5p: # why the *@*!! can't they make this consistent?
if self.germline_seqs[region][match_name].find(gl_seq) < 0:
print 'whooooaa'
print self.germline_seqs[region][match_name]
print gl_seq
sys.exit()
del_5p += self.germline_seqs[region][match_name].find(gl_seq)
try:
assert del_5p + len(gl_seq) + del_3p + len(jf_insertion) == len(self.germline_seqs[region][match_name])
except:
print ' ERROR lengths failed for %s' % unique_id
# print del_5p, len(gl_seq), del_3p, del_5p + len(gl_seq) + del_3p , len(self.germline_seqs[region][match_name])
# print gl_seq
# print self.germline_seqs[region][match_name]
line['failed'] = True
return line
# assert False
if self.args.debug:
utils.color_mutants(gl_seq, qr_seq, ref_label='gl ', extra_str=' ', print_result=True, post_str=' del: %d %d' % (del_5p, del_3p))
# try:
# match_name = joinparser.figure_out_which_damn_gene(self.germline_seqs, match_name, gl_seq, debug=self.args.debug)
# except:
# print 'ERROR couldn\'t figure out the gene for %s' % match_name
# return {}
line[region + '_gene'] = match_name
line[region + '_qr_seq'] = qr_seq
line[region + '_gl_seq'] = gl_seq
line[region + '_5p_del'] = del_5p
line[region + '_3p_del'] = del_3p
if region == 'j':
line['jf_insertion'] = jf_insertion
return line
def reconstruct_indelfo_from_indel_list(indel_list,
line,
iseq,
debug=False): # old-style files
if 'reversed_seq' in indel_list: # handle super-old files
print '%s encountered file with super old, unhandled indel format, proceeding, but indel info may be inconsistent' % (
utils.color('red', 'error'))
return
line['indelfos'][iseq] = get_empty_indel()
if len(indel_list) == 0:
return
ifo_positions = [ifo['pos'] for ifo in indel_list]
if len(ifo_positions) != len(set(ifo_positions)):
print '%s two indels at the same position, everything will be kinda messed up' % utils.color(
'red', 'error')
ifos_by_pos = {ifo['pos']: ifo for ifo in indel_list}
qr_gap_seq, gl_gap_seq = [], []
iqr, igl, iindel = 0, 0, 0
if debug:
print len(line['input_seqs'][iseq]), line['input_seqs'][iseq]
print len(line['naive_seq']), line['naive_seq']
while iqr < len(line['input_seqs'][iseq]):
if debug:
print ' %3d %3d' % (iqr, igl),
if igl >= len(
line['naive_seq']
): # if the pos is longer than the qr seq, we won't fall off the end (so i'm ignoring that case here), but we presumably will miss an indel and crash somewhere else. Note that I can't just check before the loop, since indel positions can be longer than the initial sequence lengths (i.e. before adding other indels)
offending_indels = [
ifo for p, ifo in ifos_by_pos.items()
if p >= len(line['naive_seq'])
]
print '%s %s indel position beyond end of sequence len %d (setting to invalid): %s' % (
utils.color('red', 'error'), ':'.join(line['unique_ids']),
len(line['naive_seq']), offending_indels)
raise IndelfoReconstructionError(
) # no, I don't like doing it this way, I don't like using exceptions for control flow. But, this only happens when we read in an old file with ridiculous inconsistent info (so I can't fix the underlying wrong info), and there's no way to know it's inconsistent until we get to here (so I can't just throw it away earlier)
if iindel in ifos_by_pos:
ifo = ifos_by_pos[iindel]
if ifo['type'] == 'insertion':
if ifo['seqstr'] != line['input_seqs'][iseq][iqr:iqr +
ifo['len']]:
print '%s indel info seqstr doesn\'t match input seq str:' % utils.color(
'red', 'error')
utils.color_mutants(ifo['seqstr'],
line['input_seqs'][iseq][iqr:iqr +
ifo['len']],
align=True,
print_result=True,
extra_str=' ')
qr_gap_seq += ifo['seqstr'].split()
gl_gap_seq += [ifo['len'] * utils.gap_chars[0]]
if debug:
print ' %s %s' % (ifo['seqstr'].split(),
[ifo['len'] * utils.gap_chars[0]])
iqr += ifo['len']
else:
if ifo['seqstr'] != line['naive_seq'][igl:igl + ifo['len']]:
print '%s indel info seqstr doesn\'t match naive seq str:' % utils.color(
'red', 'error')
utils.color_mutants(ifo['seqstr'],
line['naive_seq'][igl:igl +
ifo['len']],
align=True,
print_result=True,
extra_str=' ')
qr_gap_seq += [ifo['len'] * utils.gap_chars[0]]
gl_gap_seq += ifo['seqstr'].split()
if debug:
print ' %s %s' % ([ifo['len'] * utils.gap_chars[0]
], ifo['seqstr'].split())
igl += ifo['len']
del ifos_by_pos[iindel]
iindel += ifo['len']
else:
qr_gap_seq += [line['input_seqs'][iseq][iqr]]
gl_gap_seq += [line['naive_seq'][igl]]
if debug:
print ' %s %s' % (line['input_seqs'][iseq][iqr],
line['naive_seq'][igl])
iqr += 1
igl += 1
iindel += 1
line['indelfos'][iseq]['qr_gap_seq'] = ''.join(qr_gap_seq)
line['indelfos'][iseq]['gl_gap_seq'] = ''.join(gl_gap_seq)
line['indelfos'][iseq]['indels'] = indel_list
line['indelfos'][iseq]['reversed_seq'] = line['indel_reversed_seqs'][iseq]
line['indelfos'][iseq]['genes'] = {
r: line[r + '_gene']
for r in utils.regions
}
if debug:
print ' reconstructed indelfo'
print get_dbg_str(line['indelfos'][iseq])
def hamming_distance_to_true_naive(self, true_line, line, query_name, restrict_to_region='', normalize=False, padfo=None, debug=False):
"""
Hamming distance between the inferred naive sequence and the tue naive sequence.
<restrict_to_region> if set, restrict the comparison to the section of the *true* sequence assigned to the given region.
NOTE this will not in general correspond to the similarly-assigned region in the inferred naive sequence.
if <normalize> divide by sequence length
"""
true_naive_seq = true_line['naive_seq']
inferred_naive_seq = line['naive_seq']
if len(true_naive_seq) != len(inferred_naive_seq):
print '%20s true inf' % ''
for k in true_line:
print '%20s %s' % (k, true_line[k]),
if k in line:
print ' %s' % line[k]
else:
print ' NOPE'
for k in line:
if k not in true_line:
print ' not in true line %20s %s' % (k, line[k])
raise Exception('%s true and inferred sequences not the same length\n %s\n %s\n' % (line['unique_id'], true_naive_seq, inferred_naive_seq))
# assert False # read through this whole damn thing and make sure it's ok
left_hack_add_on = ''
right_hack_add_on = ''
# if len(true_line['seq']) > len(utils.remove_ambiguous_ends(line['seq'], line['fv_insertion'], line['jf_insertion'])): # ihhhmmm doesn't report the bits of the sequence it erodes off the ends, so we have to add them back on
# # if len(true_naive_seq) > len(inferred_naive_seq): # hm, now why did I use line['seq'] stuff before?
# assert False
# start = true_line['seq'].find(line['seq'])
# assert start >= 0
# end = len(line['seq']) + start
# left_hack_add_on = true_line['seq'][: start]
# right_hack_add_on = true_line['seq'][ end :]
# # extra_penalty = len(left_hack_add_on) + len(right_hack_add_on)
# inferred_naive_seq = 'N'*len(left_hack_add_on) + inferred_naive_seq + 'N'*len(right_hack_add_on)
# if debug:
# print ' adding to inferred naive seq'
if padfo is not None: # remove N padding from the inferred sequence
if debug:
print 'removing padfo'
print inferred_naive_seq
if inferred_naive_seq[padfo['padleft'] : ].count('N') == padfo['padleft']: # this fails to happen if reset_effective_erosions_and_effective_insertions already removed the Ns
inferred_naive_seq = inferred_naive_seq[padfo['padleft'] : ]
elif debug: # NOTE if no debug, we just fall through, which isok
print 'tried to remove non Ns!\n %s\n padleft %d\n' % (inferred_naive_seq, padfo['padleft'])
if padfo['padright'] > 0:
if inferred_naive_seq[ : padfo['padright']].count('N') == padfo['padright']: # this fails to happen if reset_effective_erosions_and_effective_insertions already removed the Ns
inferred_naive_seq = inferred_naive_seq[ : -padfo['padright']]
elif debug: # NOTE if no debug, we just fall through, which isok
print 'tried to remove non Ns!\n %s\n padright %d\n' % (inferred_naive_seq, padfo['padright'])
if debug:
print padfo['padleft'] * ' ' + inferred_naive_seq + padfo['padleft'] * ' '
bounds = None
if restrict_to_region != '':
bounds = true_line['regional_bounds'][restrict_to_region]
if debug:
print 'restrict to %s' % restrict_to_region
utils.color_mutants(true_naive_seq, inferred_naive_seq, print_result=True, extra_str=' ')
utils.color_mutants(true_naive_seq[bounds[0] : bounds[1]], inferred_naive_seq[bounds[0] : bounds[1]], print_result=True, extra_str=' ' + bounds[0]*' ')
true_naive_seq = true_naive_seq[bounds[0] : bounds[1]]
inferred_naive_seq = inferred_naive_seq[bounds[0] : bounds[1]]
if len(true_naive_seq) != len(inferred_naive_seq):
raise Exception('still not the same lengths for %s\n %s\n %s' % (query_name, true_naive_seq, inferred_naive_seq))
fraction, len_excluding_ambig = utils.hamming_fraction(true_naive_seq, inferred_naive_seq, return_len_excluding_ambig=True)
total_distance = int(fraction * len_excluding_ambig)
if len(true_naive_seq) == 0:
print 'WARNING zero length sequence in hamming_distance_to_true_naive'
return 0
if normalize:
return int(100 * (float(total_distance) / len(true_naive_seq)))
else:
return total_distance
codon_positions = glfo[utils.conserved_codons[args.locus][args.region] +
'-positions'] if args.region != 'd' else None
def print_str(gene, seq):
return '%s %s' % (utils.color_gene(gene, width=15), seq)
ref_gene = genes[0]
ref_seq = glfo['seqs'][args.region][ref_gene]
print print_str(
ref_gene,
utils.color_mutants(ref_seq,
ref_seq,
emphasis_positions=None if args.region == 'd' else
[codon_positions[ref_gene] + i
for i in range(3)])), ' (reference)'
for igene in range(1, len(genes)):
gene = genes[igene]
seq = glfo['seqs'][args.region][gene]
min_length = min(len(seq), len(ref_seq))
colored_seq = utils.color_mutants(
ref_seq[:min_length],
seq[:min_length],
print_isnps=True,
emphasis_positions=None if args.region == 'd' else
[codon_positions[gene] + i for i in range(3)])
print print_str(gene, colored_seq)
if min_length < len(ref_seq) and igene == 0:
