def cluster(blastdb, taxdb):
conn = sqlite3.connect(blastdb)
c = conn.cursor()
c.execute("ATTACH '" + taxdb + "' as 'tax'")
muscle_cline = MuscleCommandline(clwstrict=True)
genes = []
for iter in c.execute(
"SELECT Gene_name from blast WHERE decision = 'To cluster analysis/Chosen' GROUP BY Gene_name"
):
genes.append(iter[0])
for gene in genes:
input_dic = {}
multiple_dic = {}
two_dic = {}
problem_dic = {}
finalseqs = set()
multfinalseqs = []
unresolved = []
for iter in c.execute(
"SELECT tc_id, GI from blast where decision = 'To cluster analysis/Chosen' AND Gene_name = '"
+ gene + "'order by tc_id"):
if iter[0] not in input_dic:
input_dic[iter[0]] = [str(iter[1])]
else:
output = input_dic[iter[0]]
output.append(str(iter[1]))
input_dic[iter[0]] = output
for i in input_dic:
GIs_list = input_dic[i]
if len(GIs_list) > 2:
multiple_dic[i] = GIs_list
if len(GIs_list) == 2:
two_dic[i] = GIs_list
for i in multiple_dic:
identities = []
#get consensus of all
align = alignment_reg(multiple_dic[i], blastdb, False, gene, c)
summary_align = AlignInfo.SummaryInfo(align)
consensus = summary_align.gap_consensus(threshold=.5,
ambiguous='N')
consensus_record = SeqRecord(consensus, id="Consensus_all")
for m in multiple_dic[i]:
#align each gene to consensus
seqs = []
iterator = get_seqs_from_sqldb_GI_no_gene([m], "hseq", blastdb,
c)
for seq in iterator:
seqs.append(seq)
handle_string = StringIO()
SeqIO.write(seqs, handle_string, "fasta")
SeqIO.write(consensus_record, handle_string, "fasta")
data = handle_string.getvalue()
stdout, stderr = muscle_cline(stdin=data)
align = AlignIO.read(StringIO(stdout), "clustal")
count = 0
gaps = 0
for col in range(0, len(align[0])):
column = align[:, col]
if "-" not in column:
if column[1:] == column[:-1]:
count = count + 1
else:
gaps = gaps + 1
iden = 100 * (count / float((len(align[0]) - gaps)))
identities.append(iden)
if identities.count(max(identities)) == 1:
GI_to_pick = multiple_dic[i][identities.index(max(identities))]
c.execute(
"UPDATE blast SET Decision='Closest to consensus in cluster analysis/Chosen' WHERE GI = '"
+ GI_to_pick + "';")
GIS_not_picked = list(set(multiple_dic[i]) - set([GI_to_pick]))
GIS_not_picked_str = str(GIS_not_picked).replace("[",
"(").replace(
"]", ")")
c.execute(
"UPDATE blast SET Decision='Further from consensus in cluster analysis/Not chosen' WHERE GI IN "
+ GIS_not_picked_str + ";")
else:
GI_to_pick = [
multiple_dic[i][m] for m, x in enumerate(identities)
if x == max(identities)
]
GI_to_pick_str = str(GI_to_pick).replace("[", "(").replace(
"]", ")")
c.execute(
"UPDATE blast SET Decision='Pick one randomly/Chosen' WHERE GI IN "
+ GI_to_pick_str + ";")
GIS_not_picked = list(set(multiple_dic[i]) - set(GI_to_pick))
GIS_not_picked_str = str(GIS_not_picked).replace("[",
"(").replace(
"]", ")")
c.execute(
"UPDATE blast SET Decision='Pick one randomly/Not chosen' WHERE GI IN "
+ GIS_not_picked_str + ";")
for i in two_dic:
twodic_str = str(two_dic[i]).replace("[", "(").replace("]", ")")
c.execute(
"UPDATE blast SET Decision='Pick one randomly/Chosen' WHERE GI IN "
+ twodic_str + ";")
conn.commit()
conn.close()
def test_resolved_seqs(infile, blastdb, taxdb):
import sqlite3, sys, time
from Bio.Blast import NCBIWWW, NCBIXML
from blastlib.clean_seq_funcs import resolve_seqs, alignment_comp, alignment_reg, alignment_rev_comp, blast, blast_all, identity_calc, tiling
conn = sqlite3.connect(blastdb)
c = conn.cursor()
c.execute("ATTACH '" + taxdb + "' as 'tax'")
error_dic = {}
blast_dic_nums = {}
blast_dic_tcids = {}
seqs_to_blast = []
finalseqs = set()
multseqs = []
#do self blast and print to file
with open(infile) as fasta_file:
print("Blasting " + infile)
records = fasta_file.read()
numqueries = records.count('>')
error = True
while error == True:
try:
result_handle = NCBIWWW.qblast(
"blastn",
"nt",
records,
entrez_query=
'((Papilionoidea[Organism]) OR Hedylidae[Organism]) OR Hesperiidae[Organism]',
word_size=28,
hitlist_size=100)
error = False
except:
error = True
#get rid of this extra step of printing xml using NCBIXML
with open(infile + ".xml", "w") as save_file:
save_file.write(result_handle.read())
result_handle.close()
#open self blast file for parsing
#make dictionary of query species: query GI of those that don't have the top hit as the same species
with open(infile + ".xml") as p:
print("Parsing blast output")
blast_recs = NCBIXML.parse(p)
count = 0
for rec in blast_recs:
count += 1
print(str(round(float(count) / float(numqueries) * 100, 2)) + "%")
#figure out a new way to do this
queryAcc = str(rec.query.split()[0])
for iter in c.execute("SELECT GI FROM blast WHERE accession='" +
queryAcc + "'"):
queryGI = (str(iter[0]))
hitdic = {}
hitSp = set()
for alignment in rec.alignments:
for hsp in alignment.hsps:
identity = float(hsp.identities) / float(hsp.align_length)
if alignment.title.split("|")[1] == queryGI:
pass
else:
hitdic[str(alignment.title.split("|")[1])] = identity
maxiden = max(hitdic.values())
hitGIs = [GI for GI, iden in hitdic.iteritems() if iden == maxiden]
for iter in c.execute("SELECT tc_id FROM blast WHERE GI='" +
queryGI + "'"):
querySp = (str(iter[0]))
for i in hitGIs:
for iter in c.execute("SELECT tc_id FROM blast WHERE GI='" +
i + "'"):
hitSp.add(str(iter[0]))
#only look at top 5 if it doesnt hit the top hit - faster
if querySp not in hitSp:
hitSp = set()
if len(hitdic.values()) > 5:
maxiden = sorted(hitdic.values(), reverse=True)[0:5]
else:
maxiden = hitdic.values()
hitGIs = [
GI for GI, iden in hitdic.iteritems() if iden in maxiden
]
for i in hitGIs:
for iter in c.execute(
"SELECT tc_id FROM blast WHERE GI='" + i + "'"):
hitSp.add(str(iter[0]))
if querySp not in hitSp:
error_dic[querySp] = queryGI
else:
finalseqs.add(queryGI)
else:
finalseqs.add(queryGI)
count = 0
#error_dic['21204'] = '316994286'
##go through error dictionary and align the 'same' gene/species to see if they're weird looking
newseqs = set()
print("Checking nonmatching sequences")
for tc_id in error_dic:
count += 1
print(str(round(float(count) / float(len(error_dic)) * 100, 2)) + "%")
list_of_GIs = []
for iter in c.execute("SELECT Gene_name FROM blast WHERE GI ='" +
error_dic[tc_id] + "'"):
gene_name = str(iter[0])
for iter in c.execute("SELECT GI FROM blast WHERE tc_id = '" + tc_id +
"' and Gene_name = '" + gene_name + "'"):
list_of_GIs.append(str(iter[0]))
first_GI = list_of_GIs[0]
other_GIs = list_of_GIs[1:]
pair_check = []
#align each seq to the first one - first one acts as the 'default' direction
for x, GI in enumerate(other_GIs):
GI_pair = [first_GI, GI]
alignment = alignment_reg(GI_pair)
iden = identity_calc(alignment)
if iden < 90:
# print("Low Aligned Identity: " + str(iden))
alignment = alignment_rev_comp(GI_pair)
iden = identity_calc(alignment)
if iden < 90:
# print("Low Reverse Complement Aligned Identity: " + str(iden))
alignment = alignment_comp(GI_pair)
iden = identity_calc(alignment)
if iden < 90:
# print("Low Complement Aligned Identity: " + str(iden))
pair_check.append(0)
else:
pair_check.append(1)
# print("Complement iden: " + str(iden) + " so pair is fine")
else:
pair_check.append(1)
# print("Reverse Complement iden: " + str(iden) + " so pair is fine")
else:
pair_check.append(1)
# print("High Aligned Identity: " + str(iden) + " so pair is fine")
# print(pair_check)
if all(i == 1 for i in pair_check):
finalseqs.add(error_dic[tc_id])
newseqs.add(error_dic[tc_id])
else:
idens, start_stop = tiling(list_of_GIs, gene_name)
current_start = -1
current_stop = -1
result = []
if all(i > 70 for i in idens):
for start, stop in sorted(start_stop):
if start > current_stop:
result.append((start, stop))
current_start, current_stop = start, stop
else:
current_stop = max(current_stop, stop)
result[-1] = (current_start, current_stop)
if len(result) == len(start_stop):
# print("Seqs align to different regions of probe, choosing all")
for x in list_of_GIs:
finalseqs.add(x)
newseqs.add(x)
else:
# print('Seqs overlap: Printing to file for hand checking')
with open('these_seqs_overlap.txt', 'a') as a:
a.write(str(list_of_GIs) + '\n')
else:
# print('Somethings up with a sequence - printing to check - will blast')
pair_check.append(0)
blast_dic_nums[list_of_GIs[0]] = len(list_of_GIs)
blast_dic_tcids[list_of_GIs[0]] = tc_id
seqs_to_blast.append(list_of_GIs)
with open('seqs_to_be_blasted.txt', 'a') as a:
a.write(str(list_of_GIs) + '\n')
if len(seqs_to_blast) > 0:
print(
"Blasting error seqeuences (seqs don't align together and one doesn't align to whole)"
)
seqs_to_blast_flat = [
item for sublist in seqs_to_blast for item in sublist
]
try:
hits_all = blast_all(seqs_to_blast_flat, blast_dic_nums,
blast_dic_tcids, c)
except:
time.sleep(5)
hits_all = blast_all(seqs_to_blast_flat, blast_dic_nums,
blast_dic_tcids, c)
# print(hits_all)
print("Parsing taxonomy for error sequences")
for x, list_of_GIs in enumerate(seqs_to_blast):
hits = hits_all[x]
tc_id = blast_dic_tcids[list_of_GIs[0]]
#if theres only one lowest taxonomy hit and its not itself, change
if hits.count(min(hits)) == 1 and error_dic[tc_id] != list_of_GIs[
hits.index(min(hits))]:
finalseqs.add(list_of_GIs[hits.index(min(hits))])
# print(str(list_of_GIs[hits.index(min(hits))]) + " had closer taxonomy hit")
elif hits.count(min(
hits)) == 1 and error_dic[tc_id] == list_of_GIs[hits.index(
min(hits))]:
finalseqs.add(error_dic[tc_id])
# print(str(list_of_GIs[hits.index(min(hits))]) + " was previously chosen")
else: #there are multiple lowest taxonomy hits
# print('Taxonomies had multiple closest hits')
index_pos = []
count = 0
for x in hits:
if x == min(hits):
index_pos.append(count)
count += 1
mult_GIs = [list_of_GIs[x] for x in index_pos]
GI_to_pick = resolve_seqs(mult_GIs)
#if theres only one chosen and it wasnt the one already picked...add to change dic
if len(GI_to_pick) == 1 and error_dic[tc_id] != GI_to_pick[0]:
finalseqs.add(GI_to_pick[0])
# print(str(GI_to_pick) + " chosen")
#if theres only one max length and it was already picked
elif len(
GI_to_pick) == 1 and error_dic[tc_id] == GI_to_pick[0]:
# print(str(GI_to_pick) + " was previously chosen")
finalseqs.add(GI_to_pick[0])
else:
#this only happens if the originally picked one is crappy and the rest are the same
# print("Multiple choices: " + str(GI_to_pick))
multseqs.append(error_dic[tc_id])
#Go to cluster analysis
print('length of resolved=' + str(len(finalseqs)))
print('length of not resolved = ' + str(len(multseqs)))
with open("final_GIs.txt", "a") as o:
for m in finalseqs:
o.write(str(m) + "\n")
#to cluster
with open("multiple_gene_choices.txt", "a") as o:
for m in multseqs:
o.write(str(m) + "\n")
conn.close()
print(identities)
print('Multiple closest')
problem_dic[i] = multiple_dic[i]
GI_to_pick = [
multiple_dic[i][m] for m, x in enumerate(identities)
if x == max(identities)
]
print(GI_to_pick)
multfinalseqs.append(GI_to_pick)
for i in two_dic:
print(i)
#align the two seqs
list_of_GIs = two_dic[i]
print(list_of_GIs)
alignment = alignment_reg(list_of_GIs)
iden = identity_calc(alignment)
if iden < 95:
print("Low Aligned Identity: " + str(iden))
alignment = alignment_rev_comp(list_of_GIs)
iden = identity_calc(alignment)
if iden < 95:
#get taxonomy for query(main species)
print("Low Reverse Complement Aligned Identity: " + str(iden))
alignment = alignment_comp(list_of_GIs)
iden = identity_calc(alignment)
if iden < 95:
print("Low Complement Aligned Identity: " + str(iden))
#add tiling thing
gene_name = '_'.join(i.split('_')[1:])
idens, start_stop = tiling(list_of_GIs, gene_name)
genes.add(re.split('_|\|', i)[1])
count = 0
#deal with lengths of COI and add to dic to try and resolve
for i in dic_COI:
count += 1
print(float(count) / float(len(dic_COI)))
lengths = [int(m.split('_')[1]) for m in dic_COI[i]]
individual = [dic_COI[i][x] for x, l in enumerate(lengths) if l < 2000]
whole = [
dic_COI[i][x] for x, l in enumerate(lengths) if l > 2000 and l < 3000
]
mito = [dic_COI[i][x] for x, l in enumerate(lengths) if l > 3000]
if len(mito) > 0:
GIs_to_align = [mito[0].split("_")[0], 'GU365907']
alignment = alignment_reg(GIs_to_align)
iden = identity_calc(alignment)
if iden > 80:
#have to do span to account for random small blocks that dont align
span = 0
#get start
for l in range(len(alignment[0])):
col = alignment[:, l]
if '-' not in col:
span += 1
if span == 10:
break
elif span > 0 and '-' in col:
span = 0
start = l - 8
span = 0
def cluster(blastdb, taxdb):
Entrez.email = "*****@*****.**"
conn = sqlite3.connect(blastdb)
c = conn.cursor()
c.execute("ATTACH '" + taxdb + "' as 'tax'")
muscle_cline = MuscleCommandline(clwstrict=True)
input_dic = {}
multiple_dic = {}
two_dic = {}
problem_dic = {}
finalseqs = set()
multfinalseqs = []
unresolved = []
with open("multiple_gene_choices.txt") as o:
line = o.readline()
while line:
input_dic[line.split("\t")[0]] = line.strip().split(
"\t")[1].replace("[", "").replace("]", "").replace("'", "")
line = o.readline()
for i in input_dic:
GIs = input_dic[i]
GIs_list = GIs.split(", ")
if len(GIs_list) > 2:
multiple_dic[i] = GIs_list
if len(GIs_list) == 2:
two_dic[i] = GIs_list
for i in multiple_dic:
identities = []
joined_GIs = ",".join(multiple_dic[i])
handle = Entrez.efetch(db="nucleotide",
rettype="fasta",
retmode="text",
id=joined_GIs)
seqs = SeqIO.parse(handle, "fasta")
handle_string = StringIO()
SeqIO.write(seqs, handle_string, "fasta")
data = handle_string.getvalue()
stdout, stderr = muscle_cline(stdin=data)
align = AlignIO.read(StringIO(stdout), "clustal")
summary_align = AlignInfo.SummaryInfo(align)
consensus = summary_align.gap_consensus(threshold=.5, ambiguous='N')
consensus_record = SeqRecord(consensus, id="Consensus_all")
for m in multiple_dic[i]:
error = True
while error == True:
try:
handle = Entrez.efetch(db="nucleotide",
rettype="fasta",
retmode="text",
id=m)
error = False
except:
print('Error, trying again')
time.sleep(10)
seqs = SeqIO.read(handle, "fasta")
handle_string = StringIO()
SeqIO.write(seqs, handle_string, "fasta")
SeqIO.write(consensus_record, handle_string, "fasta")
data = handle_string.getvalue()
stdout, stderr = muscle_cline(stdin=data)
align = AlignIO.read(StringIO(stdout), "clustal")
count = 0
gaps = 0
for col in range(0, len(align[0])):
column = align[:, col]
if "-" not in column:
if column[1:] == column[:-1]:
count = count + 1
else:
gaps = gaps + 1
iden = 100 * (count / float((len(align[0]) - gaps)))
identities.append(iden)
if identities.count(max(identities)) == 1:
finalseqs.add(multiple_dic[i][identities.index(max(identities))])
else:
problem_dic[i] = multiple_dic[i]
GI_to_pick = [
multiple_dic[i][m] for m, x in enumerate(identities)
if x == max(identities)
]
multfinalseqs.append(GI_to_pick)
for i in two_dic:
#align the two seqs
list_of_GIs = two_dic[i]
alignment = alignment_reg(list_of_GIs)
iden = identity_calc(alignment)
if iden < 95:
# print("Low Aligned Identity: " + str(iden))
alignment = alignment_rev_comp(list_of_GIs)
iden = identity_calc(alignment)
if iden < 95:
#get taxonomy for query(main species)
# print("Low Reverse Complement Aligned Identity: " + str(iden))
alignment = alignment_comp(list_of_GIs)
iden = identity_calc(alignment)
if iden < 95:
# print("Low Complement Aligned Identity: " + str(iden))
#add tiling thing
gene_name = '_'.join(i.split('_')[1:])
idens, start_stop = tiling(list_of_GIs, gene_name)
current_start = -1
current_stop = -1
result = []
if all(m > 70 for m in idens):
for start, stop in sorted(start_stop):
if start > current_stop:
result.append((start, stop))
current_start, current_stop = start, stop
else:
current_stop = max(current_stop, stop)
result[-1] = (current_start, current_stop)
if len(result) == len(start_stop):
# print("Seqs align to different regions of probe, choosing all")
multfinalseqs.append(list_of_GIs)
else:
# print('Seqs overlap: Printing to file for hand checking')
with open('these_seqs_overlap_cluster.txt',
'a') as a:
unresolved.append(list_of_GIs)
a.write(str(list_of_GIs) + '\n')
else:
#get taxonomy for query(main species)
print("Parsing taxonomy for error sequences")
hits = blast(i, list_of_GIs, c)
#if theres only one lowest taxonomy hit, change
if hits.count(min(hits)) == 1:
finalseqs.add(
str(two_dic[i][hit_levels.index(min(hits))]))
# print(str(two_dic[i][hit_levels.index(min(hits))]) + " had closer taxonomy hit")
else: #there are multiple lowest taxonomy hits
multfinalseqs.append(two_dic[i])
problem_dic[i] = two_dic[i]
# print('Taxonomies had the multiple closest hits')
else:
multfinalseqs.append(two_dic[i])
# print("Complement iden: " + str(iden) + " so pair is fine")
else:
multfinalseqs.append(two_dic[i])
# print("Reverse Complement iden: " + str(iden) + " so pair is fine")
else:
multfinalseqs.append(two_dic[i])
# print("High Aligned Identity: " + str(iden) + " so pair is fine")
print("length of resolved = " + str(len(finalseqs)))
print("length of choose multiple = " + str(len(multfinalseqs)))
print("length of unresolved = " + str(len(unresolved)))
with open("final_GIs.txt", "a") as o:
for m in finalseqs:
o.write(str(m) + "\n")
with open("choose_mult.txt", "a") as o:
for m in [num for pair in multfinalseqs for num in pair]:
o.write(str(m) + "\n")
print(tc_id)
list_of_GIs = []
for iter in c.execute("SELECT Gene_name FROM blast WHERE GI ='" +
error_dic[tc_id] + "'"):
gene_name = str(iter[0])
for iter in c.execute("SELECT GI FROM blast WHERE tc_id = '" + tc_id +
"' and Gene_name = '" + gene_name + "'"):
list_of_GIs.append(str(iter[0]))
first_GI = list_of_GIs[0]
other_GIs = list_of_GIs[1:]
pair_check = []
#align each seq to the first one - first one acts as the 'default' direction
for x, GI in enumerate(other_GIs):
print("Pair #" + str(x + 1))
GI_pair = [first_GI, GI]
alignment = alignment_reg(GI_pair)
iden = identity_calc(alignment)
if iden < 90:
print("Low Aligned Identity: " + str(iden))
alignment = alignment_rev_comp(GI_pair)
iden = identity_calc(alignment)
if iden < 90:
print("Low Reverse Complement Aligned Identity: " + str(iden))
alignment = alignment_comp(GI_pair)
iden = identity_calc(alignment)
if iden < 90:
print("Low Complement Aligned Identity: " + str(iden))
pair_check.append(0)
else:
pair_check.append(1)
print("Complement iden: " + str(iden) + " so pair is fine")
def test_resolved_seqs(infile, blastdb, taxdb):
ent_query = get_blast_query(taxdb)
import sqlite3, sys, time, subprocess
from Bio.Blast import NCBIWWW, NCBIXML
from blastlib.clean_seq_funcs import resolve_seqs, alignment_comp, alignment_reg, alignment_rev_comp, blast, blast_all, identity_calc, tiling
from cleanlib.databasing import get_seqs_from_sqldb, export_fasta, create_blast_db, local_blast
conn = sqlite3.connect(blastdb)
c = conn.cursor()
c.execute("ATTACH '" + taxdb + "' as 'tax'")
error_dic = {}
blast_dic_nums = {}
blast_dic_tcids = {}
seqs_to_blast = []
finalseqs = set()
multseqs = []
#open self blast file for parsing
#make dictionary of query species: query GI of those that don't have the top hit as the same species
with open(infile) as p:
print("Parsing reciprocal blast output from " + infile)
blast_recs=NCBIXML.parse(p)
count = 0
numqueries = subprocess.check_output("grep -c '>' "+ infile.split(".xml")[0], shell=True)
for rec in blast_recs:
count += 1
print(str(round(float(count)/float(numqueries)*100, 2))+ "%")
queryAcc = str(rec.query.split()[0])
#this broke when ncbi updated
for iter in c.execute("SELECT GI FROM blast WHERE accession='" + queryAcc + "';"):
queryGI = (str(iter[0]))
#print(queryGI)
#hitdic is GIs and idens of 20 in .xml for each rec in blast_rec
hitdic = {}
hitSp = set()
count1 = 0
for alignment in rec.alignments:
for hsp in alignment.hsps:
identity=float(hsp.identities)/float(hsp.align_length)
#print(identity)
if alignment.hit_def == queryAcc:
pass
else:
for iter in c.execute("SELECT GI FROM blast WHERE accession='" + alignment.hit_def + "'"):
hitGI = (str(iter[0]))
count1 += 1
hitdic[str(hitGI)] = identity
if count1 >= 20:
break
# if alignment.title.split("|")[1] == queryGI:
# pass
# else:
# hitdic[str(alignment.title.split("|")[1])] = identity
if len(hitdic.values()) == 0:
#####################chose next one#######################
print('No matching hits from blast')
c.execute("UPDATE blast SET Decision='Chosen, but then did not blast to anything/Not chosen' WHERE GI = '" + queryGI + "';")
pass
else:
##first looks at just the top iden value
maxiden = max(hitdic.values())
try:
hitGIs = [GI for GI, iden in hitdic.iteritems() if iden == maxiden] #python2
except:
hitGIs = [GI for GI, iden in hitdic.items() if iden == maxiden] #python3
for iter in c.execute("SELECT tc_id FROM blast WHERE GI='" + queryGI + "'"):
querySp = (str(iter[0]))
for i in hitGIs:
for iter in c.execute("SELECT tc_id FROM blast WHERE GI='" + i + "'"):
hitSp.add(str(iter[0]))
##if it doesn't work, look at top 5 idens - ignore top hit b/c already looked at
if querySp not in hitSp:
hitSp = set()
if len(hitdic.values()) > 5:
maxiden = sorted(set(hitdic.values()), reverse = True)[1:5]
else:
maxiden = hitdic.values()
try:
hitGIs = [GI for GI, iden in hitdic.iteritems() if iden in maxiden] #python2
except:
hitGIs = [GI for GI, iden in hitdic.items() if iden in maxiden] #python3
#get hit species (actually tc_ids)
for i in hitGIs:
for iter in c.execute("SELECT tc_id FROM blast WHERE GI='" + i + "'"):
hitSp.add(str(iter[0]))
#if species of query not in the list of hit species
if querySp not in hitSp:
#will overwrite tiling info
#c.execute("UPDATE blast SET decision='Chosen, but does not reciprocal blast' WHERE GI='" + queryGI + "';")
if querySp not in error_dic:
error_dic[querySp] = [queryGI]
else:
output = error_dic[querySp]
output.append(querySp)
error_dic[querySp] = output
else:
c.execute("UPDATE blast SET decision='Longest or most info, good top hit/chosen' WHERE GI='" + queryGI + "';")
else:
c.execute("UPDATE blast SET decision='Longest or most info, good top hit/chosen' WHERE GI='" + queryGI + "';")
count = 0
#error_dic is dictionary that has species = GI that doesn't match species when self blast
# error_dic = {'81': [908332352], '56': '227435787', '60': '224939202', '39': '497153612', '157': '224939198', '87': '695134008', '628': '695134046', '184': '1174533403', '116': '558477133', '121': '316994106', '206': '545690375', '429': '1160414548', '431': '227436029', '480': '695134034', '105': '545690685', '651': '695134010', '413': '529377043', '373': '443611395', '252': '529388651', '354': '529386663', '335': '443611449', '584': '295078734', '375': '1559462858', '523': '1051545064', '350': '443611513', '410': '302487953', '527': '316994100', '244': '575502624', '355': '443611465', '595': '317467717', '380': '443611323', '357': '529374537', '577': '317467813', '363': '675401803', '406': '443611367', '372': '73533766', '656': '299833006'}
##go through error dictionary and align the 'same' gene/species to see if they're weird looking
print("Aligning sequences where chosen does not reciprocal blast to another")
for tc_id in error_dic:
for tile in error_dic[tc_id]:
count += 1
print(str(round(float(count)/float(len(error_dic))*100, 2))+"%")
list_of_GIs = [tile]
for iter in c.execute("SELECT Gene_name, Decision FROM blast WHERE GI ='" + tile + "'"):
gene = str(iter[0])
tilenum = str(iter[1].split()[7])
for iter in c.execute("SELECT GI FROM blast WHERE Decision = 'Short or less info, tile "+tilenum+"/Not chosen' and tc_id = '"+tc_id+"' and Gene_name = '"+gene+"'"):
list_of_GIs.append(str(iter[0]))
#first is always querygi
first_GI = list_of_GIs[0]
other_GIs = list_of_GIs[1:]
pair_check = []
#align each seq to the first one - first one acts as the 'default' direction
for x, GI in enumerate(other_GIs):
GI_pair = [first_GI, GI]
alignment = alignment_reg(GI_pair, blastdb, False, gene, c)
iden = identity_calc(alignment)
if iden < 90:
# print("Low Aligned Identity: " + str(iden))
alignment = alignment_rev_comp(GI_pair, blastdb, False, gene, c)
iden = identity_calc(alignment)
if iden < 90:
# print("Low Reverse Complement Aligned Identity: " + str(iden))
alignment = alignment_comp(GI_pair, blastdb, False, gene, c)
iden = identity_calc(alignment)
if iden < 90:
# print("Low Complement Aligned Identity: " + str(iden))
pair_check.append(0)
else:
pair_check.append(1)
# print("Complement iden: " + str(iden) + " so pair is fine")
else:
pair_check.append(1)
# print("Reverse Complement iden: " + str(iden) + " so pair is fine")
else:
pair_check.append(1)
# print("High Aligned Identity: " + str(iden) + " so pair is fine")
#print(pair_check)
#1 is when the GI aligned to chosen one just fine, 0 is when it doesn't
if all(i == 1 for i in pair_check):
c.execute("UPDATE blast SET decision='Sequence did not have same top blast species, but all aligned correctly, tile "+tilenum+"/Chosen' WHERE GI='" + tile + "';")
else:
##either all are 0s if first one is wrong or one is zero where one is wrong
# print('Somethings up with a sequence - printing to check - will blast')
seqs_to_blast.append(list_of_GIs)
list_of_GIs_str = str(list_of_GIs).replace("[", "(").replace("]", ")")
c.execute("UPDATE blast SET Decision='Chosen, but does not reciprocal blast and all do not align' WHERE GI IN " + list_of_GIs_str + ";")
# seqs_to_blast = [['695134046', '317467897'], ['316994106', '295069424', '414303142'], ['695134034', '71836048'], ['302487953', '73533700', '66096804'], ['575502624', '1723904125', '1723904119'], ['73533766', '443611441', '443611439'], ['296792260', '1531247074', '1531246852', '1531245174', '1531244766', '1531244542', '1531244412', '870902494', '520760133', '520760131', '331691381', '331690706', '331690580', '331690574', '331690568', '296467088', '296467084', '331691379', '331690792', '974999640', '974999638', '1531247630', '1531247410', '1531247308', '1531247180', '1531247130', '1531246686', '1531246650', '1531246450', '1531246148', '1531246132', '1531245926', '1531245702', '1531245624', '1531245592', '1531245462', '1531245454', '1531245440', '1531245354', '1531244888', '1531244834', '1531244804', '1531244664', '1531244614', '1531244586', '1531244178', '1531244132', '1531244016', '520760163', '520760159', '520760149', '331691355', '331690624', '331690576', '331690564', '331690544', '307641798', '296469132', '296468960', '296467194', '296466760', '304270861', '331690704', '63030162', '974999636', '227436371', '1532637164', '1532637082', '1532637024', '1531247684', '1531247306', '1531247040', '1531246466', '1531246208', '1531246108', '1531245848', '1531245842', '1531245638', '1531245614', '1531245574', '1531245376', '1531244734', '1531244658', '1531244080', '1531244012', '1531243910', '633896164', '520760155', '520759941', '331691337', '331690536', '300203102', '156619418', '156619410', '974999630', '296467200', '304270883', '1532637150', '1532637144', '1532637064', '1532637028', '1532637018', '730837173', '633896166', '520760157', '520760151', '331691341', '331691339', '331690582', '331690562', '1557923373', '1557923370', '156619416', '156619412', '1532637074', '1532637022', '1532637016', '156619414', '156619408', '730837157', '633896160', '1532637100', '1532637090', '1532637072', '1532637038', '633896162', '1042273875', '520760209', '313173207', '296468852', '88604889', '1532637050', '520760225', '520760219', '520760213', '296468054', '1532637044', '520760221', '520760193', '296792296', '312928999', '1557923367', '1557923364', '156619420', '331690820', '156619406', '520760211', '227436369', '296463378', '296463050', '227436365', '227436367', '296463374', '870901906', '870902242', '299833006', '76008949', '296464328', '296465368', '296464998', '296465182', '296464956', '296465198', '296463914']]
##sequences don't align together and chosen hit doesn't blast to same species (one area of tiling)
if len(seqs_to_blast) > 0:
for iter in c.execute("SELECT Gene_name FROM blast WHERE GI ='" + seqs_to_blast[0][0] + "'"):
gene = str(iter[0])
seqs_to_blast_flat = [item for sublist in seqs_to_blast for item in sublist]
#queryGI: nearest tax rank of hit
nearest_hit_taxo_dic = blast_all(seqs_to_blast_flat, c, gene, taxdb, blastdb)
print("Parsing taxonomy for error sequences")
for i in seqs_to_blast:
hit_taxonomies = [nearest_hit_taxo_dic[x] for x in i]
#want max tcid - that is closest to query
maxtcid = max(hit_taxonomies)
GI_to_pick = [GI for GI in enumerate(hit_taxonomies) if iden == maxtcid]
if len(GI_to_pick) == 1:
c.execute("UPDATE blast SET decision='Blast hits have closest taxonomy/Chosen' WHERE GI='" + GI_to_pick[0] + "';")
else:
GI_to_pick_str = str(GI_to_pick).replace("[", "(").replace("]", ")")
c.execute("UPDATE blast SET Decision='To cluster analysis/Chosen' WHERE GI IN " + GI_to_pick_str + ";")
conn.commit()
conn.close()
def resolve_seqs(blastdb):
import os, sys, sqlite3, re, time, itertools
from Bio import Seq, Entrez, SeqIO
from blastlib.clean_seq_funcs import resolve_seqs, alignment_reg, alignment_comp, alignment_rev_comp, identity_calc, tiling
conn = sqlite3.connect(blastdb)
c = conn.cursor()
Entrez.email = "*****@*****.**"
GI_nums_all = set()
GI_nums_single = set()
GI_nums_all_COI = set()
GI_nums_single_COI = set()
genes = set()
dic = {}
dic_COI = {}
dic_single = {}
dic_mult = {}
count2 = 1
records = []
#this gets list of all taxa/genes regardless if they have multiple gene choices or not
for iter in c.execute(
"SELECT tc_id, Gene_name, GI, hit_length FROM blast WHERE tc_id NOT NULL AND Gene_name != 'COI_trnL_COII' GROUP BY tc_id, Gene_name, GI;"
):
GI_nums_all.add(
str(iter[0]) + "_" + str(iter[1]) + "|" + str(iter[2]) + "_" +
str(iter[3]))
#this gets a list of all taxa/genes if they only have one gene choice
for iter in c.execute(
"SELECT tc_id, Gene_name, GI, hit_length FROM blast WHERE tc_id NOT NULL AND Gene_name != 'COI_trnL_COII' GROUP BY tc_id, Gene_name HAVING COUNT(*) =1;"
):
GI_nums_single.add(
str(iter[0]) + "_" + str(iter[1]) + "|" + str(iter[2]) + "_" +
str(iter[3]))
#this give me all the GIs that have multiple gene choices
GI_nums = GI_nums_all - GI_nums_single
#do the same with COI
#this gets list of all taxa/genes regardless if they have multiple gene choices or not
for iter in c.execute(
"SELECT tc_id, Gene_name, GI, hit_length FROM blast WHERE tc_id NOT NULL AND Gene_name == 'COI_trnL_COII' GROUP BY tc_id, Gene_name, GI;"
):
GI_nums_all_COI.add(
str(iter[0]) + "_" + str(iter[1]) + "|" + str(iter[2]) + "_" +
str(iter[3]))
#this gets a list of all taxa/genes if they only have one gene choice
for iter in c.execute(
"SELECT tc_id, Gene_name, GI, hit_length FROM blast WHERE tc_id NOT NULL AND Gene_name = 'COI_trnL_COII' GROUP BY tc_id HAVING COUNT(*) =1;"
):
GI_nums_single_COI.add(
str(iter[0]) + "_" + str(iter[1]) + "|" + str(iter[2]) + "_" +
str(iter[3]))
#this give me all the GIs that have multiple gene choices
GI_nums_mult_COI = GI_nums_all_COI - GI_nums_single_COI
for i in GI_nums_single_COI:
if int(i.split("_")[-1]) > 3000:
GI_nums_mult_COI.add(i)
#makes a dictionary of lists for each taxa/gene choice
for i in GI_nums:
if i.split("|")[0] in dic.keys():
dic_list = dic[i.split("|")[0]]
dic_list.append(i.split("|")[1])
dic[i.split("|")[0]] = dic_list
else:
dic[i.split("|")[0]] = [i.split("|")[1]]
genes.add(re.split('_|\|', i)[1])
#same for COI
for i in GI_nums_mult_COI:
if i.split("|")[0] in dic_COI.keys():
dic_list = dic_COI[i.split("|")[0]]
dic_list.append(i.split("|")[1])
dic_COI[i.split("|")[0]] = dic_list
else:
dic_COI[i.split("|")[0]] = [i.split("|")[1]]
genes.add(re.split('_|\|', i)[1])
count = 0
#deal with lengths of COI and add to dic to try and resolve
print("Trying to resolve COI/COII sequences")
for i in dic_COI:
count += 1
print(str(round(float(count) / float(len(dic_COI)) * 100, 2)) + '%')
lengths = [int(m.split('_')[1]) for m in dic_COI[i]]
individual = [dic_COI[i][x] for x, l in enumerate(lengths) if l < 2000]
whole = [
dic_COI[i][x] for x, l in enumerate(lengths)
if l > 2000 and l < 3000
]
mito = [dic_COI[i][x] for x, l in enumerate(lengths) if l > 3000]
if len(mito) > 0:
GIs_to_align = [mito[0].split("_")[0], 'GU365907']
alignment = alignment_reg(GIs_to_align)
iden = identity_calc(alignment)
if iden > 80:
#have to do span to account for random small blocks that dont align
span = 0
#get start
for l in range(len(alignment[0])):
col = alignment[:, l]
if '-' not in col:
span += 1
if span == 10:
break
elif span > 0 and '-' in col:
span = 0
start = l - 8
span = 0
#get stop
for l in reversed(range(len(alignment[0]))):
col = alignment[:, l]
if '-' not in col:
span += 1
if span == 10:
break
elif span > 0 and '-' in col:
span = 0
end = l + 10
handle = Entrez.efetch(db="nucleotide",
rettype="fasta",
retmode="text",
id=mito[0].split("_")[0],
seq_start=start - 1,
seq_stop=end - 2)
record = SeqIO.read(handle, "fasta")
records.append(record)
else:
print('Low iden when matching COI to whole mito')
with open("COI_hand_check.txt", "a") as a:
a.write(i + '\n')
elif len(whole) > 0:
#pick whole
if len(whole) == 1:
GI_nums_single.add(i + "|" + whole[0])
else:
#to pipeline
dic[i] = whole
if len(individual) > 0:
#pick individual
if len(individual) == 1:
GI_nums_single.add(i + "|" + individual[0])
else:
# print(individual)
result = []
ranges = {}
current_start = -1
current_stop = -1
whole_length = 0
#do tiling
for m in [x.split('_')[0] for x in individual]:
iden, start_stop = tiling([m], 'COI_trnL_COII')
start, end = start_stop[0]
#uses gi for danaus chrysippus COI_trnL_COII
if iden > 70:
#make dic of lists
if (start, end) in ranges.keys():
ranges_dic_list = ranges[(start, end)]
ranges_dic_list.append(m)
ranges[(start, end)] = ranges_dic_list
else:
ranges[(start, end)] = [m]
else:
print('Alignment below 70')
print(GIs_to_align)
if len(ranges) == 0:
print('All alignments below 70, printing to file')
with open("COI_hand_check.txt", "a") as a:
a.write(i + '\n')
#get merged range
for start, stop in sorted(ranges.keys()):
if start > current_stop:
result.append((start, stop))
current_start, current_stop = start, stop
else:
current_stop = max(current_stop, stop)
result[-1] = (current_start, current_stop)
for n in result:
whole_length += n[1] - n[0] + 1
#go through each combination of ranges to get 95% of whole range
for L in range(1, len(ranges) + 1):
max_perc = 0
for subset in itertools.combinations(ranges.keys(), L):
comb_length = 0
#for each combination, get merged length
current_start = -1
current_stop = -1
result = []
for start, stop in sorted(subset):
if start > current_stop:
result.append((start, stop))
current_start, current_stop = start, stop
else:
current_stop = max(current_stop, stop)
result[-1] = (current_start, current_stop)
for x in result:
comb_length += x[1] - x[0] + 1
if whole_length >= comb_length:
perc = float(comb_length) / float(whole_length)
if perc > max_perc:
max_perc = perc
max_subset = set()
max_subset.add(subset)
elif perc == max_perc:
max_subset.add(subset)
else:
pass
else:
pass
# goes through all combinations in a level before breaking
if max_perc > .95:
break
final_tiling = [(0, 0)] * L
for combination in max_subset:
for x, comb_frag in enumerate(sorted(combination)):
if comb_frag[1] - comb_frag[0] + 1 > final_tiling[x][
1] - final_tiling[x][0] + 1:
final_tiling[x] = comb_frag
# print(final_tiling)
possible_GIs = [ranges[x] for x in final_tiling]
# print(possible_GIs)
count = 0
for m in possible_GIs:
if len(m) == 1:
GI_nums_single.add(i + "|" + m[0] + "_0")
else:
if count == 0:
dic[i] = m
count += 1
else:
dic[i + "_" + str(count)] = m
count += 1
# merge ranges - if same number of ranges as original, keep all,
# want the least number to overlap 95% of whole range
# try each comb from low to high and if hits 95%, choose those
# if multiple higher than 95%, choose one with best %
# if multiple with same % and same numb of combs- multiple
print(dic)
# sys.exit()
SeqIO.write(records, "mito_COI.fa", "fasta")
#pulls out the GIs with first, the longest number of ATCGs and second, the longest length and makes dictionary
print("Trying to resolve all other sequences")
count = 0
for i in dic:
GIlist = []
for n in dic[i]:
GIlist.append(n.split("_")[0])
dic[i] = resolve_seqs(GIlist)
print(str(round((float(count) / float(len(dic))) * 100, 2)) + "%")
count += 1
#splits the ones that still have multiple (so the longest had multiple choices) and the ones that are resolved
for i in dic:
if len(dic[i]) > 1:
dic_mult[i] = dic[i]
else:
dic_single[i] = dic[i]
for i in genes:
finalGInums_only1 = set()
finalGInums_longest = set()
for n in dic_single.keys():
if i == re.split('_|\|', n)[1]:
finalGInums_longest.add(''.join(dic_single[n]))
for n in GI_nums_single:
if i == re.split('_|\|', n)[1]:
finalGInums_only1.add(re.split('_|\|', n)[-2])
with open("final_GIs.txt", "a") as o:
for m in finalGInums_only1:
o.write(str(m) + "\n")
#this needs to go to blast_sp_parse.py
with open(i + "_accession_nums_resolved.txt", "w") as o:
for m in finalGInums_longest:
o.write(str(m) + "\n")
#this needs to go to cluster.py
with open("multiple_gene_choices.txt", "w") as w:
for i in dic_mult:
w.write(i + "\t" + str(dic_mult[i]) + "\n")
conn.close()