def detect_and_remove_single_nonfinal_inwpcbg(inwpcbgs,PCG,GENE_IDENTIFIER_SET,
verbose=False):
"""
Allow deletion of a very shitty, single inwpCBG from the end of the list
"""
# we need at least 2 inwpCBGs in order to remove one of them
if len(inwpcbgs) <= 1: return False
lastInwpCBG = inwpcbgs[-1]
prevInwpCBG = inwpcbgs[-2]
lastNodeList = [ lastInwpCBG.get_organism_nodes(org)[0] for org in\
lastInwpCBG.organism_set().intersection(GENE_IDENTIFIER_SET) ]
prevNodeList = [ prevInwpCBG.get_organism_nodes(org)[0] for org in\
prevInwpCBG.organism_set().intersection(GENE_IDENTIFIER_SET) ]
# identical nodes -> do not delete. Only go for very obvious things
if Set(lastNodeList).intersection(prevNodeList): return False
ntdistdict = prevInwpCBG.nt_spacing_between_codingblocks([lastInwpCBG])
tcodedistdict = prevInwpCBG.tcode_spacing_between_codingblocks([lastInwpCBG])
check1 = prevInwpCBG.count_orfs_labeled_as_annotated_exon() >\
lastInwpCBG.count_orfs_labeled_as_annotated_exon()
check2 = prevInwpCBG.get_bitscore() > lastInwpCBG.get_bitscore()
check3 = len(prevNodeList) > len(lastNodeList)
check4 = float(lastInwpCBG.count_orfs_labeled_as_annotated_exon()) /\
float(len(GENE_IDENTIFIER_SET)) <= 0.33
if ntdistdict:
check5 = sum(ntdistdict.values())/float(len(ntdistdict)) >\
MIN_INTERGENIC_NT_LENGTH
else:
check5 = False
if tcodedistdict:
check6 = sum(tcodedistdict.values())/float(len(tcodedistdict)) <\
TCODE_MAX_NONCODING
else:
check6 = False
check7 = prevInwpCBG.get_projected_tailing_stop_aa_difference() <\
lastInwpCBG.get_projected_tailing_stop_aa_difference()
check8 = prevInwpCBG.get_projected_tailing_stop_nonaligned_aa_difference()<\
lastInwpCBG.get_projected_tailing_stop_nonaligned_aa_difference()
checklist = [check1,check2,check3,check4,check5,check6,check7,check8]
############################################################################
if verbose: print "NonFinal inwpCBG check:", checklist
############################################################################
if checklist.count(False) == 0:
nonfinalPCG = PacbpCollectionGraph(crossdata={},blastmatrix=PCG._blastmatrix)
# place all PacbPORFs in the nonfinalPCG
for (pacbpkey,nodeQ,nodeS), pacbporf in lastInwpCBG.pacbps.iteritems():
# add to noncodingnongenePCG
nonfinalPCG.add_node(nodeQ)
nonfinalPCG.add_node(nodeS)
nonfinalPCG.add_edge(nodeQ,nodeS,wt=pacbporf.bitscore)
nonfinalPCG.pacbps[(pacbpkey,nodeQ,nodeS)] = pacbporf
# remove from main PCG
_delete_pacbp(PCG,(pacbpkey,nodeQ,nodeS))
# return nonfinalPCG
return nonfinalPCG
else:
return False
def detect_and_remove_gtgdiscrepancy(inwpcbgs,PCG,GENE_IDENTIFIER_SET,verbose=True):
""" """
# if empty list or empty PCG provided: return False
if not inwpcbgs or not PCG or PCG.node_count() == 0: return False
# get target organism identifier
target = inwpcbgs[0]._get_target_organism()
# Make *the* GTG of the strongest X informant species
# X depends on the maximum number of gene informants (GENE_IDENTIFIER_SET);
# unigene informants are not taken into account here.
# X is defined here by:
# -- at least 3 informants (for very small number of informants)
# -- optimally half of the total numers of informants
# -- at most 8 informants
min_gtg_node_count = 3 + 1
max_gtg_node_count = 8 + 1
gtg_size = min([(len(GENE_IDENTIFIER_SET)-1)/2, max_gtg_node_count])
gtg_size = max([min_gtg_node_count,gtg_size])
btGTG = pcg2gtg_by_bitscore(PCG,target,identifier_list=GENE_IDENTIFIER_SET)
ntGTG = pcg2gtg_by_identity(PCG,target,identifier_list=GENE_IDENTIFIER_SET)
# TEMP solution because OrganismGraph != OrganismStarGraph
# make bitscore ordered list of nodes
bitscore_ordered_nodes = []
for (tNode,iNode),wt in btGTG.weights.iteritems():
if tNode==target: bitscore_ordered_nodes.append( ( wt, iNode ) )
bitscore_ordered_nodes.sort()
#if verbose: print "btGTG::", bitscore_ordered_nodes
while ntGTG.node_count() > gtg_size:
# next line causes errors because OrganismGraph != OrganismStarGraph
# this causes the target node in rare cases to be assigned as the weakest node
# informant = btGTG.weakest_connected_node()
(wt,informant) = bitscore_ordered_nodes.pop(0)
btGTG.del_node(informant)
ntGTG.del_node(informant)
if verbose: print "btGGT.weakest_connected_node() ==", informant, btGTG.get_ordered_nodes()
############################################################################
if verbose:
print "ntGTG:", ntGTG.get_ordered_nodes(),
for node in ntGTG.get_ordered_nodes():
if node == target: continue
print "%1.2f" % ntGTG.weights[(target,node)],
print ""
############################################################################
# detect inwpCBGs which are probably the result of intron alignments
gtgdiscrepancy_internal_inwpcbg_list = assign_internal_nongene_alignments(inwpcbgs,ntGTG)
# detect inwpCBGs with strong discrepancy to this GTG
gtgdiscrepancy_inwpcbg_list = assign_gtgdiscrepancy_inwpcbgs(inwpcbgs,ntGTG)
# merge both lists
if gtgdiscrepancy_internal_inwpcbg_list:
if not gtgdiscrepancy_inwpcbg_list:
gtgdiscrepancy_inwpcbg_list.extend(gtgdiscrepancy_internal_inwpcbg_list)
else:
for inwpcbg in gtgdiscrepancy_internal_inwpcbg_list:
check_str = str(inwpcbg)
if check_str not in [ str(gtgdiscrCBG) for gtgdiscrCBG in gtgdiscrepancy_inwpcbg_list ]:
gtgdiscrepancy_inwpcbg_list.append( inwpcbg )
if not gtgdiscrepancy_inwpcbg_list:
return False
# get list of inwpCBGs that have NO discrepancy
correct_inwpcbg_list = []
check_str_list = []
for discrinwpCBG in gtgdiscrepancy_inwpcbg_list:
check_str_list.append( str(discrinwpCBG) )
for inwpcbg in inwpcbgs:
if str(inwpcbg) not in check_str_list:
correct_inwpcbg_list.append( inwpcbg )
# get all pacbp keys belonging to gtgdiscrepancy inwpcbgs ONLY
gtgdiscrepancy_pacbpkeys = []
for discrinwpCBG in gtgdiscrepancy_inwpcbg_list:
for pacbpkey in discrinwpCBG.pacbps.keys():
# check if this pacbpkey is occuring in a non-removed inwpCBG
is_occurring_in_correct_inwpcbg = False
for inwp in correct_inwpcbg_list:
if pacbpkey in inwp.pacbps.keys():
is_occurring_in_correct_inwpcbg = True
break
# if is_occurring_in_correct_inwpcbg, continue and do not delete
if is_occurring_in_correct_inwpcbg:
continue
# store to gtgdiscrepancy_pacbpkeys when not stored already
if pacbpkey not in gtgdiscrepancy_pacbpkeys:
gtgdiscrepancy_pacbpkeys.append(pacbpkey)
# place all gtgdiscrepancy_pacbpkeys and PacbPORFs in the gtgdiscrepancyPCG
# and, at the same time, remove from the main PCG
gtgdiscrepancyPCG = PacbpCollectionGraph(crossdata={},blastmatrix=PCG._blastmatrix)
for key in gtgdiscrepancy_pacbpkeys:
if key not in PCG.pacbps.keys():
# !?!? TODO why not present in the PCG !?!?!
# anyway, continue here to avoid KeyError
# This PacbPORF was to be deleted rigth here,
# so it is not an extreme disaster. But... scary ;-)
continue
(pacbpkey,nodeQ,nodeS) = key
pacbporf = PCG.pacbps[key]
# add to gtgdiscrepancyPCG
gtgdiscrepancyPCG.add_node(nodeQ)
gtgdiscrepancyPCG.add_node(nodeS)
gtgdiscrepancyPCG.add_edge(nodeQ,nodeS,wt=pacbporf.bitscore)
gtgdiscrepancyPCG.pacbps[(pacbpkey,nodeQ,nodeS)] = pacbporf
# remove from main PCG
_delete_pacbp(PCG,key)
# return gtgdiscrepancyPCG
return gtgdiscrepancyPCG
def detect_and_remove_utrornonegene_inwpcbgs(inwpcbgs,PCG,verbose=True):
""" """
# if empty list or empty PCG provided: return False
if not inwpcbgs or not PCG or PCG.node_count() == 0: return False
# MAKE SHURE ALL Orfs HAVE PREDICTED TSS SITES!!
for inwpCBG in inwpcbgs: inwpCBG.scan_orfs_for_pssm_tss(min_pssm_score=TSS_MIN_PSSM_SCORE)
# get target organism identifier
target = inwpcbgs[0]._get_target_organism()
# detect inwpCBGs which are most likely 5' and 3' non coding or non gene
ncng_5p_list = assign_utrornongene5p_inwpcbgs(inwpcbgs)
ncng_3p_list = assign_utrornongene3p_inwpcbgs(inwpcbgs)
ncng_list = ncng_5p_list
ncng_list.extend(ncng_3p_list)
# return False in no inwpcbgs are assigned
if not ncng_list: return False
# get list of inwpCBGs that are NON ncng
correct_inwpcbg_list = []
check_str_list = []
for discrinwpCBG in ncng_list:
check_str_list.append( str(discrinwpCBG) )
for inwpcbg in inwpcbgs:
if str(inwpcbg) not in check_str_list:
correct_inwpcbg_list.append( inwpcbg )
# get all pacbp keys belonging to noncoding / nongene inwpcbgs ONLY
ncng_pacbpkeys = []
for ncnginwpCBG in ncng_list:
for pacbpkey in ncnginwpCBG.pacbps.keys():
# check if this pacbpkey is occuring in a non-removed inwpCBG
is_occurring_in_correct_inwpcbg = False
for inwp in correct_inwpcbg_list:
if pacbpkey in inwp.pacbps.keys():
is_occurring_in_correct_inwpcbg = True
break
# if is_occurring_in_correct_inwpcbg, continue and do not delete
if is_occurring_in_correct_inwpcbg:
continue
# store to gtgdiscrepancy_pacbpkeys when not stored already
if pacbpkey not in ncng_pacbpkeys:
ncng_pacbpkeys.append(pacbpkey)
# place all ncng_pacbpkeys and PacbPORFs in the noncodingnongenePCG
# and, at the same time, remove from the main PCG
noncodingnongenePCG = PacbpCollectionGraph(crossdata={},blastmatrix=PCG._blastmatrix)
for key in ncng_pacbpkeys:
(pacbpkey,nodeQ,nodeS) = key
pacbporf = PCG.pacbps[key]
# add to noncodingnongenePCG
noncodingnongenePCG.add_node(nodeQ)
noncodingnongenePCG.add_node(nodeS)
noncodingnongenePCG.add_edge(nodeQ,nodeS,wt=pacbporf.bitscore)
noncodingnongenePCG.pacbps[(pacbpkey,nodeQ,nodeS)] = pacbporf
# remove from main PCG
_delete_pacbp(PCG,key)
# return noncodingnongenePCG
return noncodingnongenePCG
def detect_and_remove_synteny(inwpcbgs,
PCG,
GENE_IDENTIFIER_SET,
verbose=True):
""" """
MIN_OBSERVED_VS_EXPECTED_RATIO = 0.20
observed_organism_subcombis = []
syntenic_subinwpcbgs = []
# detect syntenic genes in MAIN inwpCBGs,
# without taking strongest informants by GTG analyses
syntenic_inwpcbgs = assign_syntenic_inwpcbgs(inwpcbgs)
for syntinwpcbg in syntenic_inwpcbgs:
syntenic_subinwpcbgs.append(syntinwpcbg)
for inwpCBG in inwpcbgs:
# omit inwpCBGs with annotated exons/orfs
if inwpCBG.count_orfs_labeled_as_annotated_exon() >= 2: continue
target = inwpCBG._get_target_organism()
# make a (artificially fully connected) GeneTreeGraph
gtg = GeneTreeGraph()
gtg.add_node(target)
for (pacbpkey, nodeQ, nodeS), pacbporf in inwpCBG.pacbps.iteritems():
orgS = inwpCBG.organism_by_node(nodeS)
if orgS not in GENE_IDENTIFIER_SET: continue
gtg.add_node(orgS)
for (pacbpkey, nodeQ, nodeS), pacbporf in inwpCBG.pacbps.iteritems():
orgQ = inwpCBG.organism_by_node(nodeQ)
orgS = inwpCBG.organism_by_node(nodeS)
if orgS not in GENE_IDENTIFIER_SET: continue
gtg.add_edge(orgQ, orgS, wt=pacbporf.bitscore)
# make artificially missed edges between the informants
for org in inwpCBG.organism_set():
if org not in [orgQ, orgS] and org in GENE_IDENTIFIER_SET:
if gtg.has_edge( orgS, org ) and\
gtg.weights[(orgS, org)] > pacbporf.bitscore:
gtg.set_edge_weight(orgS, org, wt=pacbporf.bitscore)
else:
gtg.add_edge(orgS, org, wt=pacbporf.bitscore)
# omit (nearly) empty genetreegraphs
if gtg.node_count() <= 1: continue
# remove (much) weaker connected nodes as expected from the gtg
while gtg.get_nodes() and MIN_OBSERVED_VS_EXPECTED_RATIO >\
min( [ gtg.get_node_weighted_connectivity_observed_vs_expected(node) for node in gtg.get_nodes() ]):
node = gtg.weakest_connected_node()
gtg.del_node(node)
# check if already tested before; present in observed_organism_subcombis
if gtg.get_ordered_nodes() in observed_organism_subcombis: continue
# store to already tested organism subcombinations
observed_organism_subcombis.append(gtg.get_ordered_nodes())
# create a subPCG of these organisms
subPCG = PacbpCollectionGraph(crossdata={},
blastmatrix=PCG._blastmatrix)
for (pacbpkey, nodeQ, nodeS), pacbporf in PCG.pacbps.iteritems():
(orgQ, orfQid), (orgS, orfSid) = nodeQ, nodeS
if orgQ not in gtg.get_nodes(): continue
if orgS not in gtg.get_nodes(): continue
subPCG.add_node(nodeQ)
subPCG.add_node(nodeS)
subPCG.add_edge(nodeQ, nodeS, wt=pacbporf.bitscore)
subPCG.pacbps[(pacbpkey, nodeQ, nodeS)] = pacbporf
# make inwpCBGs of this subPCG
subinwpcbgs = PCG2inwpCBGS(subPCG)
# check if there are subinwpcbgs
if not subinwpcbgs: continue
########################################################################
#if verbose:
# print "subPCG organism set:", gtg.get_ordered_nodes()
# print_inwpcbgstructure(subinwpcbgs,gtg.get_ordered_nodes())
########################################################################
# create a subInwardsPointingCodingBlockGraph of these organisms
#subinwpCBG = InwardsPointingCodingBlockGraph()
#for (pacbpkey,nodeQ,nodeS), pacbporf in inwpCBG.pacbps.iteritems():
# (orgQ,orfQid),(orgS,orfSid) = nodeQ,nodeS
# if orgQ not in gtg.get_nodes(): continue
# if orgS not in gtg.get_nodes(): continue
# subinwpCBG.add_node(nodeQ)
# subinwpCBG.add_node(nodeS)
# subinwpCBG.add_edge(nodeQ,nodeS,wt=pacbporf.bitscore)
# subinwpCBG.pacbps[(pacbpkey,nodeQ,nodeS)] = pacbporf
# detect syntenic genes in this subinwpcbgs
syntenic_inwpcbgs = assign_syntenic_inwpcbgs(subinwpcbgs)
for syntinwpcbg in syntenic_inwpcbgs:
syntenic_subinwpcbgs.append(syntinwpcbg)
####################################################################
if verbose:
print "SYNTENIC!!", syntinwpcbg, syntinwpcbg.get_ordered_nodes(
)
for subCBG in subinwpcbgs:
print "syntenic in:", subCBG, subCBG.get_ordered_nodes()
####################################################################
if not syntenic_subinwpcbgs:
return False
# cleanup all inwpCBGs from the syntenic subInwpCBGs
syntenic_pacbpkeys = []
for syntinwpcbg in syntenic_subinwpcbgs:
node_set = syntinwpcbg.node_set()
for inwpCBG in inwpcbgs:
if not node_set.difference(inwpCBG.node_set()):
for pacbpkey in inwpCBG.pacbps.keys():
if pacbpkey not in syntenic_pacbpkeys:
syntenic_pacbpkeys.append(pacbpkey)
# place all syntenic_pacbpkeys and PacbPORFs in the syntenicPCG
# and, at the same time, remove from the main PCG
syntenicPCG = PacbpCollectionGraph(crossdata={},
blastmatrix=PCG._blastmatrix)
for key in syntenic_pacbpkeys:
(pacbpkey, nodeQ, nodeS) = key
pacbporf = PCG.pacbps[key]
# add to syntenicPCG
syntenicPCG.add_node(nodeQ)
syntenicPCG.add_node(nodeS)
syntenicPCG.add_edge(nodeQ, nodeS, wt=pacbporf.bitscore)
syntenicPCG.pacbps[(pacbpkey, nodeQ, nodeS)] = pacbporf
# remove from main PCG
_delete_pacbp(PCG, key)
# return syntenicPCG
return syntenicPCG
def detect_and_remove_single_nonfirst_inwpcbg(inwpcbgs,PCG,GENE_IDENTIFIER_SET,
verbose=False):
"""
Allow deletion of a very shitty, single inwpCBG from the start of the list
"""
# we need at least 2 inwpCBGs in order to remove one of them
if len(inwpcbgs) <= 1: return False
firstInwpCBG = inwpcbgs[0]
nextInwpCBG = inwpcbgs[1]
firstNodeList = [ firstInwpCBG.get_organism_nodes(org)[0] for org in\
firstInwpCBG.organism_set().intersection(GENE_IDENTIFIER_SET) ]
nextNodeList = [ nextInwpCBG.get_organism_nodes(org)[0] for org in\
nextInwpCBG.organism_set().intersection(GENE_IDENTIFIER_SET) ]
# identical nodes -> do not delete. Only go for very obvious things
if Set(firstNodeList).intersection(nextNodeList): return False
ntdistdict = firstInwpCBG.nt_spacing_between_codingblocks([nextInwpCBG])
tcodedistdict = firstInwpCBG.tcode_spacing_between_codingblocks([nextInwpCBG])
# make a long list of checks which should be True in case
# firstInwpCBG is *NOT* the first exon of this gene structure
check1 = nextInwpCBG.count_orfs_labeled_as_annotated_exon() >\
firstInwpCBG.count_orfs_labeled_as_annotated_exon()
check2 = nextInwpCBG.get_bitscore() > firstInwpCBG.get_bitscore()
check3 = len(nextNodeList) > len(firstNodeList)
check4 = float(firstInwpCBG.count_orfs_labeled_as_annotated_exon()) /\
float(len(GENE_IDENTIFIER_SET)) <= 0.33
if ntdistdict:
check5 = sum(ntdistdict.values())/float(len(ntdistdict)) >\
MIN_INTERGENIC_NT_LENGTH
else:
check5 = False
if tcodedistdict:
check6 = sum(tcodedistdict.values())/float(len(tcodedistdict)) <\
TCODE_MAX_NONCODING
else:
check6 = False
check7 = nextInwpCBG.count_orfs_labeled_as_first_exon() >=\
firstInwpCBG.count_orfs_labeled_as_first_exon()
check8 = firstInwpCBG.count_orfs_labeled_as_annotated_exon() == 0
check9 = nextInwpCBG.get_average_upstream_methionine_pssm_score() >\
firstInwpCBG.get_average_upstream_methionine_pssm_score()
checklist = [check1,check2,check3,check4,check5,check6,check7,check8,check9]
############################################################################
if verbose or True: print "NonFirst inwpCBG check:", checklist
############################################################################
if checklist.count(False) <= 1:
nonfirstPCG = PacbpCollectionGraph(crossdata={},blastmatrix=PCG._blastmatrix)
# place all PacbPORFs in the nonfirstPCG
for (pacbpkey,nodeQ,nodeS), pacbporf in firstInwpCBG.pacbps.iteritems():
# add to noncodingnongenePCG
nonfirstPCG.add_node(nodeQ)
nonfirstPCG.add_node(nodeS)
nonfirstPCG.add_edge(nodeQ,nodeS,wt=pacbporf.bitscore)
nonfirstPCG.pacbps[(pacbpkey,nodeQ,nodeS)] = pacbporf
# remove from main PCG
_delete_pacbp(PCG,(pacbpkey,nodeQ,nodeS))
# return nonfirstPCG
return nonfirstPCG
else:
return False
def detect_and_remove_single_nonfinal_inwpcbg(inwpcbgs,
PCG,
GENE_IDENTIFIER_SET,
verbose=False):
"""
Allow deletion of a very shitty, single inwpCBG from the end of the list
"""
# we need at least 2 inwpCBGs in order to remove one of them
if len(inwpcbgs) <= 1: return False
lastInwpCBG = inwpcbgs[-1]
prevInwpCBG = inwpcbgs[-2]
lastNodeList = [ lastInwpCBG.get_organism_nodes(org)[0] for org in\
lastInwpCBG.organism_set().intersection(GENE_IDENTIFIER_SET) ]
prevNodeList = [ prevInwpCBG.get_organism_nodes(org)[0] for org in\
prevInwpCBG.organism_set().intersection(GENE_IDENTIFIER_SET) ]
# identical nodes -> do not delete. Only go for very obvious things
if Set(lastNodeList).intersection(prevNodeList): return False
ntdistdict = prevInwpCBG.nt_spacing_between_codingblocks([lastInwpCBG])
tcodedistdict = prevInwpCBG.tcode_spacing_between_codingblocks(
[lastInwpCBG])
check1 = prevInwpCBG.count_orfs_labeled_as_annotated_exon() >\
lastInwpCBG.count_orfs_labeled_as_annotated_exon()
check2 = prevInwpCBG.get_bitscore() > lastInwpCBG.get_bitscore()
check3 = len(prevNodeList) > len(lastNodeList)
check4 = float(lastInwpCBG.count_orfs_labeled_as_annotated_exon()) /\
float(len(GENE_IDENTIFIER_SET)) <= 0.33
if ntdistdict:
check5 = sum(ntdistdict.values())/float(len(ntdistdict)) >\
MIN_INTERGENIC_NT_LENGTH
else:
check5 = False
if tcodedistdict:
check6 = sum(tcodedistdict.values())/float(len(tcodedistdict)) <\
TCODE_MAX_NONCODING
else:
check6 = False
check7 = prevInwpCBG.get_projected_tailing_stop_aa_difference() <\
lastInwpCBG.get_projected_tailing_stop_aa_difference()
check8 = prevInwpCBG.get_projected_tailing_stop_nonaligned_aa_difference()<\
lastInwpCBG.get_projected_tailing_stop_nonaligned_aa_difference()
checklist = [
check1, check2, check3, check4, check5, check6, check7, check8
]
############################################################################
if verbose: print "NonFinal inwpCBG check:", checklist
############################################################################
if checklist.count(False) == 0:
nonfinalPCG = PacbpCollectionGraph(crossdata={},
blastmatrix=PCG._blastmatrix)
# place all PacbPORFs in the nonfinalPCG
for (pacbpkey, nodeQ,
nodeS), pacbporf in lastInwpCBG.pacbps.iteritems():
# add to noncodingnongenePCG
nonfinalPCG.add_node(nodeQ)
nonfinalPCG.add_node(nodeS)
nonfinalPCG.add_edge(nodeQ, nodeS, wt=pacbporf.bitscore)
nonfinalPCG.pacbps[(pacbpkey, nodeQ, nodeS)] = pacbporf
# remove from main PCG
_delete_pacbp(PCG, (pacbpkey, nodeQ, nodeS))
# return nonfinalPCG
return nonfinalPCG
else:
return False
def detect_and_remove_utrornonegene_inwpcbgs(inwpcbgs, PCG, verbose=True):
""" """
# if empty list or empty PCG provided: return False
if not inwpcbgs or not PCG or PCG.node_count() == 0: return False
# MAKE SHURE ALL Orfs HAVE PREDICTED TSS SITES!!
for inwpCBG in inwpcbgs:
inwpCBG.scan_orfs_for_pssm_tss(min_pssm_score=TSS_MIN_PSSM_SCORE)
# get target organism identifier
target = inwpcbgs[0]._get_target_organism()
# detect inwpCBGs which are most likely 5' and 3' non coding or non gene
ncng_5p_list = assign_utrornongene5p_inwpcbgs(inwpcbgs)
ncng_3p_list = assign_utrornongene3p_inwpcbgs(inwpcbgs)
ncng_list = ncng_5p_list
ncng_list.extend(ncng_3p_list)
# return False in no inwpcbgs are assigned
if not ncng_list: return False
# get list of inwpCBGs that are NON ncng
correct_inwpcbg_list = []
check_str_list = []
for discrinwpCBG in ncng_list:
check_str_list.append(str(discrinwpCBG))
for inwpcbg in inwpcbgs:
if str(inwpcbg) not in check_str_list:
correct_inwpcbg_list.append(inwpcbg)
# get all pacbp keys belonging to noncoding / nongene inwpcbgs ONLY
ncng_pacbpkeys = []
for ncnginwpCBG in ncng_list:
for pacbpkey in ncnginwpCBG.pacbps.keys():
# check if this pacbpkey is occuring in a non-removed inwpCBG
is_occurring_in_correct_inwpcbg = False
for inwp in correct_inwpcbg_list:
if pacbpkey in inwp.pacbps.keys():
is_occurring_in_correct_inwpcbg = True
break
# if is_occurring_in_correct_inwpcbg, continue and do not delete
if is_occurring_in_correct_inwpcbg:
continue
# store to gtgdiscrepancy_pacbpkeys when not stored already
if pacbpkey not in ncng_pacbpkeys:
ncng_pacbpkeys.append(pacbpkey)
# place all ncng_pacbpkeys and PacbPORFs in the noncodingnongenePCG
# and, at the same time, remove from the main PCG
noncodingnongenePCG = PacbpCollectionGraph(crossdata={},
blastmatrix=PCG._blastmatrix)
for key in ncng_pacbpkeys:
(pacbpkey, nodeQ, nodeS) = key
pacbporf = PCG.pacbps[key]
# add to noncodingnongenePCG
noncodingnongenePCG.add_node(nodeQ)
noncodingnongenePCG.add_node(nodeS)
noncodingnongenePCG.add_edge(nodeQ, nodeS, wt=pacbporf.bitscore)
noncodingnongenePCG.pacbps[(pacbpkey, nodeQ, nodeS)] = pacbporf
# remove from main PCG
_delete_pacbp(PCG, key)
# return noncodingnongenePCG
return noncodingnongenePCG
def detect_and_remove_gtgdiscrepancy(inwpcbgs,
PCG,
GENE_IDENTIFIER_SET,
verbose=True):
""" """
# if empty list or empty PCG provided: return False
if not inwpcbgs or not PCG or PCG.node_count() == 0: return False
# get target organism identifier
target = inwpcbgs[0]._get_target_organism()
# Make *the* GTG of the strongest X informant species
# X depends on the maximum number of gene informants (GENE_IDENTIFIER_SET);
# unigene informants are not taken into account here.
# X is defined here by:
# -- at least 3 informants (for very small number of informants)
# -- optimally half of the total numers of informants
# -- at most 8 informants
min_gtg_node_count = 3 + 1
max_gtg_node_count = 8 + 1
gtg_size = min([(len(GENE_IDENTIFIER_SET) - 1) / 2, max_gtg_node_count])
gtg_size = max([min_gtg_node_count, gtg_size])
btGTG = pcg2gtg_by_bitscore(PCG,
target,
identifier_list=GENE_IDENTIFIER_SET)
ntGTG = pcg2gtg_by_identity(PCG,
target,
identifier_list=GENE_IDENTIFIER_SET)
# TEMP solution because OrganismGraph != OrganismStarGraph
# make bitscore ordered list of nodes
bitscore_ordered_nodes = []
for (tNode, iNode), wt in btGTG.weights.iteritems():
if tNode == target: bitscore_ordered_nodes.append((wt, iNode))
bitscore_ordered_nodes.sort()
#if verbose: print "btGTG::", bitscore_ordered_nodes
while ntGTG.node_count() > gtg_size:
# next line causes errors because OrganismGraph != OrganismStarGraph
# this causes the target node in rare cases to be assigned as the weakest node
# informant = btGTG.weakest_connected_node()
(wt, informant) = bitscore_ordered_nodes.pop(0)
btGTG.del_node(informant)
ntGTG.del_node(informant)
if verbose:
print "btGGT.weakest_connected_node() ==", informant, btGTG.get_ordered_nodes(
)
############################################################################
if verbose:
print "ntGTG:", ntGTG.get_ordered_nodes(),
for node in ntGTG.get_ordered_nodes():
if node == target: continue
print "%1.2f" % ntGTG.weights[(target, node)],
print ""
############################################################################
# detect inwpCBGs which are probably the result of intron alignments
gtgdiscrepancy_internal_inwpcbg_list = assign_internal_nongene_alignments(
inwpcbgs, ntGTG)
# detect inwpCBGs with strong discrepancy to this GTG
gtgdiscrepancy_inwpcbg_list = assign_gtgdiscrepancy_inwpcbgs(
inwpcbgs, ntGTG)
# merge both lists
if gtgdiscrepancy_internal_inwpcbg_list:
if not gtgdiscrepancy_inwpcbg_list:
gtgdiscrepancy_inwpcbg_list.extend(
gtgdiscrepancy_internal_inwpcbg_list)
else:
for inwpcbg in gtgdiscrepancy_internal_inwpcbg_list:
check_str = str(inwpcbg)
if check_str not in [
str(gtgdiscrCBG)
for gtgdiscrCBG in gtgdiscrepancy_inwpcbg_list
]:
gtgdiscrepancy_inwpcbg_list.append(inwpcbg)
if not gtgdiscrepancy_inwpcbg_list:
return False
# get list of inwpCBGs that have NO discrepancy
correct_inwpcbg_list = []
check_str_list = []
for discrinwpCBG in gtgdiscrepancy_inwpcbg_list:
check_str_list.append(str(discrinwpCBG))
for inwpcbg in inwpcbgs:
if str(inwpcbg) not in check_str_list:
correct_inwpcbg_list.append(inwpcbg)
# get all pacbp keys belonging to gtgdiscrepancy inwpcbgs ONLY
gtgdiscrepancy_pacbpkeys = []
for discrinwpCBG in gtgdiscrepancy_inwpcbg_list:
for pacbpkey in discrinwpCBG.pacbps.keys():
# check if this pacbpkey is occuring in a non-removed inwpCBG
is_occurring_in_correct_inwpcbg = False
for inwp in correct_inwpcbg_list:
if pacbpkey in inwp.pacbps.keys():
is_occurring_in_correct_inwpcbg = True
break
# if is_occurring_in_correct_inwpcbg, continue and do not delete
if is_occurring_in_correct_inwpcbg:
continue
# store to gtgdiscrepancy_pacbpkeys when not stored already
if pacbpkey not in gtgdiscrepancy_pacbpkeys:
gtgdiscrepancy_pacbpkeys.append(pacbpkey)
# place all gtgdiscrepancy_pacbpkeys and PacbPORFs in the gtgdiscrepancyPCG
# and, at the same time, remove from the main PCG
gtgdiscrepancyPCG = PacbpCollectionGraph(crossdata={},
blastmatrix=PCG._blastmatrix)
for key in gtgdiscrepancy_pacbpkeys:
if key not in PCG.pacbps.keys():
# !?!? TODO why not present in the PCG !?!?!
# anyway, continue here to avoid KeyError
# This PacbPORF was to be deleted rigth here,
# so it is not an extreme disaster. But... scary ;-)
continue
(pacbpkey, nodeQ, nodeS) = key
pacbporf = PCG.pacbps[key]
# add to gtgdiscrepancyPCG
gtgdiscrepancyPCG.add_node(nodeQ)
gtgdiscrepancyPCG.add_node(nodeS)
gtgdiscrepancyPCG.add_edge(nodeQ, nodeS, wt=pacbporf.bitscore)
gtgdiscrepancyPCG.pacbps[(pacbpkey, nodeQ, nodeS)] = pacbporf
# remove from main PCG
_delete_pacbp(PCG, key)
# return gtgdiscrepancyPCG
return gtgdiscrepancyPCG
def detect_and_remove_single_nonfirst_inwpcbg(inwpcbgs,
PCG,
GENE_IDENTIFIER_SET,
verbose=False):
"""
Allow deletion of a very shitty, single inwpCBG from the start of the list
"""
# we need at least 2 inwpCBGs in order to remove one of them
if len(inwpcbgs) <= 1: return False
firstInwpCBG = inwpcbgs[0]
nextInwpCBG = inwpcbgs[1]
firstNodeList = [ firstInwpCBG.get_organism_nodes(org)[0] for org in\
firstInwpCBG.organism_set().intersection(GENE_IDENTIFIER_SET) ]
nextNodeList = [ nextInwpCBG.get_organism_nodes(org)[0] for org in\
nextInwpCBG.organism_set().intersection(GENE_IDENTIFIER_SET) ]
# identical nodes -> do not delete. Only go for very obvious things
if Set(firstNodeList).intersection(nextNodeList): return False
ntdistdict = firstInwpCBG.nt_spacing_between_codingblocks([nextInwpCBG])
tcodedistdict = firstInwpCBG.tcode_spacing_between_codingblocks(
[nextInwpCBG])
# make a long list of checks which should be True in case
# firstInwpCBG is *NOT* the first exon of this gene structure
check1 = nextInwpCBG.count_orfs_labeled_as_annotated_exon() >\
firstInwpCBG.count_orfs_labeled_as_annotated_exon()
check2 = nextInwpCBG.get_bitscore() > firstInwpCBG.get_bitscore()
check3 = len(nextNodeList) > len(firstNodeList)
check4 = float(firstInwpCBG.count_orfs_labeled_as_annotated_exon()) /\
float(len(GENE_IDENTIFIER_SET)) <= 0.33
if ntdistdict:
check5 = sum(ntdistdict.values())/float(len(ntdistdict)) >\
MIN_INTERGENIC_NT_LENGTH
else:
check5 = False
if tcodedistdict:
check6 = sum(tcodedistdict.values())/float(len(tcodedistdict)) <\
TCODE_MAX_NONCODING
else:
check6 = False
check7 = nextInwpCBG.count_orfs_labeled_as_first_exon() >=\
firstInwpCBG.count_orfs_labeled_as_first_exon()
check8 = firstInwpCBG.count_orfs_labeled_as_annotated_exon() == 0
check9 = nextInwpCBG.get_average_upstream_methionine_pssm_score() >\
firstInwpCBG.get_average_upstream_methionine_pssm_score()
checklist = [
check1, check2, check3, check4, check5, check6, check7, check8, check9
]
############################################################################
if verbose or True: print "NonFirst inwpCBG check:", checklist
############################################################################
if checklist.count(False) <= 1:
nonfirstPCG = PacbpCollectionGraph(crossdata={},
blastmatrix=PCG._blastmatrix)
# place all PacbPORFs in the nonfirstPCG
for (pacbpkey, nodeQ,
nodeS), pacbporf in firstInwpCBG.pacbps.iteritems():
# add to noncodingnongenePCG
nonfirstPCG.add_node(nodeQ)
nonfirstPCG.add_node(nodeS)
nonfirstPCG.add_edge(nodeQ, nodeS, wt=pacbporf.bitscore)
nonfirstPCG.pacbps[(pacbpkey, nodeQ, nodeS)] = pacbporf
# remove from main PCG
_delete_pacbp(PCG, (pacbpkey, nodeQ, nodeS))
# return nonfirstPCG
return nonfirstPCG
else:
return False
def detect_and_remove_synteny(inwpcbgs,PCG,GENE_IDENTIFIER_SET,verbose=True):
""" """
MIN_OBSERVED_VS_EXPECTED_RATIO = 0.20
observed_organism_subcombis = []
syntenic_subinwpcbgs = []
# detect syntenic genes in MAIN inwpCBGs,
# without taking strongest informants by GTG analyses
syntenic_inwpcbgs = assign_syntenic_inwpcbgs(inwpcbgs)
for syntinwpcbg in syntenic_inwpcbgs:
syntenic_subinwpcbgs.append(syntinwpcbg)
for inwpCBG in inwpcbgs:
# omit inwpCBGs with annotated exons/orfs
if inwpCBG.count_orfs_labeled_as_annotated_exon() >= 2: continue
target = inwpCBG._get_target_organism()
# make a (artificially fully connected) GeneTreeGraph
gtg = GeneTreeGraph()
gtg.add_node(target)
for (pacbpkey,nodeQ,nodeS),pacbporf in inwpCBG.pacbps.iteritems():
orgS = inwpCBG.organism_by_node(nodeS)
if orgS not in GENE_IDENTIFIER_SET: continue
gtg.add_node(orgS)
for (pacbpkey,nodeQ,nodeS),pacbporf in inwpCBG.pacbps.iteritems():
orgQ = inwpCBG.organism_by_node(nodeQ)
orgS = inwpCBG.organism_by_node(nodeS)
if orgS not in GENE_IDENTIFIER_SET: continue
gtg.add_edge( orgQ, orgS, wt = pacbporf.bitscore )
# make artificially missed edges between the informants
for org in inwpCBG.organism_set():
if org not in [orgQ,orgS] and org in GENE_IDENTIFIER_SET:
if gtg.has_edge( orgS, org ) and\
gtg.weights[(orgS, org)] > pacbporf.bitscore:
gtg.set_edge_weight(orgS,org,wt = pacbporf.bitscore)
else:
gtg.add_edge( orgS, org, wt = pacbporf.bitscore )
# omit (nearly) empty genetreegraphs
if gtg.node_count() <= 1: continue
# remove (much) weaker connected nodes as expected from the gtg
while gtg.get_nodes() and MIN_OBSERVED_VS_EXPECTED_RATIO >\
min( [ gtg.get_node_weighted_connectivity_observed_vs_expected(node) for node in gtg.get_nodes() ]):
node = gtg.weakest_connected_node()
gtg.del_node(node)
# check if already tested before; present in observed_organism_subcombis
if gtg.get_ordered_nodes() in observed_organism_subcombis: continue
# store to already tested organism subcombinations
observed_organism_subcombis.append( gtg.get_ordered_nodes() )
# create a subPCG of these organisms
subPCG = PacbpCollectionGraph(crossdata={},
blastmatrix=PCG._blastmatrix)
for (pacbpkey,nodeQ,nodeS), pacbporf in PCG.pacbps.iteritems():
(orgQ,orfQid),(orgS,orfSid) = nodeQ,nodeS
if orgQ not in gtg.get_nodes(): continue
if orgS not in gtg.get_nodes(): continue
subPCG.add_node(nodeQ)
subPCG.add_node(nodeS)
subPCG.add_edge(nodeQ,nodeS,wt=pacbporf.bitscore)
subPCG.pacbps[(pacbpkey,nodeQ,nodeS)] = pacbporf
# make inwpCBGs of this subPCG
subinwpcbgs = PCG2inwpCBGS(subPCG)
# check if there are subinwpcbgs
if not subinwpcbgs: continue
########################################################################
#if verbose:
# print "subPCG organism set:", gtg.get_ordered_nodes()
# print_inwpcbgstructure(subinwpcbgs,gtg.get_ordered_nodes())
########################################################################
# create a subInwardsPointingCodingBlockGraph of these organisms
#subinwpCBG = InwardsPointingCodingBlockGraph()
#for (pacbpkey,nodeQ,nodeS), pacbporf in inwpCBG.pacbps.iteritems():
# (orgQ,orfQid),(orgS,orfSid) = nodeQ,nodeS
# if orgQ not in gtg.get_nodes(): continue
# if orgS not in gtg.get_nodes(): continue
# subinwpCBG.add_node(nodeQ)
# subinwpCBG.add_node(nodeS)
# subinwpCBG.add_edge(nodeQ,nodeS,wt=pacbporf.bitscore)
# subinwpCBG.pacbps[(pacbpkey,nodeQ,nodeS)] = pacbporf
# detect syntenic genes in this subinwpcbgs
syntenic_inwpcbgs = assign_syntenic_inwpcbgs(subinwpcbgs)
for syntinwpcbg in syntenic_inwpcbgs:
syntenic_subinwpcbgs.append(syntinwpcbg)
####################################################################
if verbose:
print "SYNTENIC!!", syntinwpcbg, syntinwpcbg.get_ordered_nodes()
for subCBG in subinwpcbgs:
print "syntenic in:", subCBG, subCBG.get_ordered_nodes()
####################################################################
if not syntenic_subinwpcbgs:
return False
# cleanup all inwpCBGs from the syntenic subInwpCBGs
syntenic_pacbpkeys = []
for syntinwpcbg in syntenic_subinwpcbgs:
node_set = syntinwpcbg.node_set()
for inwpCBG in inwpcbgs:
if not node_set.difference(inwpCBG.node_set()):
for pacbpkey in inwpCBG.pacbps.keys():
if pacbpkey not in syntenic_pacbpkeys:
syntenic_pacbpkeys.append(pacbpkey)
# place all syntenic_pacbpkeys and PacbPORFs in the syntenicPCG
# and, at the same time, remove from the main PCG
syntenicPCG = PacbpCollectionGraph(crossdata={},blastmatrix=PCG._blastmatrix)
for key in syntenic_pacbpkeys:
(pacbpkey,nodeQ,nodeS) = key
pacbporf = PCG.pacbps[key]
# add to syntenicPCG
syntenicPCG.add_node(nodeQ)
syntenicPCG.add_node(nodeS)
syntenicPCG.add_edge(nodeQ,nodeS,wt=pacbporf.bitscore)
syntenicPCG.pacbps[(pacbpkey,nodeQ,nodeS)] = pacbporf
# remove from main PCG
_delete_pacbp(PCG,key)
# return syntenicPCG
return syntenicPCG
