def __init__(self, **args):
self.status = "New"
self['allFA'] = list()
self['analyseFA'] = AnalyseFA()
self['compareFA'] = CompareFA()
for k, v in args.items():
self[k] = v
def compareRiceAffymetrixReleases(projectDir):
"""
This function compare the properties of 10 release of Affymetrix annotations for a Rice array.
"""
projectName = "Affymetrix"
organism = "rice"
#Read rice microarray target sequence to define the set of gene products
fileName = "%s/ReferenceSet/%s.fasta" % (projectDir, organism)
refSet = RefSet(organism, fileName, refType="Fasta")
#Read GO ontoloy
fileName = "%s/OBO/go_daily-termdb.obo-xml" % (projectDir)
G = readGOoboXML(fileName, force=False)
#Read 11 release of Affymetrix Functional annotations
release = arange(20, 32)
Affy = dict()
for r in release:
FA = FuncAnnot(str(r), refSet, G, organism=organism)
fileName = "%s/Annotation/Affy_%s.na%d.annot.csv" % (projectDir,
organism, r)
FA.read(fileName, fileType="AFFY")
Affy[r] = FA
#Analyse Functional annotations
analyseFA = AnalyseFA()
batchList = [
"obsolete", "unconnected", "removeUnconnected", "coverage", "richness"
]
batchExecute(batchList, analyseFA, [Affy[r] for r in release])
#Plot statistics of Functional annotations
outDir = "%s/Graph/%s" % (projectDir, organism)
createDir(outDir)
plotFA = PlotFA(xlabel="Affymetrix Release number",
outDir=outDir,
name=projectName,
organism=organism,
ext="pdf")
batchExecute(batchList,
plotFA, [Affy[r] for r in release],
doGrid=True,
lloc="upper right")
#Compare release 20 and 31
compareFA = CompareFA()
batchList = ["venn", "funcSim"]
batchExecute(batchList, compareFA, [Affy[20], Affy[31]])
#Plot statistics of the comparison
batchList = ["venn", "funcSymSim"]
batchExecute(batchList,
plotFA,
compareFA, [Affy[20], Affy[31]],
doGrid=True,
tit="")
#Find the worse semantic similarity between 20 and 31
worseFunctionalSimilarity(projectDir, organism, Affy[20], Affy[31])
#Study the evolution of Glutamine Synthetase (GS) annotations
GSAnnotations(projectDir, organism, G, Affy)
def compareEvidence(projectDir):
"""
This function compare electronically infered and manually curated annotations to experimental annotations
"""
projectName = "EvidenceCode"
organism = "allSpecies"
refSet = RefSet(organism)
allOrg = [
"Arabidopsis_thaliana", "Drosophila_melanogaster",
"Mycobacterium_tuberculosis_ATCC_25618", "Schizosaccharomyces_pombe",
"Bos_taurus", "Escherichia_coli_ATCC_27325",
"Mycobacterium_tuberculosis_Oshkosh", "Caenorhabditis_elegans",
"Escherichia_coli_MG1655", "Oryza_sativa", "Synechocystis_sp",
"Candida_albicans_SC5314", "Gallus_gallus",
"Pseudomonas_fluorescens_Pf-5", "Danio_rerio", "Homo_sapiens",
"Rattus_norvegicus"
]
for refOrg in allOrg:
#Define the set of gene products
fileName = "%s/EvidenceCode/%s/two_experimental_evidence.goa" % (
projectDir, refOrg)
refSet.add(fileName, refType="GAF")
#Read GO ontoloy
fileName = "%s/OBO/go_daily-termdb.obo-xml" % (projectDir)
G = readGOoboXML(fileName, force=False)
#Read all annotations
fileType = "GAF"
evidenceCodes = ["EXP2", "IC", "TAS", "ISS", "NAS", "IEA"]
allFA = dict()
#-----------------------------------------------
#Read Functional annotations obtained by experiments
pipeName = "EXP2"
EXP2 = FuncAnnot(pipeName, refSet, G, organism=organism)
for refOrg in allOrg:
fileName = "%s/EvidenceCode/%s/two_experimental_evidence.goa" % (
projectDir, refOrg)
FA = FuncAnnot(pipeName, refSet, G, organism=refOrg)
FA.read(fileName, fileType=fileType)
EXP2.add(FA)
allFA[pipeName] = EXP2
#-----------------------------------------------
#Read Functional annotations obtained by human curation
for pipeName in ["IC", "TAS", "ISS", "NAS"]:
EV = FuncAnnot(pipeName, refSet, G, organism=organism)
for refOrg in allOrg:
fileName = "%s/EvidenceCode/%s/%s.goa" % (projectDir, refOrg,
pipeName)
if not os.path.exists(fileName):
continue
FA = FuncAnnot(pipeName, refSet, G, organism=refOrg)
FA.read(fileName, fileType=fileType)
EV.add(FA)
allFA[pipeName] = EV
#Merge FAs Assigned by Human Curator
FA = FuncAnnot("AHC", refSet, G, organism=organism)
#for evidence in ["IC", "TAS", "ISS", "NAS"]:
for evidence in ["IC", "ISS", "NAS"]:
FA.add(allFA[evidence])
allFA["AHC"] = FA
#-----------------------------------------------
#Read Functional annotations obtained without human curation
for pipeName in ["IEA"]:
EV = FuncAnnot(pipeName, refSet, G, organism=organism)
for refOrg in allOrg:
fileName = "%s/EvidenceCode/%s/%s.goa" % (projectDir, refOrg,
pipeName)
if not os.path.exists(fileName):
continue
FA = FuncAnnot(pipeName, refSet, G, organism=refOrg)
FA.read(fileName, fileType=fileType)
EV.add(FA)
allFA[pipeName] = EV
#-----------------------------------------------
listFA = ["EXP2", "AHC", "IEA"]
#Analyse Functional annotations
analyseFA = AnalyseFA()
batchList = [
"obsolete", "unconnected", "removeUnconnected", "coverage", "richness",
"numberAnnot", "coherence", "redundancy", "removeRedundancy",
"compactness", "specificity", "informationContent"
]
batchExecute(batchList, analyseFA,
[allFA[evidence] for evidence in listFA])
#Plot statistics of Functional annotations
outDir = "%s/Graph/%s" % (projectDir, organism)
createDir(outDir)
plotFA = PlotFA(xlabel="Evidence Codes",
outDir=outDir,
name=projectName,
organism=organism)
batchExecute(batchList,
plotFA, [allFA[evidence] for evidence in listFA],
doGrid=True)
batchList = ["coherenceHisto2D", "numberAnnotHisto2D"]
batchExecute(batchList,
plotFA, [allFA[evidence] for evidence in listFA],
doGrid=True)
#Compare Functional annotations
compareFA = CompareFA()
batchList = ["venn", "funcSim"]
batchExecute(batchList, compareFA,
[allFA[evidence] for evidence in listFA])
batchList = ["recall", "precision"]
batchExecute(batchList, compareFA,
[allFA[evidence] for evidence in listFA])
#Plot statistics of the comparison between Functional annotations
batchList = ["venn", "funcSymSim"]
batchExecute(batchList, plotFA, compareFA,
[allFA[evidence] for evidence in listFA])
batchList = ["recall", "precision"]
batchExecute(batchList, plotFA, compareFA,
[allFA[evidence] for evidence in listFA])
#-----------------------------------------------
#Export statistics to Excel
outDir = "%s/Export/%s" % (projectDir, organism)
createDir(outDir)
exportList = [
"unconnected", "coverage", "richness", "numberAnnot", "coherence",
"compactness", "specificity", "informationContent", "redundancy"
]
reportFA = ReportFA(outDir=outDir, name=projectName, organism=organism)
reportFA.printStatistics([allFA[evidence] for evidence in listFA],
exportList)
reportFA.saveStatistics([allFA[evidence] for evidence in listFA],
exportList)
#-----------------------------------------------
# Invididual contributions of evidence codes
contribution = dict()
for ec in ["IC", "TAS", "ISS", "NAS", "IEA"]:
contribution[ec] = set([
(gp, go) for aspect in
["cellular_component", "molecular_function", "biological_process"]
for gp in allFA[ec].GPtoGO[aspect]
for go in allFA[ec].GPtoGO[aspect][gp]
])
total_Annotation = sum(
[len(contribution[ec]) for ec in ["IC", "TAS", "ISS", "NAS"]])
for ec in ["IC", "TAS", "ISS", "NAS"]:
print "%.02f %% of the annotations are supported by %s" % (
100. * len(contribution[ec]) / total_Annotation, ec)
batchList = ["recall", "precision"]
batchExecute(batchList, compareFA, [
allFA[evidence]
for evidence in ["EXP2", "ISS", "TAS", "NAS", "IC", "AHC", "IEA"]
])
#-----------------------------------------------
reference = "EXP2"
#plotEvidence=["AHC", "IEA"]
plotEvidence = ["ISS", "TAS", "NAS", "IC", "AHC", "IEA"]
evidenceMarker = dict(zip(plotEvidence, ['s', 'd', 'D', '*', 'p', 'h']))
evidenceSize = dict(zip(plotEvidence, [8, 8, 8, 8, 15, 15]))
aspectColor = dict(zip(allAspect, ["blue", "green", "red", "cyan"]))
fig = figure(figsize=(8, 8))
for evidence in plotEvidence:
for aspect in allAspect:
if aspect == "All_aspects_of_GO":
continue
allX = compareFA['recall'][aspect][(evidence, reference)].values()
allY = compareFA['precision'][aspect][(evidence,
reference)].values()
meanX = mean(allX)
errX = std(allX) / sqrt(len(allX))
meanY = mean(allY)
errY = std(allY) / sqrt(len(allY))
errorbar(meanX,
meanY,
xerr=errX,
yerr=errY,
alpha=0.9,
hold=True,
mfc=aspectColor[aspect],
ecolor=aspectColor[aspect],
marker=evidenceMarker[evidence],
ms=evidenceSize[evidence])
xlabel("Verspoor Hierarchical Recall")
ylabel("Verspoor Hierarchical Precision")
allMarker = ['o', 'o', 'o', 's', 'd', 'D', '*', 'p', 'h']
allColor = [
"green", "red", "cyan", "white", "white", "white", "white", "white",
"white"
]
allLabel = [
aspect.replace("_", " ") for aspect in allAspect
if not aspect == "All_aspects_of_GO"
]
allLabel.extend(plotEvidence)
foo = [
Line2D(arange(5), arange(5), ls='-', marker=m, color=c, label=l)
for m, c, l in zip(allMarker, allColor, allLabel)
]
leg = legend(foo, allLabel, loc="upper left", numpoints=1)
leg.legendPatch.set_alpha(0.5)
grid()
outDir = "%s/Graph/%s" % (projectDir, organism)
createDir(outDir)
figName = "%s/PrecisionVSRecall.png" % outDir
savefig(figName)
def compareWheatPipelines(projectDir):
"""
Add some comments here
"""
organism = "wheat"
#Read rice microarray target sequence to define the set of gene products
fileName = "%s/ReferenceSet/%s.fasta" % (projectDir, organism)
refSet = RefSet(organism=organism, fileName=fileName, refType="Fasta")
#Read GO ontoloy
fileName = "%s/OBO/go_daily-termdb.obo-xml" % (projectDir)
G = readGOoboXML(fileName, force=False)
projectName = "MATT"
#Read Functional annotations
allFileName = list()
allFileName.append(
"%s/Annotation/blast2goPaths_fin_aracyc_%s_unionBest.tab" %
(projectDir, organism))
allFileName.append("%s/Annotation/pfam2goPaths2_%s_unionBest.tab" %
(projectDir, organism))
allFileName.append(
"%s/Annotation/pfam2goPaths2_%s_unionBest___blast2goPaths_fin_aracyc_%s_unionBest_merged.tab"
% (projectDir, organism, organism))
allFileName.append("%s/Annotation/COPSA_%s.tab" % (projectDir, organism))
allFileName.append("%s/Annotation/Affy_%s.annot.csv" %
(projectDir, organism))
allFileName.append("%s/Annotation/B2G_%s.annot" % (projectDir, organism))
allPipeName = ["Blast", "Pfam", "Merge", "COPSA", "AFFY", "B2G"]
allFileType = ["GP2GO", "GP2GO", "GP2GO", "GP2GO", "AFFY", "B2G"]
pipeline = dict()
for pipeName, fileName, fileType in zip(allPipeName, allFileName,
allFileType):
FA = FuncAnnot(pipeName, refSet, G, organism=organism)
FA.read(fileName, fileType=fileType)
pipeline[pipeName] = FA
#Analyse Functional annotations
analyseFA = AnalyseFA()
batchList = [
"obsolete", "unconnected", "removeUnconnected", "coverage", "richness",
"numberAnnot", "coherence", "redundancy", "compactness", "specificity",
"informationContent"
]
batchExecute(batchList, analyseFA,
[pipeline[pipeName] for pipeName in allPipeName])
#Plot statistics of Functional annotations
outDir = "%s/Graph/%s" % (projectDir, organism)
createDir(outDir)
plotFA = PlotFA(xlabel="Annotation pipelines",
outDir=outDir,
name=projectName,
organism=organism)
batchExecute(batchList,
plotFA, [pipeline[pipeName] for pipeName in allPipeName],
doGrid=True)
batchList = ["coherenceHisto2D", "numberAnnotHisto2D"]
batchExecute(batchList,
plotFA, [pipeline[pipeName] for pipeName in allPipeName],
doGrid=True)
#Compare Functional annotations
compareFA = CompareFA()
batchList = ["venn", "funcSim"]
batchExecute(batchList, compareFA,
[pipeline[pipeName] for pipeName in ["COPSA", "AFFY", "B2G"]])
#Plot statistics of the comparison between Functional annotations
batchList = ["venn", "funcSymSim"]
batchExecute(batchList, plotFA, compareFA,
[pipeline[pipeName] for pipeName in ["COPSA", "AFFY", "B2G"]])
#-----------------------------------------------
#Export statistics to Excel
outDir = "%s/Export/%s" % (projectDir, organism)
createDir(outDir)
exportList = [
"unconnected", "coverage", "numberAnnot", "richness", "coherence",
"compactness", "specificity", "informationContent", "redundancy"
]
reportFA = ReportFA(outDir=outDir, name=projectName, organism=organism)
reportFA.printStatistics([pipeline[pipeName] for pipeName in allPipeName],
exportList)
reportFA.saveStatistics([pipeline[pipeName] for pipeName in allPipeName],
exportList)
def compareBovinePipelines(projectDir):
"""
This function compare the properties of 3 functional annotations for a Bovine array.
"""
projectName = "bovinePipeline"
organism = "bovine"
logger.info(
"◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦"
)
logger.info(
"This function compare the properties of 3 functional annotations for a Bovine array."
)
logger.info(
"◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦"
)
logger.info("name of the project : %s " % projectName)
#Read bovine microarray probe set to define the set of gene products
fileName = "%s/ReferenceSet/%s.fasta" % (projectDir, organism)
refSet = RefSet(organism=organism, fileName=fileName, refType="Fasta")
#Read GO ontoloy
fileName = "%s/OBO/go_daily-termdb.obo-xml" % (projectDir)
G = readGOoboXML(fileName, force=False)
#Read Functional annotations
allFileName = list()
allFileName.append("%s/Annotation/Affy_%s.na31.annot.csv" %
(projectDir, organism))
allFileName.append("%s/Annotation/B2G_%s.annot" % (projectDir, organism))
allFileName.append("%s/Annotation/AID_%s.txt" % (projectDir, organism))
allPipeName = ["AFFY", "B2G", "AID"]
allFileType = allPipeName
pipeline = dict()
for pipeName, fileName, fileType in zip(allPipeName, allFileName,
allFileType):
FA = FuncAnnot(pipeName, refSet, G, organism=organism)
FA.read(fileName, fileType=fileType)
pipeline[pipeName] = FA
#-----------------------------------------------
#Analyse Functional annotations
analyseFA = AnalyseFA()
#batchList=["obsolete", "unconnected", "removeUnconnected", "coverage", "richness", "numberAnnot", "coherence", "redundancy", "compactness", "specificity", "informationContent"]
batchList = [
"obsolete", "unconnected", "removeUnconnected", "coverage", "richness",
"numberAnnot", "redundancy", "specificity", "informationContent"
]
batchExecute(batchList, analyseFA,
[pipeline[pipeName] for pipeName in allPipeName])
#How big are the largest annotation sets ?
analyseFA.largestSet([pipeline[pipeName] for pipeName in allPipeName])
logger.info("The largest sets of annotations are :")
for pipeName in allPipeName:
FA = pipeline[pipeName]
logger.info("\t%d for %s" %
(FA['largestSet']['All_aspects_of_GO'], FA.name))
#Plot statistics of Functional annotations
outDir = "%s/Graph/%s" % (projectDir, organism)
createDir(outDir)
plotFA = PlotFA(xlabel="Annotation pipelines",
outDir=outDir,
name=projectName,
organism=organism,
ext="png")
batchExecute(batchList,
plotFA, [pipeline[pipeName] for pipeName in allPipeName],
doGrid=True)
#batchList=["coherenceHisto2D", "numberAnnotHisto2D"]
batchList = ["numberAnnotHisto2D"]
batchExecute(batchList,
plotFA, [pipeline[pipeName] for pipeName in allPipeName],
doGrid=True,
tit="")
#-----------------------------------------------
#Compare Functional annotations
compareFA = CompareFA()
batchList = ["venn", "funcSim"]
batchExecute(batchList, compareFA,
[pipeline[pipeName] for pipeName in allPipeName])
#Plot statistics of the comparison between Functional annotations
batchList = ["venn", "funcSymSim"]
batchExecute(batchList,
plotFA,
compareFA, [pipeline[pipeName] for pipeName in allPipeName],
tit="")
#-----------------------------------------------
#Export statistics to Excel
outDir = "%s/Export/%s" % (projectDir, organism)
createDir(outDir)
#exportList=["unconnected", "coverage", "richness", "numberAnnot", "coherence", "compactness", "specificity", "informationContent", "redundancy"]
exportList = [
"unconnected", "coverage", "richness", "numberAnnot", "specificity",
"informationContent", "redundancy"
]
reportFA = ReportFA(outDir=outDir, name=projectName, organism=organism)
reportFA.printStatistics([pipeline[pipeName] for pipeName in allPipeName],
exportList)
reportFA.saveStatistics([pipeline[pipeName] for pipeName in allPipeName],
exportList)
logger.info(
"◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦◦"
)
logger.info("")