class OWGOEnrichmentAnalysis(widget.OWWidget):
name = "GO Browser"
description = "Enrichment analysis for Gene Ontology terms."
icon = "../widgets/icons/GOBrowser.svg"
priority = 2020
inputs = [("Cluster Data", Orange.data.Table,
"setDataset", widget.Single + widget.Default),
("Reference Data", Orange.data.Table,
"setReferenceDataset")]
outputs = [("Data on Selected Genes", Orange.data.Table),
("Data on Unselected Genes", Orange.data.Table),
("Data on Unknown Genes", Orange.data.Table),
("Enrichment Report", Orange.data.Table)]
settingsHandler = settings.DomainContextHandler()
annotationIndex = settings.ContextSetting(0)
geneAttrIndex = settings.ContextSetting(0)
useAttrNames = settings.ContextSetting(False)
geneMatcherSettings = settings.Setting([True, False, False, False])
useReferenceDataset = settings.Setting(False)
aspectIndex = settings.Setting(0)
useEvidenceType = settings.Setting(
{et: True for et in go.evidenceTypesOrdered})
filterByNumOfInstances = settings.Setting(False)
minNumOfInstances = settings.Setting(1)
filterByPValue = settings.Setting(True)
maxPValue = settings.Setting(0.2)
filterByPValue_nofdr = settings.Setting(False)
maxPValue_nofdr = settings.Setting(0.01)
probFunc = settings.Setting(0)
selectionDirectAnnotation = settings.Setting(0)
selectionDisjoint = settings.Setting(0)
selectionAddTermAsClass = settings.Setting(0)
Ready, Initializing, Running = 0, 1, 2
def __init__(self, parent=None):
super().__init__(self, parent)
self.clusterDataset = None
self.referenceDataset = None
self.ontology = None
self.annotations = None
self.loadedAnnotationCode = "---"
self.treeStructRootKey = None
self.probFunctions = [stats.Binomial(), stats.Hypergeometric()]
self.selectedTerms = []
self.selectionChanging = 0
self.__state = OWGOEnrichmentAnalysis.Initializing
self.annotationCodes = []
#############
## GUI
#############
self.tabs = gui.tabWidget(self.controlArea)
## Input tab
self.inputTab = gui.createTabPage(self.tabs, "Input")
box = gui.widgetBox(self.inputTab, "Info")
self.infoLabel = gui.widgetLabel(box, "No data on input\n")
gui.button(box, self, "Ontology/Annotation Info",
callback=self.ShowInfo,
tooltip="Show information on loaded ontology and annotations")
box = gui.widgetBox(self.inputTab, "Organism")
self.annotationComboBox = gui.comboBox(
box, self, "annotationIndex", items=self.annotationCodes,
callback=self._updateEnrichment, tooltip="Select organism")
genebox = gui.widgetBox(self.inputTab, "Gene Names")
self.geneAttrIndexCombo = gui.comboBox(
genebox, self, "geneAttrIndex", callback=self._updateEnrichment,
tooltip="Use this attribute to extract gene names from input data")
self.geneAttrIndexCombo.setDisabled(self.useAttrNames)
cb = gui.checkBox(genebox, self, "useAttrNames", "Use column names",
tooltip="Use column names for gene names",
callback=self._updateEnrichment)
cb.toggled[bool].connect(self.geneAttrIndexCombo.setDisabled)
gui.button(genebox, self, "Gene matcher settings",
callback=self.UpdateGeneMatcher,
tooltip="Open gene matching settings dialog")
self.referenceRadioBox = gui.radioButtonsInBox(
self.inputTab, self, "useReferenceDataset",
["Entire genome", "Reference set (input)"],
tooltips=["Use entire genome for reference",
"Use genes from Referece Examples input signal as reference"],
box="Reference", callback=self._updateEnrichment)
self.referenceRadioBox.buttons[1].setDisabled(True)
gui.radioButtonsInBox(
self.inputTab, self, "aspectIndex",
["Biological process", "Cellular component", "Molecular function"],
box="Aspect", callback=self._updateEnrichment)
## Filter tab
self.filterTab = gui.createTabPage(self.tabs, "Filter")
box = gui.widgetBox(self.filterTab, "Filter GO Term Nodes")
gui.checkBox(box, self, "filterByNumOfInstances", "Genes",
callback=self.FilterAndDisplayGraph,
tooltip="Filter by number of input genes mapped to a term")
ibox = gui.indentedBox(box)
gui.spin(ibox, self, 'minNumOfInstances', 1, 100,
step=1, label='#:', labelWidth=15,
callback=self.FilterAndDisplayGraph,
callbackOnReturn=True,
tooltip="Min. number of input genes mapped to a term")
gui.checkBox(box, self, "filterByPValue_nofdr", "p-value",
callback=self.FilterAndDisplayGraph,
tooltip="Filter by term p-value")
gui.doubleSpin(gui.indentedBox(box), self, 'maxPValue_nofdr', 1e-8, 1,
step=1e-8, label='p:', labelWidth=15,
callback=self.FilterAndDisplayGraph,
callbackOnReturn=True,
tooltip="Max term p-value")
#use filterByPValue for FDR, as it was the default in prior versions
gui.checkBox(box, self, "filterByPValue", "FDR",
callback=self.FilterAndDisplayGraph,
tooltip="Filter by term FDR")
gui.doubleSpin(gui.indentedBox(box), self, 'maxPValue', 1e-8, 1,
step=1e-8, label='p:', labelWidth=15,
callback=self.FilterAndDisplayGraph,
callbackOnReturn=True,
tooltip="Max term p-value")
box = gui.widgetBox(box, "Significance test")
gui.radioButtonsInBox(box, self, "probFunc", ["Binomial", "Hypergeometric"],
tooltips=["Use binomial distribution test",
"Use hypergeometric distribution test"],
callback=self._updateEnrichment)
box = gui.widgetBox(self.filterTab, "Evidence codes in annotation",
addSpace=True)
self.evidenceCheckBoxDict = {}
for etype in go.evidenceTypesOrdered:
ecb = QCheckBox(
etype, toolTip=go.evidenceTypes[etype],
checked=self.useEvidenceType[etype])
ecb.toggled.connect(self.__on_evidenceChanged)
box.layout().addWidget(ecb)
self.evidenceCheckBoxDict[etype] = ecb
## Select tab
self.selectTab = gui.createTabPage(self.tabs, "Select")
box = gui.radioButtonsInBox(
self.selectTab, self, "selectionDirectAnnotation",
["Directly or Indirectly", "Directly"],
box="Annotated genes",
callback=self.ExampleSelection)
box = gui.widgetBox(self.selectTab, "Output", addSpace=True)
gui.radioButtonsInBox(
box, self, "selectionDisjoint",
btnLabels=["All selected genes",
"Term-specific genes",
"Common term genes"],
tooltips=["Outputs genes annotated to all selected GO terms",
"Outputs genes that appear in only one of selected GO terms",
"Outputs genes common to all selected GO terms"],
callback=[self.ExampleSelection,
self.UpdateAddClassButton])
self.addClassCB = gui.checkBox(
box, self, "selectionAddTermAsClass", "Add GO Term as class",
callback=self.ExampleSelection)
# ListView for DAG, and table for significant GOIDs
self.DAGcolumns = ['GO term', 'Cluster', 'Reference', 'p-value',
'FDR', 'Genes', 'Enrichment']
self.splitter = QSplitter(Qt.Vertical, self.mainArea)
self.mainArea.layout().addWidget(self.splitter)
# list view
self.listView = GOTreeWidget(self.splitter)
self.listView.setSelectionMode(QTreeView.ExtendedSelection)
self.listView.setAllColumnsShowFocus(1)
self.listView.setColumnCount(len(self.DAGcolumns))
self.listView.setHeaderLabels(self.DAGcolumns)
self.listView.header().setSectionsClickable(True)
self.listView.header().setSortIndicatorShown(True)
self.listView.setSortingEnabled(True)
self.listView.setItemDelegateForColumn(
6, EnrichmentColumnItemDelegate(self))
self.listView.setRootIsDecorated(True)
self.listView.itemSelectionChanged.connect(self.ViewSelectionChanged)
# table of significant GO terms
self.sigTerms = QTreeWidget(self.splitter)
self.sigTerms.setColumnCount(len(self.DAGcolumns))
self.sigTerms.setHeaderLabels(self.DAGcolumns)
self.sigTerms.setSortingEnabled(True)
self.sigTerms.setSelectionMode(QTreeView.ExtendedSelection)
self.sigTerms.setItemDelegateForColumn(
6, EnrichmentColumnItemDelegate(self))
self.sigTerms.itemSelectionChanged.connect(self.TableSelectionChanged)
self.sigTableTermsSorted = []
self.graph = {}
self.inputTab.layout().addStretch(1)
self.filterTab.layout().addStretch(1)
self.selectTab.layout().addStretch(1)
self.setBlocking(True)
self._executor = ThreadExecutor()
self._init = EnsureDownloaded(
[(taxonomy.Taxonomy.DOMAIN, taxonomy.Taxonomy.FILENAME),
("GO", "taxonomy.pickle")]
)
self._init.finished.connect(self.__initialize_finish)
self._executor.submit(self._init)
def sizeHint(self):
return QSize(1000, 700)
def __initialize_finish(self):
self.setBlocking(False)
try:
self.annotationFiles = listAvailable()
except ConnectTimeout:
self.error(2, "Internet connection error, unable to load data. " + \
"Check connection and create a new GO Browser widget.")
self.filterTab.setEnabled(False)
self.inputTab.setEnabled(False)
self.selectTab.setEnabled(False)
self.listView.setEnabled(False)
self.sigTerms.setEnabled(False)
else:
self.annotationCodes = sorted(self.annotationFiles.keys())
self.annotationComboBox.clear()
self.annotationComboBox.addItems(self.annotationCodes)
self.annotationComboBox.setCurrentIndex(self.annotationIndex)
self.__state = OWGOEnrichmentAnalysis.Ready
def __on_evidenceChanged(self):
for etype, cb in self.evidenceCheckBoxDict.items():
self.useEvidenceType[etype] = cb.isChecked()
self._updateEnrichment()
def UpdateGeneMatcher(self):
"""Open the Gene matcher settings dialog."""
dialog = GeneMatcherDialog(self, defaults=self.geneMatcherSettings, modal=True)
if dialog.exec_() != QDialog.Rejected:
self.geneMatcherSettings = [getattr(dialog, item[0]) for item in dialog.items]
if self.annotations:
self.SetGeneMatcher()
self._updateEnrichment()
def clear(self):
self.infoLabel.setText("No data on input\n")
self.warning(0)
self.warning(1)
self.geneAttrIndexCombo.clear()
self.ClearGraph()
self.send("Data on Selected Genes", None)
self.send("Data on Unselected Genes", None)
self.send("Data on Unknown Genes", None)
self.send("Enrichment Report", None)
def setDataset(self, data=None):
if self.__state == OWGOEnrichmentAnalysis.Initializing:
self.__initialize_finish()
self.closeContext()
self.clear()
self.clusterDataset = data
if data is not None:
domain = data.domain
allvars = domain.variables + domain.metas
self.candidateGeneAttrs = [var for var in allvars if isstring(var)]
self.geneAttrIndexCombo.clear()
for var in self.candidateGeneAttrs:
self.geneAttrIndexCombo.addItem(*gui.attributeItem(var))
taxid = data_hints.get_hint(data, "taxid", "")
code = None
try:
code = go.from_taxid(taxid)
except KeyError:
pass
except Exception as ex:
print(ex)
if code is not None:
filename = "gene_association.%s.tar.gz" % code
if filename in self.annotationFiles.values():
self.annotationIndex = \
[i for i, name in enumerate(self.annotationCodes) \
if self.annotationFiles[name] == filename].pop()
self.useAttrNames = data_hints.get_hint(data, "genesinrows",
self.useAttrNames)
self.openContext(data)
self.geneAttrIndex = min(self.geneAttrIndex,
len(self.candidateGeneAttrs) - 1)
if len(self.candidateGeneAttrs) == 0:
self.useAttrNames = True
self.geneAttrIndex = -1
elif self.geneAttrIndex < len(self.candidateGeneAttrs):
self.geneAttrIndex = len(self.candidateGeneAttrs) - 1
self._updateEnrichment()
def setReferenceDataset(self, data=None):
self.referenceDataset = data
self.referenceRadioBox.buttons[1].setDisabled(not bool(data))
self.referenceRadioBox.buttons[1].setText("Reference set")
if self.clusterDataset is not None and self.useReferenceDataset:
self.useReferenceDataset = 0 if not data else 1
graph = self.Enrichment()
self.SetGraph(graph)
elif self.clusterDataset:
self.__updateReferenceSetButton()
def handleNewSignals(self):
super().handleNewSignals()
def _updateEnrichment(self):
if self.clusterDataset is not None and \
self.__state == OWGOEnrichmentAnalysis.Ready:
pb = gui.ProgressBar(self, 100)
self.Load(pb=pb)
graph = self.Enrichment(pb=pb)
self.FilterUnknownGenes()
self.SetGraph(graph)
def __updateReferenceSetButton(self):
allgenes, refgenes = None, None
if self.referenceDataset:
try:
allgenes = self.genesFromTable(self.referenceDataset)
except Exception:
allgenes = []
refgenes, unknown = self.FilterAnnotatedGenes(allgenes)
self.referenceRadioBox.buttons[1].setDisabled(not bool(allgenes))
self.referenceRadioBox.buttons[1].setText("Reference set " + ("(%i genes, %i matched)" % (len(allgenes), len(refgenes)) if allgenes and refgenes else ""))
def genesFromTable(self, data):
if self.useAttrNames:
genes = [v.name for v in data.domain.variables]
else:
attr = self.candidateGeneAttrs[min(self.geneAttrIndex, len(self.candidateGeneAttrs) - 1)]
genes = [str(ex[attr]) for ex in data if not numpy.isnan(ex[attr])]
if any("," in gene for gene in genes):
self.information(0, "Separators detected in gene names. Assuming multiple genes per example.")
genes = reduce(operator.iadd, (genes.split(",") for genes in genes), [])
return genes
def FilterAnnotatedGenes(self, genes):
matchedgenes = self.annotations.get_gene_names_translator(genes).values()
return matchedgenes, [gene for gene in genes if gene not in matchedgenes]
def FilterUnknownGenes(self):
if not self.useAttrNames and self.candidateGeneAttrs:
geneAttr = self.candidateGeneAttrs[min(self.geneAttrIndex, len(self.candidateGeneAttrs)-1)]
indices = []
for i, ex in enumerate(self.clusterDataset):
if not any(self.annotations.genematcher.match(n.strip()) for n in str(ex[geneAttr]).split(",")):
indices.append(i)
if indices:
data = self.clusterDataset[indices]
else:
data = None
self.send("Data on Unknown Genes", data)
else:
self.send("Data on Unknown Genes", None)
def Load(self, pb=None):
if self.__state == OWGOEnrichmentAnalysis.Ready:
go_files, tax_files = serverfiles.listfiles("GO"), serverfiles.listfiles("Taxonomy")
calls = []
pb, finish = (gui.ProgressBar(self, 0), True) if pb is None else (pb, False)
count = 0
if not tax_files:
calls.append(("Taxonomy", "ncbi_taxnomy.tar.gz"))
count += 1
org = self.annotationCodes[min(self.annotationIndex, len(self.annotationCodes)-1)]
if org != self.loadedAnnotationCode:
count += 1
if self.annotationFiles[org] not in go_files:
calls.append(("GO", self.annotationFiles[org]))
count += 1
if "gene_ontology_edit.obo.tar.gz" not in go_files:
calls.append(("GO", "gene_ontology_edit.obo.tar.gz"))
count += 1
if not self.ontology:
count += 1
pb.iter += count * 100
for args in calls:
serverfiles.localpath_download(*args, **dict(callback=pb.advance))
i = len(calls)
if not self.ontology:
self.ontology = go.Ontology(progress_callback=lambda value: pb.advance())
i += 1
if org != self.loadedAnnotationCode:
self.annotations = None
gc.collect() # Force run garbage collection
code = self.annotationFiles[org].split(".")[-3]
self.annotations = go.Annotations(code, genematcher=gene.GMDirect(), progress_callback=lambda value: pb.advance())
i += 1
self.loadedAnnotationCode = org
count = defaultdict(int)
geneSets = defaultdict(set)
for anno in self.annotations.annotations:
count[anno.evidence] += 1
geneSets[anno.evidence].add(anno.geneName)
for etype in go.evidenceTypesOrdered:
ecb = self.evidenceCheckBoxDict[etype]
ecb.setEnabled(bool(count[etype]))
ecb.setText(etype + ": %i annots(%i genes)" % (count[etype], len(geneSets[etype])))
if finish:
pb.finish()
def SetGeneMatcher(self):
if self.annotations:
taxid = self.annotations.taxid
matchers = []
for matcher, use in zip([gene.GMGO, gene.GMKEGG, gene.GMNCBI, gene.GMAffy], self.geneMatcherSettings):
if use:
try:
if taxid == "352472":
matchers.extend([matcher(taxid), gene.GMDicty(),
[matcher(taxid), gene.GMDicty()]])
# The reason machers are duplicated is that we want `matcher` or `GMDicty` to
# match genes by them self if possible. Only use the joint matcher if they fail.
else:
matchers.append(matcher(taxid))
except Exception as ex:
print(ex)
self.annotations.genematcher = gene.matcher(matchers)
self.annotations.genematcher.set_targets(self.annotations.gene_names)
def Enrichment(self, pb=None):
assert self.clusterDataset is not None
pb = gui.ProgressBar(self, 100) if pb is None else pb
if not self.annotations.ontology:
self.annotations.ontology = self.ontology
if isinstance(self.annotations.genematcher, gene.GMDirect):
self.SetGeneMatcher()
self.error(1)
self.warning([0, 1])
if self.useAttrNames:
clusterGenes = [v.name for v in self.clusterDataset.domain.attributes]
self.information(0)
elif 0 <= self.geneAttrIndex < len(self.candidateGeneAttrs):
geneAttr = self.candidateGeneAttrs[self.geneAttrIndex]
clusterGenes = [str(ex[geneAttr]) for ex in self.clusterDataset
if not numpy.isnan(ex[geneAttr])]
if any("," in gene for gene in clusterGenes):
self.information(0, "Separators detected in cluster gene names. Assuming multiple genes per example.")
clusterGenes = reduce(operator.iadd, (genes.split(",") for genes in clusterGenes), [])
else:
self.information(0)
else:
self.error(1, "Failed to extract gene names from input dataset!")
return {}
genesSetCount = len(set(clusterGenes))
self.clusterGenes = clusterGenes = self.annotations.get_gene_names_translator(clusterGenes).values()
self.infoLabel.setText("%i unique genes on input\n%i (%.1f%%) genes with known annotations" % (genesSetCount, len(clusterGenes), 100.0*len(clusterGenes)/genesSetCount if genesSetCount else 0.0))
referenceGenes = None
if not self.useReferenceDataset or self.referenceDataset is None:
self.information(2)
self.information(1)
referenceGenes = self.annotations.gene_names
elif self.referenceDataset is not None:
if self.useAttrNames:
referenceGenes = [v.name for v in self.referenceDataset.domain.attributes]
self.information(1)
elif geneAttr in (self.referenceDataset.domain.variables +
self.referenceDataset.domain.metas):
referenceGenes = [str(ex[geneAttr]) for ex in self.referenceDataset
if not numpy.isnan(ex[geneAttr])]
if any("," in gene for gene in clusterGenes):
self.information(1, "Separators detected in reference gene names. Assuming multiple genes per example.")
referenceGenes = reduce(operator.iadd, (genes.split(",") for genes in referenceGenes), [])
else:
self.information(1)
else:
self.information(1)
referenceGenes = None
if referenceGenes is None:
referenceGenes = list(self.annotations.gene_names)
self.referenceRadioBox.buttons[1].setText("Reference set")
self.referenceRadioBox.buttons[1].setDisabled(True)
self.information(2, "Unable to extract gene names from reference dataset. Using entire genome for reference")
self.useReferenceDataset = 0
else:
refc = len(referenceGenes)
referenceGenes = self.annotations.get_gene_names_translator(referenceGenes).values()
self.referenceRadioBox.buttons[1].setText("Reference set (%i genes, %i matched)" % (refc, len(referenceGenes)))
self.referenceRadioBox.buttons[1].setDisabled(False)
self.information(2)
else:
self.useReferenceDataset = 0
if not referenceGenes:
self.error(1, "No valid reference set")
return {}
self.referenceGenes = referenceGenes
evidences = []
for etype in go.evidenceTypesOrdered:
if self.useEvidenceType[etype]:
evidences.append(etype)
aspect = ["P", "C", "F"][self.aspectIndex]
if clusterGenes:
self.terms = terms = self.annotations.get_enriched_terms(
clusterGenes, referenceGenes, evidences, aspect=aspect,
prob=self.probFunctions[self.probFunc], use_fdr=False,
progress_callback=lambda value: pb.advance())
ids = []
pvals = []
for i, d in self.terms.items():
ids.append(i)
pvals.append(d[1])
for i, fdr in zip(ids, stats.FDR(pvals)): # save FDR as the last part of the tuple
terms[i] = tuple(list(terms[i]) + [ fdr ])
else:
self.terms = terms = {}
if not self.terms:
self.warning(0, "No enriched terms found.")
else:
self.warning(0)
pb.finish()
self.treeStructDict = {}
ids = self.terms.keys()
self.treeStructRootKey = None
parents = {}
for id in ids:
parents[id] = set([term for _, term in self.ontology[id].related])
children = {}
for term in self.terms:
children[term] = set([id for id in ids if term in parents[id]])
for term in self.terms:
self.treeStructDict[term] = TreeNode(self.terms[term], children[term])
if not self.ontology[term].related and not getattr(self.ontology[term], "is_obsolete", False):
self.treeStructRootKey = term
return terms
def FilterGraph(self, graph):
if self.filterByPValue_nofdr:
graph = go.filterByPValue(graph, self.maxPValue_nofdr)
if self.filterByPValue: #FDR
graph = dict(filter(lambda item: item[1][3] <= self.maxPValue, graph.items()))
if self.filterByNumOfInstances:
graph = dict(filter(lambda item: len(item[1][0]) >= self.minNumOfInstances, graph.items()))
return graph
def FilterAndDisplayGraph(self):
if self.clusterDataset:
self.graph = self.FilterGraph(self.originalGraph)
if self.originalGraph and not self.graph:
self.warning(1, "All found terms were filtered out.")
else:
self.warning(1)
self.ClearGraph()
self.DisplayGraph()
def SetGraph(self, graph=None):
self.originalGraph = graph
if graph:
self.FilterAndDisplayGraph()
else:
self.graph = {}
self.ClearGraph()
def ClearGraph(self):
self.listView.clear()
self.listViewItems=[]
self.sigTerms.clear()
def DisplayGraph(self):
fromParentDict = {}
self.termListViewItemDict = {}
self.listViewItems = []
enrichment = lambda t: len(t[0]) / t[2] * (len(self.referenceGenes) / len(self.clusterGenes))
maxFoldEnrichment = max([enrichment(term) for term in self.graph.values()] or [1])
def addNode(term, parent, parentDisplayNode):
if (parent, term) in fromParentDict:
return
if term in self.graph:
displayNode = GOTreeWidgetItem(self.ontology[term], self.graph[term], len(self.clusterGenes), len(self.referenceGenes), maxFoldEnrichment, parentDisplayNode)
displayNode.goId = term
self.listViewItems.append(displayNode)
if term in self.termListViewItemDict:
self.termListViewItemDict[term].append(displayNode)
else:
self.termListViewItemDict[term] = [displayNode]
fromParentDict[(parent, term)] = True
parent = term
else:
displayNode = parentDisplayNode
for c in self.treeStructDict[term].children:
addNode(c, parent, displayNode)
if self.treeStructDict:
addNode(self.treeStructRootKey, None, self.listView)
terms = self.graph.items()
terms = sorted(terms, key=lambda item: item[1][1])
self.sigTableTermsSorted = [t[0] for t in terms]
self.sigTerms.clear()
for i, (t_id, (genes, p_value, refCount, fdr)) in enumerate(terms):
item = GOTreeWidgetItem(self.ontology[t_id],
(genes, p_value, refCount, fdr),
len(self.clusterGenes),
len(self.referenceGenes),
maxFoldEnrichment,
self.sigTerms)
item.goId = t_id
self.listView.expandAll()
for i in range(5):
self.listView.resizeColumnToContents(i)
self.sigTerms.resizeColumnToContents(i)
self.sigTerms.resizeColumnToContents(6)
width = min(self.listView.columnWidth(0), 350)
self.listView.setColumnWidth(0, width)
self.sigTerms.setColumnWidth(0, width)
# Create and send the enrichemnt report table.
termsDomain = Orange.data.Domain(
[], [],
# All is meta!
[Orange.data.StringVariable("GO Term Id"),
Orange.data.StringVariable("GO Term Name"),
Orange.data.ContinuousVariable("Cluster Frequency"),
Orange.data.ContinuousVariable("Genes in Cluster", number_of_decimals=0),
Orange.data.ContinuousVariable("Reference Frequency"),
Orange.data.ContinuousVariable("Genes in Reference", number_of_decimals=0),
Orange.data.ContinuousVariable("p-value"),
Orange.data.ContinuousVariable("FDR"),
Orange.data.ContinuousVariable("Enrichment"),
Orange.data.StringVariable("Genes")])
terms = [[t_id,
self.ontology[t_id].name,
len(genes) / len(self.clusterGenes),
len(genes),
r_count / len(self.referenceGenes),
r_count,
p_value,
fdr,
len(genes) / len(self.clusterGenes) * \
len(self.referenceGenes) / r_count,
",".join(genes)
]
for t_id, (genes, p_value, r_count, fdr) in terms]
if terms:
X = numpy.empty((len(terms), 0))
M = numpy.array(terms, dtype=object)
termsTable = Orange.data.Table.from_numpy(termsDomain, X, metas=M)
else:
termsTable = Orange.data.Table(termsDomain)
self.send("Enrichment Report", termsTable)
def ViewSelectionChanged(self):
if self.selectionChanging:
return
self.selectionChanging = 1
self.selectedTerms = []
selected = self.listView.selectedItems()
self.selectedTerms = list(set([lvi.term.id for lvi in selected]))
self.ExampleSelection()
self.selectionChanging = 0
def TableSelectionChanged(self):
if self.selectionChanging:
return
self.selectionChanging = 1
self.selectedTerms = []
selectedIds = set([self.sigTerms.itemFromIndex(index).goId for index in self.sigTerms.selectedIndexes()])
for i in range(self.sigTerms.topLevelItemCount()):
item = self.sigTerms.topLevelItem(i)
selected = item.goId in selectedIds
term = item.goId
if selected:
self.selectedTerms.append(term)
for lvi in self.termListViewItemDict[term]:
try:
lvi.setSelected(selected)
if selected:
lvi.setExpanded(True)
except RuntimeError: # Underlying C/C++ object deleted
pass
self.ExampleSelection()
self.selectionChanging = 0
def UpdateAddClassButton(self):
self.addClassCB.setEnabled(self.selectionDisjoint == 1)
def ExampleSelection(self):
self.commit()
def commit(self):
if self.clusterDataset is None:
return
terms = set(self.selectedTerms)
genes = reduce(operator.ior,
(set(self.graph[term][0]) for term in terms), set())
evidences = []
for etype in go.evidenceTypesOrdered:
if self.useEvidenceType[etype]:
# if getattr(self, "useEvidence" + etype):
evidences.append(etype)
allTerms = self.annotations.get_annotated_terms(
genes, direct_annotation_only=self.selectionDirectAnnotation,
evidence_codes=evidences)
if self.selectionDisjoint > 0:
count = defaultdict(int)
for term in self.selectedTerms:
for g in allTerms.get(term, []):
count[g] += 1
ccount = 1 if self.selectionDisjoint == 1 else len(self.selectedTerms)
selectedGenes = [gene for gene, c in count.items()
if c == ccount and gene in genes]
else:
selectedGenes = reduce(
operator.ior,
(set(allTerms.get(term, [])) for term in self.selectedTerms),
set())
if self.useAttrNames:
vars = [self.clusterDataset.domain[gene]
for gene in set(selectedGenes)]
domain = Orange.data.Domain(
vars, self.clusterDataset.domain.class_vars,
self.clusterDataset.domain.metas)
newdata = self.clusterDataset.from_table(domain, self.clusterDataset)
self.send("Data on Selected Genes", newdata)
self.send("Data on Unselected Genes", None)
elif self.candidateGeneAttrs:
selectedExamples = []
unselectedExamples = []
geneAttr = self.candidateGeneAttrs[min(self.geneAttrIndex, len(self.candidateGeneAttrs)-1)]
if self.selectionDisjoint == 1:
goVar = Orange.data.DiscreteVariable(
"GO Term", values=list(self.selectedTerms))
newDomain = Orange.data.Domain(
self.clusterDataset.domain.variables, goVar,
self.clusterDataset.domain.metas)
goColumn = []
for i, ex in enumerate(self.clusterDataset):
if not numpy.isnan(ex[geneAttr]) and any(gene in selectedGenes for gene in str(ex[geneAttr]).split(",")):
if self.selectionDisjoint == 1 and self.selectionAddTermAsClass:
terms = filter(lambda term: any(gene in self.graph[term][0] for gene in str(ex[geneAttr]).split(",")) , self.selectedTerms)
term = sorted(terms)[0]
goColumn.append(goVar.values.index(term))
selectedExamples.append(i)
else:
unselectedExamples.append(i)
if selectedExamples:
selectedExamples = self.clusterDataset[selectedExamples]
if self.selectionDisjoint == 1 and self.selectionAddTermAsClass:
selectedExamples = Orange.data.Table.from_table(newDomain, selectedExamples)
view, issparse = selectedExamples.get_column_view(goVar)
assert not issparse
view[:] = goColumn
else:
selectedExamples = None
if unselectedExamples:
unselectedExamples = self.clusterDataset[unselectedExamples]
else:
unselectedExamples = None
self.send("Data on Selected Genes", selectedExamples)
self.send("Data on Unselected Genes", unselectedExamples)
def ShowInfo(self):
dialog = QDialog(self)
dialog.setModal(False)
dialog.setLayout(QVBoxLayout())
label = QLabel(dialog)
label.setText("Ontology:\n" + self.ontology.header
if self.ontology else "Ontology not loaded!")
dialog.layout().addWidget(label)
label = QLabel(dialog)
label.setText("Annotations:\n" + self.annotations.header.replace("!", "")
if self.annotations else "Annotations not loaded!")
dialog.layout().addWidget(label)
dialog.show()
def onDeleteWidget(self):
"""Called before the widget is removed from the canvas.
"""
self.annotations = None
self.ontology = None
gc.collect() # Force collection
class OWWordEnrichment(OWWidget):
# Basic widget info
name = "Word Enrichment"
description = "Word enrichment analysis for selected documents."
icon = "icons/SetEnrichment.svg"
priority = 600
# Input/output
class Inputs:
selected_data = Input("Selected Data", Table)
data = Input("Data", Table)
want_main_area = True
class Error(OWWidget.Error):
no_bow_features = Msg('No bag-of-words features!')
no_words_overlap = Msg('No words overlap!')
empty_selection = Msg('Selected data is empty!')
all_selected = Msg('All examples can not be selected!')
# Settings
filter_by_p = Setting(False)
filter_p_value = Setting(0.01)
filter_by_fdr = Setting(True)
filter_fdr_value = Setting(0.2)
def __init__(self):
super().__init__()
# Init data
self.data = None
self.selected_data = None
self.selected_data_transformed = None # used for transforming the 'selected data' into the 'data' domain
self.words = []
self.p_values = []
self.fdr_values = []
# Info section
fbox = gui.widgetBox(self.controlArea, "Info")
self.info_all = gui.label(fbox, self, 'Cluster words:')
self.info_sel = gui.label(fbox, self, 'Selected words:')
self.info_fil = gui.label(fbox, self, 'After filtering:')
# Filtering settings
fbox = gui.widgetBox(self.controlArea, "Filter")
hbox = gui.widgetBox(fbox, orientation=0)
self.chb_p = gui.checkBox(hbox,
self,
"filter_by_p",
"p-value",
callback=self.filter_and_display,
tooltip="Filter by word p-value")
self.spin_p = gui.doubleSpin(hbox,
self,
'filter_p_value',
1e-4,
1,
step=1e-4,
labelWidth=15,
callback=self.filter_and_display,
callbackOnReturn=True,
tooltip="Max p-value for word")
self.spin_p.setEnabled(self.filter_by_p)
hbox = gui.widgetBox(fbox, orientation=0)
self.chb_fdr = gui.checkBox(hbox,
self,
"filter_by_fdr",
"FDR",
callback=self.filter_and_display,
tooltip="Filter by word FDR")
self.spin_fdr = gui.doubleSpin(hbox,
self,
'filter_fdr_value',
1e-4,
1,
step=1e-4,
labelWidth=15,
callback=self.filter_and_display,
callbackOnReturn=True,
tooltip="Max p-value for word")
self.spin_fdr.setEnabled(self.filter_by_fdr)
gui.rubber(self.controlArea)
# Word's list view
self.cols = ['Word', 'p-value', 'FDR']
self.sig_words = QTreeWidget()
self.sig_words.setColumnCount(len(self.cols))
self.sig_words.setHeaderLabels(self.cols)
self.sig_words.setSortingEnabled(True)
self.sig_words.setSelectionMode(QTreeView.ExtendedSelection)
self.sig_words.sortByColumn(2, 0) # 0 is ascending order
for i in range(len(self.cols)):
self.sig_words.resizeColumnToContents(i)
self.mainArea.layout().addWidget(self.sig_words)
@Inputs.data
def set_data(self, data=None):
self.data = data
@Inputs.selected_data
def set_data_selected(self, data=None):
self.selected_data = data
def handleNewSignals(self):
self.check_data()
def get_bow_domain(self):
domain = self.data.domain
return Domain(attributes=[
a for a in domain.attributes
if a.attributes.get('bow-feature', False)
],
class_vars=domain.class_vars,
metas=domain.metas,
source=domain)
def check_data(self):
self.Error.clear()
if isinstance(self.data, Table) and \
isinstance(self.selected_data, Table):
if len(self.selected_data) == 0:
self.Error.empty_selection()
self.clear()
return
# keep only BoW features
bow_domain = self.get_bow_domain()
if len(bow_domain.attributes) == 0:
self.Error.no_bow_features()
self.clear()
return
self.data = Corpus.from_table(bow_domain, self.data)
self.selected_data_transformed = Corpus.from_table(
bow_domain, self.selected_data)
if np_sp_sum(self.selected_data_transformed.X) == 0:
self.Error.no_words_overlap()
self.clear()
elif len(self.data) == len(self.selected_data):
self.Error.all_selected()
self.clear()
else:
self.apply()
else:
self.clear()
def clear(self):
self.sig_words.clear()
self.info_all.setText('Cluster words:')
self.info_sel.setText('Selected words:')
self.info_fil.setText('After filtering:')
def filter_enabled(self, b):
self.chb_p.setEnabled(b)
self.chb_fdr.setEnabled(b)
self.spin_p.setEnabled(b)
self.spin_fdr.setEnabled(b)
def filter_and_display(self):
self.spin_p.setEnabled(self.filter_by_p)
self.spin_fdr.setEnabled(self.filter_by_fdr)
self.sig_words.clear()
if self.selected_data_transformed is None: # do nothing when no Data
return
count = 0
if self.words:
for word, pval, fval in zip(self.words, self.p_values,
self.fdr_values):
if (not self.filter_by_p or pval <= self.filter_p_value) and \
(not self.filter_by_fdr or fval <= self.filter_fdr_value):
it = EATreeWidgetItem(word, pval, fval, self.sig_words)
self.sig_words.addTopLevelItem(it)
count += 1
for i in range(len(self.cols)):
self.sig_words.resizeColumnToContents(i)
self.info_all.setText('Cluster words: {}'.format(
len(self.selected_data_transformed.domain.attributes)))
self.info_sel.setText('Selected words: {}'.format(
np.count_nonzero(
np_sp_sum(self.selected_data_transformed.X, axis=0))))
if not self.filter_by_p and not self.filter_by_fdr:
self.info_fil.setText('After filtering:')
self.info_fil.setEnabled(False)
else:
self.info_fil.setEnabled(True)
self.info_fil.setText('After filtering: {}'.format(count))
def progress(self, p):
self.progressBarSet(p)
def apply(self):
self.clear()
self.progressBarInit()
self.filter_enabled(False)
self.words = [
i.name for i in self.selected_data_transformed.domain.attributes
]
self.p_values = hypergeom_p_values(self.data.X,
self.selected_data_transformed.X,
callback=self.progress)
self.fdr_values = false_discovery_rate(self.p_values)
self.filter_and_display()
self.filter_enabled(True)
self.progressBarFinished()
class OWWordEnrichment(OWWidget):
# Basic widget info
name = "Word Enrichment"
description = "Word enrichment analysis for selected documents."
icon = "icons/SetEnrichment.svg"
priority = 600
# Input/output
class Inputs:
selected_data = Input("Selected Data", Table)
data = Input("Data", Table)
want_main_area = True
class Error(OWWidget.Error):
no_bow_features = Msg('No bag-of-words features!')
no_words_overlap = Msg('No words overlap!')
empty_selection = Msg('Selected data is empty!')
all_selected = Msg('All examples can not be selected!')
# Settings
filter_by_p = Setting(False)
filter_p_value = Setting(0.01)
filter_by_fdr = Setting(True)
filter_fdr_value = Setting(0.2)
def __init__(self):
super().__init__()
# Init data
self.data = None
self.selected_data = None
self.selected_data_transformed = None # used for transforming the 'selected data' into the 'data' domain
self.words = []
self.p_values = []
self.fdr_values = []
# Info section
fbox = gui.widgetBox(self.controlArea, "Info")
self.info_all = gui.label(fbox, self, 'Cluster words:')
self.info_sel = gui.label(fbox, self, 'Selected words:')
self.info_fil = gui.label(fbox, self, 'After filtering:')
# Filtering settings
fbox = gui.widgetBox(self.controlArea, "Filter")
hbox = gui.widgetBox(fbox, orientation=0)
self.chb_p = gui.checkBox(hbox, self, "filter_by_p", "p-value",
callback=self.filter_and_display,
tooltip="Filter by word p-value")
self.spin_p = gui.doubleSpin(hbox, self, 'filter_p_value',
1e-4, 1, step=1e-4, labelWidth=15,
callback=self.filter_and_display,
callbackOnReturn=True,
tooltip="Max p-value for word")
self.spin_p.setEnabled(self.filter_by_p)
hbox = gui.widgetBox(fbox, orientation=0)
self.chb_fdr = gui.checkBox(hbox, self, "filter_by_fdr", "FDR",
callback=self.filter_and_display,
tooltip="Filter by word FDR")
self.spin_fdr = gui.doubleSpin(hbox, self, 'filter_fdr_value',
1e-4, 1, step=1e-4, labelWidth=15,
callback=self.filter_and_display,
callbackOnReturn=True,
tooltip="Max p-value for word")
self.spin_fdr.setEnabled(self.filter_by_fdr)
gui.rubber(self.controlArea)
# Word's list view
self.cols = ['Word', 'p-value', 'FDR']
self.sig_words = QTreeWidget()
self.sig_words.setColumnCount(len(self.cols))
self.sig_words.setHeaderLabels(self.cols)
self.sig_words.setSortingEnabled(True)
self.sig_words.setSelectionMode(QTreeView.ExtendedSelection)
self.sig_words.sortByColumn(2, 0) # 0 is ascending order
for i in range(len(self.cols)):
self.sig_words.resizeColumnToContents(i)
self.mainArea.layout().addWidget(self.sig_words)
def sizeHint(self):
return QSize(450, 240)
@Inputs.data
def set_data(self, data=None):
self.data = data
@Inputs.selected_data
def set_data_selected(self, data=None):
self.selected_data = data
def handleNewSignals(self):
self.check_data()
def get_bow_domain(self):
domain = self.data.domain
return Domain(
attributes=[a for a in domain.attributes
if a.attributes.get('bow-feature', False)],
class_vars=domain.class_vars,
metas=domain.metas,
source=domain)
def check_data(self):
self.Error.clear()
if isinstance(self.data, Table) and \
isinstance(self.selected_data, Table):
if len(self.selected_data) == 0:
self.Error.empty_selection()
self.clear()
return
# keep only BoW features
bow_domain = self.get_bow_domain()
if len(bow_domain.attributes) == 0:
self.Error.no_bow_features()
self.clear()
return
self.data = Corpus.from_table(bow_domain, self.data)
self.selected_data_transformed = Corpus.from_table(bow_domain, self.selected_data)
if np_sp_sum(self.selected_data_transformed.X) == 0:
self.Error.no_words_overlap()
self.clear()
elif len(self.data) == len(self.selected_data):
self.Error.all_selected()
self.clear()
else:
self.apply()
else:
self.clear()
def clear(self):
self.sig_words.clear()
self.info_all.setText('Cluster words:')
self.info_sel.setText('Selected words:')
self.info_fil.setText('After filtering:')
def filter_enabled(self, b):
self.chb_p.setEnabled(b)
self.chb_fdr.setEnabled(b)
self.spin_p.setEnabled(b)
self.spin_fdr.setEnabled(b)
def filter_and_display(self):
self.spin_p.setEnabled(self.filter_by_p)
self.spin_fdr.setEnabled(self.filter_by_fdr)
self.sig_words.clear()
if self.selected_data_transformed is None: # do nothing when no Data
return
count = 0
if self.words:
for word, pval, fval in zip(self.words, self.p_values, self.fdr_values):
if (not self.filter_by_p or pval <= self.filter_p_value) and \
(not self.filter_by_fdr or fval <= self.filter_fdr_value):
it = EATreeWidgetItem(word, pval, fval, self.sig_words)
self.sig_words.addTopLevelItem(it)
count += 1
for i in range(len(self.cols)):
self.sig_words.resizeColumnToContents(i)
self.info_all.setText('Cluster words: {}'.format(len(self.selected_data_transformed.domain.attributes)))
self.info_sel.setText('Selected words: {}'.format(np.count_nonzero(np_sp_sum(self.selected_data_transformed.X, axis=0))))
if not self.filter_by_p and not self.filter_by_fdr:
self.info_fil.setText('After filtering:')
self.info_fil.setEnabled(False)
else:
self.info_fil.setEnabled(True)
self.info_fil.setText('After filtering: {}'.format(count))
def progress(self, p):
self.progressBarSet(p)
def apply(self):
self.clear()
self.progressBarInit()
self.filter_enabled(False)
self.words = [i.name for i in self.selected_data_transformed.domain.attributes]
self.p_values = hypergeom_p_values(self.data.X,
self.selected_data_transformed.X,
callback=self.progress)
self.fdr_values = false_discovery_rate(self.p_values)
self.filter_and_display()
self.filter_enabled(True)
self.progressBarFinished()
def tree_to_table(self):
view = [self.cols]
items = self.sig_words.topLevelItemCount()
for i in range(items):
line = []
for j in range(3):
line.append(self.sig_words.topLevelItem(i).text(j))
view.append(line)
return(view)
def send_report(self):
if self.words:
self.report_table("Enriched words", self.tree_to_table())
class OWGOEnrichmentAnalysis(widget.OWWidget):
name = "GO Browser"
description = "Enrichment analysis for Gene Ontology terms."
icon = "../widgets/icons/GOBrowser.svg"
priority = 2020
inputs = [("Cluster Data", Orange.data.Table, "setDataset",
widget.Single + widget.Default),
("Reference Data", Orange.data.Table, "setReferenceDataset")]
outputs = [("Data on Selected Genes", Orange.data.Table),
("Data on Unselected Genes", Orange.data.Table),
("Data on Unknown Genes", Orange.data.Table),
("Enrichment Report", Orange.data.Table)]
settingsHandler = settings.DomainContextHandler()
annotationIndex = settings.ContextSetting(0)
geneAttrIndex = settings.ContextSetting(0)
useAttrNames = settings.ContextSetting(False)
geneMatcherSettings = settings.Setting([True, False, False, False])
useReferenceDataset = settings.Setting(False)
aspectIndex = settings.Setting(0)
useEvidenceType = settings.Setting(
{et: True
for et in go.evidenceTypesOrdered})
filterByNumOfInstances = settings.Setting(False)
minNumOfInstances = settings.Setting(1)
filterByPValue = settings.Setting(True)
maxPValue = settings.Setting(0.2)
filterByPValue_nofdr = settings.Setting(False)
maxPValue_nofdr = settings.Setting(0.01)
probFunc = settings.Setting(0)
selectionDirectAnnotation = settings.Setting(0)
selectionDisjoint = settings.Setting(0)
selectionAddTermAsClass = settings.Setting(0)
Ready, Initializing, Running = 0, 1, 2
def __init__(self, parent=None):
super().__init__(self, parent)
self.clusterDataset = None
self.referenceDataset = None
self.ontology = None
self.annotations = None
self.loadedAnnotationCode = "---"
self.treeStructRootKey = None
self.probFunctions = [stats.Binomial(), stats.Hypergeometric()]
self.selectedTerms = []
self.selectionChanging = 0
self.__state = OWGOEnrichmentAnalysis.Initializing
self.annotationCodes = []
#############
## GUI
#############
self.tabs = gui.tabWidget(self.controlArea)
## Input tab
self.inputTab = gui.createTabPage(self.tabs, "Input")
box = gui.widgetBox(self.inputTab, "Info")
self.infoLabel = gui.widgetLabel(box, "No data on input\n")
gui.button(
box,
self,
"Ontology/Annotation Info",
callback=self.ShowInfo,
tooltip="Show information on loaded ontology and annotations")
box = gui.widgetBox(self.inputTab, "Organism")
self.annotationComboBox = gui.comboBox(box,
self,
"annotationIndex",
items=self.annotationCodes,
callback=self._updateEnrichment,
tooltip="Select organism")
genebox = gui.widgetBox(self.inputTab, "Gene Names")
self.geneAttrIndexCombo = gui.comboBox(
genebox,
self,
"geneAttrIndex",
callback=self._updateEnrichment,
tooltip="Use this attribute to extract gene names from input data")
self.geneAttrIndexCombo.setDisabled(self.useAttrNames)
cb = gui.checkBox(genebox,
self,
"useAttrNames",
"Use column names",
tooltip="Use column names for gene names",
callback=self._updateEnrichment)
cb.toggled[bool].connect(self.geneAttrIndexCombo.setDisabled)
gui.button(genebox,
self,
"Gene matcher settings",
callback=self.UpdateGeneMatcher,
tooltip="Open gene matching settings dialog")
self.referenceRadioBox = gui.radioButtonsInBox(
self.inputTab,
self,
"useReferenceDataset", ["Entire genome", "Reference set (input)"],
tooltips=[
"Use entire genome for reference",
"Use genes from Referece Examples input signal as reference"
],
box="Reference",
callback=self._updateEnrichment)
self.referenceRadioBox.buttons[1].setDisabled(True)
gui.radioButtonsInBox(
self.inputTab,
self,
"aspectIndex",
["Biological process", "Cellular component", "Molecular function"],
box="Aspect",
callback=self._updateEnrichment)
## Filter tab
self.filterTab = gui.createTabPage(self.tabs, "Filter")
box = gui.widgetBox(self.filterTab, "Filter GO Term Nodes")
gui.checkBox(
box,
self,
"filterByNumOfInstances",
"Genes",
callback=self.FilterAndDisplayGraph,
tooltip="Filter by number of input genes mapped to a term")
ibox = gui.indentedBox(box)
gui.spin(ibox,
self,
'minNumOfInstances',
1,
100,
step=1,
label='#:',
labelWidth=15,
callback=self.FilterAndDisplayGraph,
callbackOnReturn=True,
tooltip="Min. number of input genes mapped to a term")
gui.checkBox(box,
self,
"filterByPValue_nofdr",
"p-value",
callback=self.FilterAndDisplayGraph,
tooltip="Filter by term p-value")
gui.doubleSpin(gui.indentedBox(box),
self,
'maxPValue_nofdr',
1e-8,
1,
step=1e-8,
label='p:',
labelWidth=15,
callback=self.FilterAndDisplayGraph,
callbackOnReturn=True,
tooltip="Max term p-value")
#use filterByPValue for FDR, as it was the default in prior versions
gui.checkBox(box,
self,
"filterByPValue",
"FDR",
callback=self.FilterAndDisplayGraph,
tooltip="Filter by term FDR")
gui.doubleSpin(gui.indentedBox(box),
self,
'maxPValue',
1e-8,
1,
step=1e-8,
label='p:',
labelWidth=15,
callback=self.FilterAndDisplayGraph,
callbackOnReturn=True,
tooltip="Max term p-value")
box = gui.widgetBox(box, "Significance test")
gui.radioButtonsInBox(box,
self,
"probFunc", ["Binomial", "Hypergeometric"],
tooltips=[
"Use binomial distribution test",
"Use hypergeometric distribution test"
],
callback=self._updateEnrichment)
box = gui.widgetBox(self.filterTab,
"Evidence codes in annotation",
addSpace=True)
self.evidenceCheckBoxDict = {}
for etype in go.evidenceTypesOrdered:
ecb = QCheckBox(etype,
toolTip=go.evidenceTypes[etype],
checked=self.useEvidenceType[etype])
ecb.toggled.connect(self.__on_evidenceChanged)
box.layout().addWidget(ecb)
self.evidenceCheckBoxDict[etype] = ecb
## Select tab
self.selectTab = gui.createTabPage(self.tabs, "Select")
box = gui.radioButtonsInBox(self.selectTab,
self,
"selectionDirectAnnotation",
["Directly or Indirectly", "Directly"],
box="Annotated genes",
callback=self.ExampleSelection)
box = gui.widgetBox(self.selectTab, "Output", addSpace=True)
gui.radioButtonsInBox(
box,
self,
"selectionDisjoint",
btnLabels=[
"All selected genes", "Term-specific genes",
"Common term genes"
],
tooltips=[
"Outputs genes annotated to all selected GO terms",
"Outputs genes that appear in only one of selected GO terms",
"Outputs genes common to all selected GO terms"
],
callback=[self.ExampleSelection, self.UpdateAddClassButton])
self.addClassCB = gui.checkBox(box,
self,
"selectionAddTermAsClass",
"Add GO Term as class",
callback=self.ExampleSelection)
# ListView for DAG, and table for significant GOIDs
self.DAGcolumns = [
'GO term', 'Cluster', 'Reference', 'p-value', 'FDR', 'Genes',
'Enrichment'
]
self.splitter = QSplitter(Qt.Vertical, self.mainArea)
self.mainArea.layout().addWidget(self.splitter)
# list view
self.listView = GOTreeWidget(self.splitter)
self.listView.setSelectionMode(QTreeView.ExtendedSelection)
self.listView.setAllColumnsShowFocus(1)
self.listView.setColumnCount(len(self.DAGcolumns))
self.listView.setHeaderLabels(self.DAGcolumns)
self.listView.header().setSectionsClickable(True)
self.listView.header().setSortIndicatorShown(True)
self.listView.setSortingEnabled(True)
self.listView.setItemDelegateForColumn(
6, EnrichmentColumnItemDelegate(self))
self.listView.setRootIsDecorated(True)
self.listView.itemSelectionChanged.connect(self.ViewSelectionChanged)
# table of significant GO terms
self.sigTerms = QTreeWidget(self.splitter)
self.sigTerms.setColumnCount(len(self.DAGcolumns))
self.sigTerms.setHeaderLabels(self.DAGcolumns)
self.sigTerms.setSortingEnabled(True)
self.sigTerms.setSelectionMode(QTreeView.ExtendedSelection)
self.sigTerms.setItemDelegateForColumn(
6, EnrichmentColumnItemDelegate(self))
self.sigTerms.itemSelectionChanged.connect(self.TableSelectionChanged)
self.sigTableTermsSorted = []
self.graph = {}
self.inputTab.layout().addStretch(1)
self.filterTab.layout().addStretch(1)
self.selectTab.layout().addStretch(1)
self.setBlocking(True)
self._executor = ThreadExecutor()
self._init = EnsureDownloaded([(taxonomy.Taxonomy.DOMAIN,
taxonomy.Taxonomy.FILENAME),
("GO", "taxonomy.pickle")])
self._init.finished.connect(self.__initialize_finish)
self._executor.submit(self._init)
def sizeHint(self):
return QSize(1000, 700)
def __initialize_finish(self):
self.setBlocking(False)
try:
self.annotationFiles = listAvailable()
except ConnectTimeout:
self.error(2, "Internet connection error, unable to load data. " + \
"Check connection and create a new GO Browser widget.")
self.filterTab.setEnabled(False)
self.inputTab.setEnabled(False)
self.selectTab.setEnabled(False)
self.listView.setEnabled(False)
self.sigTerms.setEnabled(False)
else:
self.annotationCodes = sorted(self.annotationFiles.keys())
self.annotationComboBox.clear()
self.annotationComboBox.addItems(self.annotationCodes)
self.annotationComboBox.setCurrentIndex(self.annotationIndex)
self.__state = OWGOEnrichmentAnalysis.Ready
def __on_evidenceChanged(self):
for etype, cb in self.evidenceCheckBoxDict.items():
self.useEvidenceType[etype] = cb.isChecked()
self._updateEnrichment()
def UpdateGeneMatcher(self):
"""Open the Gene matcher settings dialog."""
dialog = GeneMatcherDialog(self,
defaults=self.geneMatcherSettings,
modal=True)
if dialog.exec_() != QDialog.Rejected:
self.geneMatcherSettings = [
getattr(dialog, item[0]) for item in dialog.items
]
if self.annotations:
self.SetGeneMatcher()
self._updateEnrichment()
def clear(self):
self.infoLabel.setText("No data on input\n")
self.warning(0)
self.warning(1)
self.geneAttrIndexCombo.clear()
self.ClearGraph()
self.send("Data on Selected Genes", None)
self.send("Data on Unselected Genes", None)
self.send("Data on Unknown Genes", None)
self.send("Enrichment Report", None)
def setDataset(self, data=None):
if self.__state == OWGOEnrichmentAnalysis.Initializing:
self.__initialize_finish()
self.closeContext()
self.clear()
self.clusterDataset = data
if data is not None:
domain = data.domain
allvars = domain.variables + domain.metas
self.candidateGeneAttrs = [var for var in allvars if isstring(var)]
self.geneAttrIndexCombo.clear()
for var in self.candidateGeneAttrs:
self.geneAttrIndexCombo.addItem(*gui.attributeItem(var))
taxid = data_hints.get_hint(data, "taxid", "")
code = None
try:
code = go.from_taxid(taxid)
except KeyError:
pass
except Exception as ex:
print(ex)
if code is not None:
filename = "gene_association.%s.tar.gz" % code
if filename in self.annotationFiles.values():
self.annotationIndex = \
[i for i, name in enumerate(self.annotationCodes) \
if self.annotationFiles[name] == filename].pop()
self.useAttrNames = data_hints.get_hint(data, "genesinrows",
self.useAttrNames)
self.openContext(data)
self.geneAttrIndex = min(self.geneAttrIndex,
len(self.candidateGeneAttrs) - 1)
if len(self.candidateGeneAttrs) == 0:
self.useAttrNames = True
self.geneAttrIndex = -1
elif self.geneAttrIndex < len(self.candidateGeneAttrs):
self.geneAttrIndex = len(self.candidateGeneAttrs) - 1
self._updateEnrichment()
def setReferenceDataset(self, data=None):
self.referenceDataset = data
self.referenceRadioBox.buttons[1].setDisabled(not bool(data))
self.referenceRadioBox.buttons[1].setText("Reference set")
if self.clusterDataset is not None and self.useReferenceDataset:
self.useReferenceDataset = 0 if not data else 1
graph = self.Enrichment()
self.SetGraph(graph)
elif self.clusterDataset:
self.__updateReferenceSetButton()
def handleNewSignals(self):
super().handleNewSignals()
def _updateEnrichment(self):
if self.clusterDataset is not None and \
self.__state == OWGOEnrichmentAnalysis.Ready:
pb = gui.ProgressBar(self, 100)
self.Load(pb=pb)
graph = self.Enrichment(pb=pb)
self.FilterUnknownGenes()
self.SetGraph(graph)
def __updateReferenceSetButton(self):
allgenes, refgenes = None, None
if self.referenceDataset:
try:
allgenes = self.genesFromTable(self.referenceDataset)
except Exception:
allgenes = []
refgenes, unknown = self.FilterAnnotatedGenes(allgenes)
self.referenceRadioBox.buttons[1].setDisabled(not bool(allgenes))
self.referenceRadioBox.buttons[1].setText("Reference set " + (
"(%i genes, %i matched)" %
(len(allgenes), len(refgenes)) if allgenes and refgenes else ""))
def genesFromTable(self, data):
if self.useAttrNames:
genes = [v.name for v in data.domain.variables]
else:
attr = self.candidateGeneAttrs[min(
self.geneAttrIndex,
len(self.candidateGeneAttrs) - 1)]
genes = [str(ex[attr]) for ex in data if not numpy.isnan(ex[attr])]
if any("," in gene for gene in genes):
self.information(
0,
"Separators detected in gene names. Assuming multiple genes per example."
)
genes = reduce(operator.iadd,
(genes.split(",") for genes in genes), [])
return genes
def FilterAnnotatedGenes(self, genes):
matchedgenes = self.annotations.get_gene_names_translator(
genes).values()
return matchedgenes, [
gene for gene in genes if gene not in matchedgenes
]
def FilterUnknownGenes(self):
if not self.useAttrNames and self.candidateGeneAttrs:
geneAttr = self.candidateGeneAttrs[min(
self.geneAttrIndex,
len(self.candidateGeneAttrs) - 1)]
indices = []
for i, ex in enumerate(self.clusterDataset):
if not any(
self.annotations.genematcher.match(n.strip())
for n in str(ex[geneAttr]).split(",")):
indices.append(i)
if indices:
data = self.clusterDataset[indices]
else:
data = None
self.send("Data on Unknown Genes", data)
else:
self.send("Data on Unknown Genes", None)
def Load(self, pb=None):
if self.__state == OWGOEnrichmentAnalysis.Ready:
go_files, tax_files = serverfiles.listfiles(
"GO"), serverfiles.listfiles("Taxonomy")
calls = []
pb, finish = (gui.ProgressBar(self, 0),
True) if pb is None else (pb, False)
count = 0
if not tax_files:
calls.append(("Taxonomy", "ncbi_taxnomy.tar.gz"))
count += 1
org = self.annotationCodes[min(self.annotationIndex,
len(self.annotationCodes) - 1)]
if org != self.loadedAnnotationCode:
count += 1
if self.annotationFiles[org] not in go_files:
calls.append(("GO", self.annotationFiles[org]))
count += 1
if "gene_ontology_edit.obo.tar.gz" not in go_files:
calls.append(("GO", "gene_ontology_edit.obo.tar.gz"))
count += 1
if not self.ontology:
count += 1
pb.iter += count * 100
for args in calls:
serverfiles.localpath_download(*args,
**dict(callback=pb.advance))
i = len(calls)
if not self.ontology:
self.ontology = go.Ontology(
progress_callback=lambda value: pb.advance())
i += 1
if org != self.loadedAnnotationCode:
self.annotations = None
gc.collect() # Force run garbage collection
code = self.annotationFiles[org].split(".")[-3]
self.annotations = go.Annotations(
code,
genematcher=gene.GMDirect(),
progress_callback=lambda value: pb.advance())
i += 1
self.loadedAnnotationCode = org
count = defaultdict(int)
geneSets = defaultdict(set)
for anno in self.annotations.annotations:
count[anno.evidence] += 1
geneSets[anno.evidence].add(anno.geneName)
for etype in go.evidenceTypesOrdered:
ecb = self.evidenceCheckBoxDict[etype]
ecb.setEnabled(bool(count[etype]))
ecb.setText(etype + ": %i annots(%i genes)" %
(count[etype], len(geneSets[etype])))
if finish:
pb.finish()
def SetGeneMatcher(self):
if self.annotations:
taxid = self.annotations.taxid
matchers = []
for matcher, use in zip(
[gene.GMGO, gene.GMKEGG, gene.GMNCBI, gene.GMAffy],
self.geneMatcherSettings):
if use:
try:
if taxid == "352472":
matchers.extend([
matcher(taxid),
gene.GMDicty(),
[matcher(taxid),
gene.GMDicty()]
])
# The reason machers are duplicated is that we want `matcher` or `GMDicty` to
# match genes by them self if possible. Only use the joint matcher if they fail.
else:
matchers.append(matcher(taxid))
except Exception as ex:
print(ex)
self.annotations.genematcher = gene.matcher(matchers)
self.annotations.genematcher.set_targets(
self.annotations.gene_names)
def Enrichment(self, pb=None):
assert self.clusterDataset is not None
pb = gui.ProgressBar(self, 100) if pb is None else pb
if not self.annotations.ontology:
self.annotations.ontology = self.ontology
if isinstance(self.annotations.genematcher, gene.GMDirect):
self.SetGeneMatcher()
self.error(1)
self.warning([0, 1])
if self.useAttrNames:
clusterGenes = [
v.name for v in self.clusterDataset.domain.attributes
]
self.information(0)
elif 0 <= self.geneAttrIndex < len(self.candidateGeneAttrs):
geneAttr = self.candidateGeneAttrs[self.geneAttrIndex]
clusterGenes = [
str(ex[geneAttr]) for ex in self.clusterDataset
if not numpy.isnan(ex[geneAttr])
]
if any("," in gene for gene in clusterGenes):
self.information(
0,
"Separators detected in cluster gene names. Assuming multiple genes per example."
)
clusterGenes = reduce(operator.iadd,
(genes.split(",")
for genes in clusterGenes), [])
else:
self.information(0)
else:
self.error(1, "Failed to extract gene names from input dataset!")
return {}
genesSetCount = len(set(clusterGenes))
self.clusterGenes = clusterGenes = self.annotations.get_gene_names_translator(
clusterGenes).values()
self.infoLabel.setText(
"%i unique genes on input\n%i (%.1f%%) genes with known annotations"
% (genesSetCount, len(clusterGenes), 100.0 * len(clusterGenes) /
genesSetCount if genesSetCount else 0.0))
referenceGenes = None
if not self.useReferenceDataset or self.referenceDataset is None:
self.information(2)
self.information(1)
referenceGenes = self.annotations.gene_names
elif self.referenceDataset is not None:
if self.useAttrNames:
referenceGenes = [
v.name for v in self.referenceDataset.domain.attributes
]
self.information(1)
elif geneAttr in (self.referenceDataset.domain.variables +
self.referenceDataset.domain.metas):
referenceGenes = [
str(ex[geneAttr]) for ex in self.referenceDataset
if not numpy.isnan(ex[geneAttr])
]
if any("," in gene for gene in clusterGenes):
self.information(
1,
"Separators detected in reference gene names. Assuming multiple genes per example."
)
referenceGenes = reduce(operator.iadd,
(genes.split(",")
for genes in referenceGenes), [])
else:
self.information(1)
else:
self.information(1)
referenceGenes = None
if referenceGenes is None:
referenceGenes = list(self.annotations.gene_names)
self.referenceRadioBox.buttons[1].setText("Reference set")
self.referenceRadioBox.buttons[1].setDisabled(True)
self.information(
2,
"Unable to extract gene names from reference dataset. Using entire genome for reference"
)
self.useReferenceDataset = 0
else:
refc = len(referenceGenes)
referenceGenes = self.annotations.get_gene_names_translator(
referenceGenes).values()
self.referenceRadioBox.buttons[1].setText(
"Reference set (%i genes, %i matched)" %
(refc, len(referenceGenes)))
self.referenceRadioBox.buttons[1].setDisabled(False)
self.information(2)
else:
self.useReferenceDataset = 0
if not referenceGenes:
self.error(1, "No valid reference set")
return {}
self.referenceGenes = referenceGenes
evidences = []
for etype in go.evidenceTypesOrdered:
if self.useEvidenceType[etype]:
evidences.append(etype)
aspect = ["P", "C", "F"][self.aspectIndex]
if clusterGenes:
self.terms = terms = self.annotations.get_enriched_terms(
clusterGenes,
referenceGenes,
evidences,
aspect=aspect,
prob=self.probFunctions[self.probFunc],
use_fdr=False,
progress_callback=lambda value: pb.advance())
ids = []
pvals = []
for i, d in self.terms.items():
ids.append(i)
pvals.append(d[1])
for i, fdr in zip(ids, stats.FDR(
pvals)): # save FDR as the last part of the tuple
terms[i] = tuple(list(terms[i]) + [fdr])
else:
self.terms = terms = {}
if not self.terms:
self.warning(0, "No enriched terms found.")
else:
self.warning(0)
pb.finish()
self.treeStructDict = {}
ids = self.terms.keys()
self.treeStructRootKey = None
parents = {}
for id in ids:
parents[id] = set([term for _, term in self.ontology[id].related])
children = {}
for term in self.terms:
children[term] = set([id for id in ids if term in parents[id]])
for term in self.terms:
self.treeStructDict[term] = TreeNode(self.terms[term],
children[term])
if not self.ontology[term].related and not getattr(
self.ontology[term], "is_obsolete", False):
self.treeStructRootKey = term
return terms
def FilterGraph(self, graph):
if self.filterByPValue_nofdr:
graph = go.filterByPValue(graph, self.maxPValue_nofdr)
if self.filterByPValue: #FDR
graph = dict(
filter(lambda item: item[1][3] <= self.maxPValue,
graph.items()))
if self.filterByNumOfInstances:
graph = dict(
filter(lambda item: len(item[1][0]) >= self.minNumOfInstances,
graph.items()))
return graph
def FilterAndDisplayGraph(self):
if self.clusterDataset:
self.graph = self.FilterGraph(self.originalGraph)
if self.originalGraph and not self.graph:
self.warning(1, "All found terms were filtered out.")
else:
self.warning(1)
self.ClearGraph()
self.DisplayGraph()
def SetGraph(self, graph=None):
self.originalGraph = graph
if graph:
self.FilterAndDisplayGraph()
else:
self.graph = {}
self.ClearGraph()
def ClearGraph(self):
self.listView.clear()
self.listViewItems = []
self.sigTerms.clear()
def DisplayGraph(self):
fromParentDict = {}
self.termListViewItemDict = {}
self.listViewItems = []
enrichment = lambda t: len(t[0]) / t[2] * (len(self.referenceGenes) /
len(self.clusterGenes))
maxFoldEnrichment = max(
[enrichment(term) for term in self.graph.values()] or [1])
def addNode(term, parent, parentDisplayNode):
if (parent, term) in fromParentDict:
return
if term in self.graph:
displayNode = GOTreeWidgetItem(self.ontology[term],
self.graph[term],
len(self.clusterGenes),
len(self.referenceGenes),
maxFoldEnrichment,
parentDisplayNode)
displayNode.goId = term
self.listViewItems.append(displayNode)
if term in self.termListViewItemDict:
self.termListViewItemDict[term].append(displayNode)
else:
self.termListViewItemDict[term] = [displayNode]
fromParentDict[(parent, term)] = True
parent = term
else:
displayNode = parentDisplayNode
for c in self.treeStructDict[term].children:
addNode(c, parent, displayNode)
if self.treeStructDict:
addNode(self.treeStructRootKey, None, self.listView)
terms = self.graph.items()
terms = sorted(terms, key=lambda item: item[1][1])
self.sigTableTermsSorted = [t[0] for t in terms]
self.sigTerms.clear()
for i, (t_id, (genes, p_value, refCount, fdr)) in enumerate(terms):
item = GOTreeWidgetItem(self.ontology[t_id],
(genes, p_value, refCount, fdr),
len(self.clusterGenes),
len(self.referenceGenes),
maxFoldEnrichment, self.sigTerms)
item.goId = t_id
self.listView.expandAll()
for i in range(5):
self.listView.resizeColumnToContents(i)
self.sigTerms.resizeColumnToContents(i)
self.sigTerms.resizeColumnToContents(6)
width = min(self.listView.columnWidth(0), 350)
self.listView.setColumnWidth(0, width)
self.sigTerms.setColumnWidth(0, width)
# Create and send the enrichemnt report table.
termsDomain = Orange.data.Domain(
[],
[],
# All is meta!
[
Orange.data.StringVariable("GO Term Id"),
Orange.data.StringVariable("GO Term Name"),
Orange.data.ContinuousVariable("Cluster Frequency"),
Orange.data.ContinuousVariable("Genes in Cluster",
number_of_decimals=0),
Orange.data.ContinuousVariable("Reference Frequency"),
Orange.data.ContinuousVariable("Genes in Reference",
number_of_decimals=0),
Orange.data.ContinuousVariable("p-value"),
Orange.data.ContinuousVariable("FDR"),
Orange.data.ContinuousVariable("Enrichment"),
Orange.data.StringVariable("Genes")
])
terms = [[t_id,
self.ontology[t_id].name,
len(genes) / len(self.clusterGenes),
len(genes),
r_count / len(self.referenceGenes),
r_count,
p_value,
fdr,
len(genes) / len(self.clusterGenes) * \
len(self.referenceGenes) / r_count,
",".join(genes)
]
for t_id, (genes, p_value, r_count, fdr) in terms]
if terms:
X = numpy.empty((len(terms), 0))
M = numpy.array(terms, dtype=object)
termsTable = Orange.data.Table.from_numpy(termsDomain, X, metas=M)
else:
termsTable = Orange.data.Table(termsDomain)
self.send("Enrichment Report", termsTable)
def ViewSelectionChanged(self):
if self.selectionChanging:
return
self.selectionChanging = 1
self.selectedTerms = []
selected = self.listView.selectedItems()
self.selectedTerms = list(set([lvi.term.id for lvi in selected]))
self.ExampleSelection()
self.selectionChanging = 0
def TableSelectionChanged(self):
if self.selectionChanging:
return
self.selectionChanging = 1
self.selectedTerms = []
selectedIds = set([
self.sigTerms.itemFromIndex(index).goId
for index in self.sigTerms.selectedIndexes()
])
for i in range(self.sigTerms.topLevelItemCount()):
item = self.sigTerms.topLevelItem(i)
selected = item.goId in selectedIds
term = item.goId
if selected:
self.selectedTerms.append(term)
for lvi in self.termListViewItemDict[term]:
try:
lvi.setSelected(selected)
if selected:
lvi.setExpanded(True)
except RuntimeError: # Underlying C/C++ object deleted
pass
self.ExampleSelection()
self.selectionChanging = 0
def UpdateAddClassButton(self):
self.addClassCB.setEnabled(self.selectionDisjoint == 1)
def ExampleSelection(self):
self.commit()
def commit(self):
if self.clusterDataset is None:
return
terms = set(self.selectedTerms)
genes = reduce(operator.ior,
(set(self.graph[term][0]) for term in terms), set())
evidences = []
for etype in go.evidenceTypesOrdered:
if self.useEvidenceType[etype]:
# if getattr(self, "useEvidence" + etype):
evidences.append(etype)
allTerms = self.annotations.get_annotated_terms(
genes,
direct_annotation_only=self.selectionDirectAnnotation,
evidence_codes=evidences)
if self.selectionDisjoint > 0:
count = defaultdict(int)
for term in self.selectedTerms:
for g in allTerms.get(term, []):
count[g] += 1
ccount = 1 if self.selectionDisjoint == 1 else len(
self.selectedTerms)
selectedGenes = [
gene for gene, c in count.items()
if c == ccount and gene in genes
]
else:
selectedGenes = reduce(operator.ior,
(set(allTerms.get(term, []))
for term in self.selectedTerms), set())
if self.useAttrNames:
vars = [
self.clusterDataset.domain[gene] for gene in set(selectedGenes)
]
domain = Orange.data.Domain(vars,
self.clusterDataset.domain.class_vars,
self.clusterDataset.domain.metas)
newdata = self.clusterDataset.from_table(domain,
self.clusterDataset)
self.send("Data on Selected Genes", newdata)
self.send("Data on Unselected Genes", None)
elif self.candidateGeneAttrs:
selectedExamples = []
unselectedExamples = []
geneAttr = self.candidateGeneAttrs[min(
self.geneAttrIndex,
len(self.candidateGeneAttrs) - 1)]
if self.selectionDisjoint == 1:
goVar = Orange.data.DiscreteVariable("GO Term",
values=list(
self.selectedTerms))
newDomain = Orange.data.Domain(
self.clusterDataset.domain.variables, goVar,
self.clusterDataset.domain.metas)
goColumn = []
for i, ex in enumerate(self.clusterDataset):
if not numpy.isnan(ex[geneAttr]) and any(
gene in selectedGenes
for gene in str(ex[geneAttr]).split(",")):
if self.selectionDisjoint == 1 and self.selectionAddTermAsClass:
terms = filter(
lambda term: any(gene in self.graph[term][0]
for gene in str(ex[geneAttr]).
split(",")), self.selectedTerms)
term = sorted(terms)[0]
goColumn.append(goVar.values.index(term))
selectedExamples.append(i)
else:
unselectedExamples.append(i)
if selectedExamples:
selectedExamples = self.clusterDataset[selectedExamples]
if self.selectionDisjoint == 1 and self.selectionAddTermAsClass:
selectedExamples = Orange.data.Table.from_table(
newDomain, selectedExamples)
view, issparse = selectedExamples.get_column_view(goVar)
assert not issparse
view[:] = goColumn
else:
selectedExamples = None
if unselectedExamples:
unselectedExamples = self.clusterDataset[unselectedExamples]
else:
unselectedExamples = None
self.send("Data on Selected Genes", selectedExamples)
self.send("Data on Unselected Genes", unselectedExamples)
def ShowInfo(self):
dialog = QDialog(self)
dialog.setModal(False)
dialog.setLayout(QVBoxLayout())
label = QLabel(dialog)
label.setText(
"Ontology:\n" +
self.ontology.header if self.ontology else "Ontology not loaded!")
dialog.layout().addWidget(label)
label = QLabel(dialog)
label.setText("Annotations:\n" +
self.annotations.header.replace("!", "") if self.
annotations else "Annotations not loaded!")
dialog.layout().addWidget(label)
dialog.show()
def onDeleteWidget(self):
"""Called before the widget is removed from the canvas.
"""
self.annotations = None
self.ontology = None
gc.collect() # Force collection
class OWGOBrowser(widget.OWWidget):
name = "GO Browser"
description = "Enrichment analysis for Gene Ontology terms."
icon = "../widgets/icons/OWGOBrowser.svg"
priority = 7
inputs = [("Cluster Data", Orange.data.Table,
"setDataset", widget.Single + widget.Default),
("Reference Data", Orange.data.Table,
"setReferenceDataset")]
outputs = [("Data on Selected Genes", Orange.data.Table),
("Enrichment Report", Orange.data.Table)]
settingsHandler = settings.DomainContextHandler()
geneAttrIndex = settings.ContextSetting(0)
useAttrNames = settings.ContextSetting(False)
useReferenceDataset = settings.Setting(False)
aspectIndex = settings.Setting(0)
useEvidenceType = settings.Setting(
{et: True for et in go.evidenceTypesOrdered})
filterByNumOfInstances = settings.Setting(False)
minNumOfInstances = settings.Setting(1)
filterByPValue = settings.Setting(True)
maxPValue = settings.Setting(0.2)
filterByPValue_nofdr = settings.Setting(False)
maxPValue_nofdr = settings.Setting(0.01)
probFunc = settings.Setting(0)
selectionDirectAnnotation = settings.Setting(0)
selectionDisjoint = settings.Setting(0)
class Error(widget.OWWidget.Error):
serverfiles_unavailable = widget.Msg('Can not locate annotation files, '
'please check your connection and try again.')
missing_annotation = widget.Msg(ERROR_ON_MISSING_ANNOTATION)
missing_gene_id = widget.Msg(ERROR_ON_MISSING_GENE_ID)
missing_tax_id = widget.Msg(ERROR_ON_MISSING_TAX_ID)
def __init__(self, parent=None):
super().__init__(self, parent)
self.input_data = None
self.ref_data = None
self.ontology = None
self.annotations = None
self.loaded_annotation_code = None
self.treeStructRootKey = None
self.probFunctions = [statistics.Binomial(), statistics.Hypergeometric()]
self.selectedTerms = []
self.selectionChanging = 0
self.__state = State.Ready
self.__scheduletimer = QTimer(self, singleShot=True)
self.__scheduletimer.timeout.connect(self.__update)
#############
# GUI
#############
self.tabs = gui.tabWidget(self.controlArea)
# Input tab
self.inputTab = gui.createTabPage(self.tabs, "Input")
box = gui.widgetBox(self.inputTab, "Info")
self.infoLabel = gui.widgetLabel(box, "No data on input\n")
gui.button(box, self, "Ontology/Annotation Info",
callback=self.ShowInfo,
tooltip="Show information on loaded ontology and annotations")
self.referenceRadioBox = gui.radioButtonsInBox(
self.inputTab, self, "useReferenceDataset",
["Entire genome", "Reference set (input)"],
tooltips=["Use entire genome for reference",
"Use genes from Referece Examples input signal as reference"],
box="Reference", callback=self.__invalidate)
self.referenceRadioBox.buttons[1].setDisabled(True)
gui.radioButtonsInBox(
self.inputTab, self, "aspectIndex",
["Biological process", "Cellular component", "Molecular function"],
box="Aspect", callback=self.__invalidate)
# Filter tab
self.filterTab = gui.createTabPage(self.tabs, "Filter")
box = gui.widgetBox(self.filterTab, "Filter GO Term Nodes")
gui.checkBox(box, self, "filterByNumOfInstances", "Genes",
callback=self.FilterAndDisplayGraph,
tooltip="Filter by number of input genes mapped to a term")
ibox = gui.indentedBox(box)
gui.spin(ibox, self, 'minNumOfInstances', 1, 100,
step=1, label='#:', labelWidth=15,
callback=self.FilterAndDisplayGraph,
callbackOnReturn=True,
tooltip="Min. number of input genes mapped to a term")
gui.checkBox(box, self, "filterByPValue_nofdr", "p-value",
callback=self.FilterAndDisplayGraph,
tooltip="Filter by term p-value")
gui.doubleSpin(gui.indentedBox(box), self, 'maxPValue_nofdr', 1e-8, 1,
step=1e-8, label='p:', labelWidth=15,
callback=self.FilterAndDisplayGraph,
callbackOnReturn=True,
tooltip="Max term p-value")
# use filterByPValue for FDR, as it was the default in prior versions
gui.checkBox(box, self, "filterByPValue", "FDR",
callback=self.FilterAndDisplayGraph,
tooltip="Filter by term FDR")
gui.doubleSpin(gui.indentedBox(box), self, 'maxPValue', 1e-8, 1,
step=1e-8, label='p:', labelWidth=15,
callback=self.FilterAndDisplayGraph,
callbackOnReturn=True,
tooltip="Max term p-value")
box = gui.widgetBox(box, "Significance test")
gui.radioButtonsInBox(box, self, "probFunc", ["Binomial", "Hypergeometric"],
tooltips=["Use binomial distribution test",
"Use hypergeometric distribution test"],
callback=self.__invalidate) # TODO: only update the p values
box = gui.widgetBox(self.filterTab, "Evidence codes in annotation",
addSpace=True)
self.evidenceCheckBoxDict = {}
for etype in go.evidenceTypesOrdered:
ecb = QCheckBox(
etype, toolTip=go.evidenceTypes[etype],
checked=self.useEvidenceType[etype])
ecb.toggled.connect(self.__on_evidenceChanged)
box.layout().addWidget(ecb)
self.evidenceCheckBoxDict[etype] = ecb
# Select tab
self.selectTab = gui.createTabPage(self.tabs, "Select")
box = gui.radioButtonsInBox(
self.selectTab, self, "selectionDirectAnnotation",
["Directly or Indirectly", "Directly"],
box="Annotated genes",
callback=self.ExampleSelection)
box = gui.widgetBox(self.selectTab, "Output", addSpace=True)
gui.radioButtonsInBox(
box, self, "selectionDisjoint",
btnLabels=["All selected genes",
"Term-specific genes",
"Common term genes"],
tooltips=["Outputs genes annotated to all selected GO terms",
"Outputs genes that appear in only one of selected GO terms",
"Outputs genes common to all selected GO terms"],
callback=self.ExampleSelection)
# ListView for DAG, and table for significant GOIDs
self.DAGcolumns = ['GO term', 'Cluster', 'Reference', 'p-value',
'FDR', 'Genes', 'Enrichment']
self.splitter = QSplitter(Qt.Vertical, self.mainArea)
self.mainArea.layout().addWidget(self.splitter)
# list view
self.listView = GOTreeWidget(self.splitter)
self.listView.setSelectionMode(QTreeView.ExtendedSelection)
self.listView.setAllColumnsShowFocus(1)
self.listView.setColumnCount(len(self.DAGcolumns))
self.listView.setHeaderLabels(self.DAGcolumns)
self.listView.header().setSectionsClickable(True)
self.listView.header().setSortIndicatorShown(True)
self.listView.header().setSortIndicator(self.DAGcolumns.index('p-value'), Qt.AscendingOrder)
self.listView.setSortingEnabled(True)
self.listView.setItemDelegateForColumn(
6, EnrichmentColumnItemDelegate(self))
self.listView.setRootIsDecorated(True)
self.listView.itemSelectionChanged.connect(self.ViewSelectionChanged)
# table of significant GO terms
self.sigTerms = QTreeWidget(self.splitter)
self.sigTerms.setColumnCount(len(self.DAGcolumns))
self.sigTerms.setHeaderLabels(self.DAGcolumns)
self.sigTerms.setSortingEnabled(True)
self.sigTerms.setSelectionMode(QTreeView.ExtendedSelection)
self.sigTerms.header().setSortIndicator(self.DAGcolumns.index('p-value'), Qt.AscendingOrder)
self.sigTerms.setItemDelegateForColumn(
6, EnrichmentColumnItemDelegate(self))
self.sigTerms.itemSelectionChanged.connect(self.TableSelectionChanged)
self.sigTableTermsSorted = []
self.graph = {}
self.originalGraph = None
self.inputTab.layout().addStretch(1)
self.filterTab.layout().addStretch(1)
self.selectTab.layout().addStretch(1)
class AnnotationSlot(SimpleNamespace):
taxid = ... # type: str
name = ... # type: str
filename = ... # type:str
@staticmethod
def parse_tax_id(f_name):
return f_name.split('.')[1]
try:
remote_files = serverfiles.ServerFiles().listfiles(DOMAIN)
except (ConnectTimeout, RequestException, ConnectionError):
# TODO: Warn user about failed connection to the remote server
remote_files = []
self.available_annotations = [
AnnotationSlot(
taxid=AnnotationSlot.parse_tax_id(annotation_file),
name=taxonomy.common_taxid_to_name(AnnotationSlot.parse_tax_id(annotation_file)),
filename=FILENAME_ANNOTATION.format(AnnotationSlot.parse_tax_id(annotation_file))
)
for _, annotation_file in set(remote_files + serverfiles.listfiles(DOMAIN))
if annotation_file != FILENAME_ONTOLOGY
]
self._executor = ThreadExecutor()
def sizeHint(self):
return QSize(1000, 700)
def __on_evidenceChanged(self):
for etype, cb in self.evidenceCheckBoxDict.items():
self.useEvidenceType[etype] = cb.isChecked()
self.__invalidate()
def clear(self):
self.infoLabel.setText("No data on input\n")
self.warning(0)
self.warning(1)
self.ClearGraph()
self.send("Data on Selected Genes", None)
self.send("Enrichment Report", None)
def setDataset(self, data=None):
self.closeContext()
self.clear()
self.Error.clear()
if data:
self.input_data = data
self.tax_id = str(self.input_data.attributes.get(TAX_ID, None))
self.use_attr_names = self.input_data.attributes.get(GENE_AS_ATTRIBUTE_NAME, None)
self.gene_id_attribute = self.input_data.attributes.get(GENE_ID_ATTRIBUTE, None)
self.gene_id_column = self.input_data.attributes.get(GENE_ID_COLUMN, None)
self.annotation_index = None
if not(self.use_attr_names is not None
and ((self.gene_id_attribute is None) ^ (self.gene_id_column is None))):
if self.tax_id is None:
self.Error.missing_annotation()
return
self.Error.missing_gene_id()
return
elif self.tax_id is None:
self.Error.missing_tax_id()
return
_c2i = {a.taxid: i for i, a in enumerate(self.available_annotations)}
try:
self.annotation_index = _c2i[self.tax_id]
except KeyError:
self.Error.serverfiles_unavailable()
# raise ValueError('Taxonomy {} not supported.'.format(self.tax_id))
return
self.__invalidate()
def setReferenceDataset(self, data=None):
self.Error.clear()
if data:
self.ref_data = data
self.ref_tax_id = str(self.ref_data.attributes.get(TAX_ID, None))
self.ref_use_attr_names = self.ref_data.attributes.get(GENE_AS_ATTRIBUTE_NAME, None)
self.ref_gene_id_attribute = self.ref_data.attributes.get(GENE_ID_ATTRIBUTE, None)
self.ref_gene_id_column = self.ref_data.attributes.get(GENE_ID_COLUMN, None)
if not (self.ref_use_attr_names is not None
and ((self.ref_gene_id_attribute is None) ^ (self.ref_gene_id_column is None))):
if self.ref_tax_id is None:
self.Error.missing_annotation()
return
self.Error.missing_gene_id()
return
elif self.ref_tax_id is None:
self.Error.missing_tax_id()
return
self.referenceRadioBox.buttons[1].setDisabled(not bool(data))
self.referenceRadioBox.buttons[1].setText("Reference set")
if self.input_data is not None and self.useReferenceDataset:
self.useReferenceDataset = 0 if not data else 1
self.__invalidate()
@Slot()
def __invalidate(self):
# Invalidate the current results or pending task and schedule an
# update.
self.__scheduletimer.start()
if self.__state != State.Ready:
self.__state |= State.Stale
self.SetGraph({})
self.ref_genes = None
self.input_genes = None
def __invalidateAnnotations(self):
self.annotations = None
self.loaded_annotation_code = None
if self.input_data:
self.infoLabel.setText("...\n")
self.__invalidate()
@Slot()
def __update(self):
self.__scheduletimer.stop()
if self.input_data is None:
return
if self.__state & State.Running:
self.__state |= State.Stale
elif self.__state & State.Downloading:
self.__state |= State.Stale
elif self.__state & State.Ready:
if self.__ensure_data():
self.Load()
self.Enrichment()
else:
assert self.__state & State.Downloading
assert self.isBlocking()
def __get_ref_genes(self):
self.ref_genes = []
if self.ref_use_attr_names:
for variable in self.input_data.domain.attributes:
self.ref_genes.append(str(variable.attributes.get(self.ref_gene_id_attribute, '?')))
else:
genes, _ = self.ref_data.get_column_view(self.ref_gene_id_column)
self.ref_genes = [str(g) for g in genes]
def __get_input_genes(self):
self.input_genes = []
if self.use_attr_names:
for variable in self.input_data.domain.attributes:
self.input_genes .append(str(variable.attributes.get(self.gene_id_attribute, '?')))
else:
genes, _ = self.input_data.get_column_view(self.gene_id_column)
self.input_genes = [str(g) for g in genes]
def FilterAnnotatedGenes(self, genes):
matchedgenes = self.annotations.get_gene_names_translator(genes).values()
return matchedgenes, [gene for gene in genes if gene not in matchedgenes]
def __start_download(self, files_list):
# type: (List[Tuple[str, str]]) -> None
task = EnsureDownloaded(files_list)
task.progress.connect(self._progressBarSet)
f = self._executor.submit(task)
fw = FutureWatcher(f, self)
fw.finished.connect(self.__download_finish)
fw.finished.connect(fw.deleteLater)
fw.resultReady.connect(self.__invalidate)
self.progressBarInit(processEvents=None)
self.setBlocking(True)
self.setStatusMessage("Downloading")
self.__state = State.Downloading
@Slot(Future)
def __download_finish(self, result):
# type: (Future[None]) -> None
assert QThread.currentThread() is self.thread()
assert result.done()
self.setBlocking(False)
self.setStatusMessage("")
self.progressBarFinished(processEvents=False)
try:
result.result()
except ConnectTimeout:
logging.getLogger(__name__).error("Error:")
self.error(2, "Internet connection error, unable to load data. " +
"Check connection and create a new GO Browser widget.")
except RequestException as err:
logging.getLogger(__name__).error("Error:")
self.error(2, "Internet error:\n" + str(err))
except BaseException as err:
logging.getLogger(__name__).error("Error:")
self.error(2, "Error:\n" + str(err))
raise
else:
self.error(2)
finally:
self.__state = State.Ready
def __ensure_data(self):
# Ensure that all required database (ontology and annotations for
# the current selected organism are present. If not start a download in
# the background. Return True if all dbs are present and false
# otherwise
assert self.__state == State.Ready
annotation = self.available_annotations[self.annotation_index]
go_files = [fname for domain, fname in serverfiles.listfiles(DOMAIN)]
files = []
if annotation.filename not in go_files:
files.append(("GO", annotation.filename))
if FILENAME_ONTOLOGY not in go_files:
files.append((DOMAIN, FILENAME_ONTOLOGY))
if files:
self.__start_download(files)
assert self.__state == State.Downloading
return False
else:
return True
def Load(self):
a = self.available_annotations[self.annotation_index]
if self.ontology is None:
self.ontology = go.Ontology()
if a.taxid != self.loaded_annotation_code:
self.annotations = None
gc.collect() # Force run garbage collection
self.annotations = go.Annotations(a.taxid)
self.loaded_annotation_code = a.taxid
count = defaultdict(int)
geneSets = defaultdict(set)
for anno in self.annotations.annotations:
count[anno.evidence] += 1
geneSets[anno.evidence].add(anno.gene_id)
for etype in go.evidenceTypesOrdered:
ecb = self.evidenceCheckBoxDict[etype]
ecb.setEnabled(bool(count[etype]))
ecb.setText(etype + ": %i annots(%i genes)" % (count[etype], len(geneSets[etype])))
def Enrichment(self):
assert self.input_data is not None
assert self.__state == State.Ready
if not self.annotations.ontology:
self.annotations.ontology = self.ontology
self.error(1)
self.warning([0, 1])
self.__get_input_genes()
self.input_genes = set(self.input_genes)
self.known_input_genes = self.annotations.get_genes_with_known_annotation(self.input_genes)
# self.clusterGenes = clusterGenes = self.annotations.map_to_ncbi_id(self.input_genes).values()
self.infoLabel.setText("%i unique genes on input\n%i (%.1f%%) genes with known annotations" %
(len(self.input_genes), len(self.known_input_genes),
100.0*len(self.known_input_genes)/len(self.input_genes)
if len(self.input_genes) else 0.0))
if not self.useReferenceDataset or self.ref_data is None:
self.information(2)
self.information(1)
self.ref_genes = self.annotations.genes()
self.ref_genes = set(self.ref_genes)
elif self.ref_data is not None:
self.__get_ref_genes()
self.ref_genes = set(self.ref_genes)
ref_count = len(self.ref_genes)
if ref_count == 0:
self.ref_genes = self.annotations.genes()
self.referenceRadioBox.buttons[1].setText("Reference set")
self.referenceRadioBox.buttons[1].setDisabled(True)
self.information(2, "Unable to extract gene names from reference dataset. "
"Using entire genome for reference")
self.useReferenceDataset = 0
else:
self.referenceRadioBox.buttons[1].setText("Reference set ({} genes)".format(ref_count))
self.referenceRadioBox.buttons[1].setDisabled(False)
self.information(2)
else:
self.useReferenceDataset = 0
self.ref_genes = []
if not self.ref_genes:
self.error(1, "No valid reference set")
return {}
evidences = []
for etype in go.evidenceTypesOrdered:
if self.useEvidenceType[etype]:
evidences.append(etype)
aspect = ['Process', 'Component', 'Function'][self.aspectIndex]
self.progressBarInit(processEvents=False)
self.setBlocking(True)
self.__state = State.Running
if self.input_genes:
f = self._executor.submit(
self.annotations.get_enriched_terms,
self.input_genes, self.ref_genes, evidences, aspect=aspect,
prob=self.probFunctions[self.probFunc], use_fdr=False,
progress_callback=methodinvoke(
self, "_progressBarSet", (float,))
)
fw = FutureWatcher(f, parent=self)
fw.done.connect(self.__on_enrichment_done)
fw.done.connect(fw.deleteLater)
return
else:
f = Future()
f.set_result({})
self.__on_enrichment_done(f)
def __on_enrichment_done(self, results):
# type: (Future[Dict[str, tuple]]) -> None
self.progressBarFinished(processEvents=False)
self.setBlocking(False)
self.setStatusMessage("")
if self.__state & State.Stale:
self.__state = State.Ready
self.__invalidate()
return
self.__state = State.Ready
try:
results = results.result() # type: Dict[str, tuple]
except Exception as ex:
results = {}
error = str(ex)
self.error(1, error)
if results:
terms = list(results.items())
fdr_vals = statistics.FDR([d[1] for _, d in terms])
terms = [(key, d + (fdr,))
for (key, d), fdr in zip(terms, fdr_vals)]
terms = dict(terms)
else:
terms = {}
self.terms = terms
if not self.terms:
self.warning(0, "No enriched terms found.")
else:
self.warning(0)
self.treeStructDict = {}
ids = self.terms.keys()
self.treeStructRootKey = None
parents = {}
for id in ids:
parents[id] = set([term for _, term in self.ontology[id].related])
children = {}
for term in self.terms:
children[term] = set([id for id in ids if term in parents[id]])
for term in self.terms:
self.treeStructDict[term] = TreeNode(self.terms[term], children[term])
if not self.ontology[term].related and not getattr(self.ontology[term], "is_obsolete", False):
self.treeStructRootKey = term
self.SetGraph(terms)
self._updateEnrichmentReportOutput()
self.commit()
def _updateEnrichmentReportOutput(self):
terms = sorted(self.terms.items(), key=lambda item: item[1][1])
# Create and send the enrichemnt report table.
termsDomain = Orange.data.Domain(
[], [],
# All is meta!
[Orange.data.StringVariable("GO Term Id"),
Orange.data.StringVariable("GO Term Name"),
Orange.data.ContinuousVariable("Cluster Frequency"),
Orange.data.ContinuousVariable("Genes in Cluster",
number_of_decimals=0),
Orange.data.ContinuousVariable("Reference Frequency"),
Orange.data.ContinuousVariable("Genes in Reference",
number_of_decimals=0),
Orange.data.ContinuousVariable("p-value"),
Orange.data.ContinuousVariable("FDR"),
Orange.data.ContinuousVariable("Enrichment"),
Orange.data.StringVariable("Genes")])
terms = [[t_id,
self.ontology[t_id].name,
len(genes) / len(self.input_genes),
len(genes),
r_count / len(self.ref_genes),
r_count,
p_value,
fdr,
len(genes) / len(self.input_genes) * \
len(self.ref_genes) / r_count,
",".join(genes)
]
for t_id, (genes, p_value, r_count, fdr) in terms
if genes and r_count]
if terms:
X = numpy.empty((len(terms), 0))
M = numpy.array(terms, dtype=object)
termsTable = Orange.data.Table.from_numpy(termsDomain, X,
metas=M)
else:
termsTable = None
self.send("Enrichment Report", termsTable)
@Slot(float)
def _progressBarSet(self, value):
assert QThread.currentThread() is self.thread()
self.progressBarSet(value, processEvents=None)
@Slot()
def _progressBarFinish(self):
assert QThread.currentThread() is self.thread()
self.progressBarFinished(processEvents=None)
def FilterGraph(self, graph):
if self.filterByPValue_nofdr:
graph = go.filterByPValue(graph, self.maxPValue_nofdr)
if self.filterByPValue: # FDR
graph = dict(filter(lambda item: item[1][3] <= self.maxPValue, graph.items()))
if self.filterByNumOfInstances:
graph = dict(filter(lambda item: len(item[1][0]) >= self.minNumOfInstances, graph.items()))
return graph
def FilterAndDisplayGraph(self):
if self.input_data and self.originalGraph is not None:
self.graph = self.FilterGraph(self.originalGraph)
if self.originalGraph and not self.graph:
self.warning(1, "All found terms were filtered out.")
else:
self.warning(1)
self.ClearGraph()
self.DisplayGraph()
def SetGraph(self, graph=None):
self.originalGraph = graph
if graph:
self.FilterAndDisplayGraph()
else:
self.graph = {}
self.ClearGraph()
def ClearGraph(self):
self.listView.clear()
self.listViewItems=[]
self.sigTerms.clear()
def DisplayGraph(self):
fromParentDict = {}
self.termListViewItemDict = {}
self.listViewItems = []
def enrichment(t):
try:
return len(t[0]) / t[2] * (len(self.ref_genes) / len(self.input_genes))
except ZeroDivisionError:
# TODO: find out why this happens
return 0
maxFoldEnrichment = max([enrichment(term) for term in self.graph.values()] or [1])
def addNode(term, parent, parentDisplayNode):
if (parent, term) in fromParentDict:
return
if term in self.graph:
displayNode = GOTreeWidgetItem(self.ontology[term], self.graph[term], len(self.input_genes),
len(self.ref_genes), maxFoldEnrichment, parentDisplayNode)
displayNode.goId = term
self.listViewItems.append(displayNode)
if term in self.termListViewItemDict:
self.termListViewItemDict[term].append(displayNode)
else:
self.termListViewItemDict[term] = [displayNode]
fromParentDict[(parent, term)] = True
parent = term
else:
displayNode = parentDisplayNode
for c in self.treeStructDict[term].children:
addNode(c, parent, displayNode)
if self.treeStructDict:
addNode(self.treeStructRootKey, None, self.listView)
terms = self.graph.items()
terms = sorted(terms, key=lambda item: item[1][1])
self.sigTableTermsSorted = [t[0] for t in terms]
self.sigTerms.clear()
for i, (t_id, (genes, p_value, refCount, fdr)) in enumerate(terms):
item = GOTreeWidgetItem(self.ontology[t_id],
(genes, p_value, refCount, fdr),
len(self.input_genes),
len(self.ref_genes),
maxFoldEnrichment,
self.sigTerms)
item.goId = t_id
self.listView.expandAll()
for i in range(5):
self.listView.resizeColumnToContents(i)
self.sigTerms.resizeColumnToContents(i)
self.sigTerms.resizeColumnToContents(6)
width = min(self.listView.columnWidth(0), 350)
self.listView.setColumnWidth(0, width)
self.sigTerms.setColumnWidth(0, width)
def ViewSelectionChanged(self):
if self.selectionChanging:
return
self.selectionChanging = 1
self.selectedTerms = []
selected = self.listView.selectedItems()
self.selectedTerms = list(set([lvi.term.id for lvi in selected]))
self.ExampleSelection()
self.selectionChanging = 0
def TableSelectionChanged(self):
if self.selectionChanging:
return
self.selectionChanging = 1
self.selectedTerms = []
selectedIds = set([self.sigTerms.itemFromIndex(index).goId for index in self.sigTerms.selectedIndexes()])
for i in range(self.sigTerms.topLevelItemCount()):
item = self.sigTerms.topLevelItem(i)
selected = item.goId in selectedIds
term = item.goId
if selected:
self.selectedTerms.append(term)
for lvi in self.termListViewItemDict[term]:
try:
lvi.setSelected(selected)
if selected:
lvi.setExpanded(True)
except RuntimeError: # Underlying C/C++ object deleted
pass
self.selectionChanging = 0
self.ExampleSelection()
def ExampleSelection(self):
self.commit()
def commit(self):
if self.input_data is None or self.originalGraph is None or \
self.annotations is None:
return
if self.__state & State.Stale:
return
terms = set(self.selectedTerms)
genes = reduce(operator.ior,
(set(self.graph[term][0]) for term in terms), set())
evidences = []
for etype in go.evidenceTypesOrdered:
if self.useEvidenceType[etype]:
evidences.append(etype)
allTerms = self.annotations.get_annotated_terms(
genes, direct_annotation_only=self.selectionDirectAnnotation,
evidence_codes=evidences)
if self.selectionDisjoint > 0:
count = defaultdict(int)
for term in self.selectedTerms:
for g in allTerms.get(term, []):
count[g] += 1
ccount = 1 if self.selectionDisjoint == 1 else len(self.selectedTerms)
selected_genes = [gene for gene, c in count.items() if c == ccount and gene in genes]
else:
selected_genes = reduce(
operator.ior,
(set(allTerms.get(term, [])) for term in self.selectedTerms), set())
if self.use_attr_names:
selected = [column for column in self.input_data.domain.attributes
if self.gene_id_attribute in column.attributes and
str(column.attributes[self.gene_id_attribute]) in set(selected_genes)]
domain = Orange.data.Domain(selected, self.input_data.domain.class_vars, self.input_data.domain.metas)
new_data = self.input_data.from_table(domain, self.input_data)
self.send("Data on Selected Genes", new_data)
else:
selected_rows = []
for row_index, row in enumerate(self.input_data):
gene_in_row = str(row[self.gene_id_column])
if gene_in_row in self.input_genes and gene_in_row in selected_genes:
selected_rows.append(row_index)
if selected_rows:
selected = self.input_data[selected_rows]
else:
selected = None
self.send("Data on Selected Genes", selected)
def ShowInfo(self):
dialog = QDialog(self)
dialog.setModal(False)
dialog.setLayout(QVBoxLayout())
label = QLabel(dialog)
label.setText("Ontology:\n" + self.ontology.header
if self.ontology else "Ontology not loaded!")
dialog.layout().addWidget(label)
label = QLabel(dialog)
label.setText("Annotations:\n" + self.annotations.header.replace("!", "")
if self.annotations else "Annotations not loaded!")
dialog.layout().addWidget(label)
dialog.show()
def onDeleteWidget(self):
"""Called before the widget is removed from the canvas.
"""
self.annotations = None
self.ontology = None
gc.collect() # Force collection
class OWKEGGPathwayBrowser(widget.OWWidget):
name = "KEGG Pathways"
description = "Browse KEGG pathways that include an input set of genes."
icon = "../widgets/icons/OWKEGGPathwayBrowser.svg"
priority = 70
inputs = [("Data", Orange.data.Table, "SetData", widget.Default),
("Reference", Orange.data.Table, "SetRefData")]
outputs = [("Selected Data", Orange.data.Table, widget.Default),
("Unselected Data", Orange.data.Table)]
autoCommit = settings.Setting(False)
autoResize = settings.Setting(True)
useReference = settings.Setting(False)
showOrthology = settings.Setting(True)
Ready, Initializing, Running = 0, 1, 2
class Error(widget.OWWidget.Error):
missing_annotation = widget.Msg(ERROR_ON_MISSING_ANNOTATION)
missing_gene_id = widget.Msg(ERROR_ON_MISSING_GENE_ID)
missing_tax_id = widget.Msg(ERROR_ON_MISSING_TAX_ID)
def __init__(self, parent=None):
super().__init__(parent)
self._changedFlag = False
self.__invalidated = False
self.__runstate = OWKEGGPathwayBrowser.Initializing
self.__in_setProgress = False
self.controlArea.setMaximumWidth(250)
box = gui.widgetBox(self.controlArea, "Info")
self.infoLabel = gui.widgetLabel(box, "No data on input\n")
gui.separator(self.controlArea)
gui.checkBox(self.controlArea,
self,
"useReference",
"From signal",
box="Reference",
callback=self.Update)
gui.separator(self.controlArea)
gui.checkBox(
self.controlArea,
self,
"showOrthology",
"Show pathways in full orthology",
box="Orthology",
callback=self.UpdateListView,
)
gui.checkBox(
self.controlArea,
self,
"autoResize",
"Resize to fit",
box="Image",
callback=self.UpdatePathwayViewTransform,
)
box = gui.widgetBox(self.controlArea, "Cache Control")
gui.button(
box,
self,
"Clear cache",
callback=self.ClearCache,
tooltip="Clear all locally cached KEGG data.",
default=False,
autoDefault=False,
)
gui.separator(self.controlArea)
gui.auto_commit(self.controlArea, self, "autoCommit", "Commit")
gui.rubber(self.controlArea)
spliter = QSplitter(Qt.Vertical, self.mainArea)
self.pathwayView = PathwayView(self, spliter)
self.pathwayView.scene().selectionChanged.connect(
self._onSelectionChanged)
self.mainArea.layout().addWidget(spliter)
self.listView = QTreeWidget(allColumnsShowFocus=True,
selectionMode=QTreeWidget.SingleSelection,
sortingEnabled=True,
maximumHeight=200)
spliter.addWidget(self.listView)
self.listView.setColumnCount(4)
self.listView.setHeaderLabels(
["Pathway", "P value", "Genes", "Reference"])
self.listView.itemSelectionChanged.connect(self.UpdatePathwayView)
select = QAction("Select All", self, shortcut=QKeySequence.SelectAll)
select.triggered.connect(self.selectAll)
self.addAction(select)
self.data = None
self.input_genes = []
self.tax_id = None
self.use_attr_names = None
self.gene_id_attribute = None
self.gene_id_column = None
self.ref_data = None
self.ref_genes = []
self.ref_tax_id = None
self.ref_use_attr_names = None
self.ref_gene_id_attribute = None
self.ref_gene_id_column = None
self.pathways = {}
self.ncbi_gene_map = []
self.org = None
self._executor = concurrent.ThreadExecutor()
self.setEnabled(False)
self.setBlocking(True)
progress = concurrent.methodinvoke(self, "setProgress", (float, ))
def get_genome():
"""Return a KEGGGenome with the common org entries precached."""
genome = kegg.KEGGGenome()
essential = genome.essential_organisms()
common = genome.common_organisms()
# Remove duplicates of essential from common.
# (essential + common list as defined here will be used in the
# GUI.)
common = [c for c in common if c not in essential]
# TODO: Add option to specify additional organisms not
# in the common list.
keys = list(map(genome.org_code_to_entry_key, essential + common))
genome.pre_cache(keys, progress_callback=progress)
return (keys, genome)
self._genomeTask = task = concurrent.Task(function=get_genome)
task.finished.connect(self.__initialize_finish)
self.progressBarInit()
self.infoLabel.setText("Fetching organism definitions\n")
self._executor.submit(task)
def __initialize_finish(self):
if self.__runstate != OWKEGGPathwayBrowser.Initializing:
return
try:
keys, genome = self._genomeTask.result()
except Exception as err:
self.error(0, str(err))
raise
self.progressBarFinished()
self.setEnabled(True)
self.setBlocking(False)
self.infoLabel.setText("No data on input\n")
def clear(self):
"""
Clear the widget state.
"""
self.pathways = {}
self.org = None
self.infoLabel.setText("No data on input\n")
self.listView.clear()
self.pathwayView.SetPathway(None)
self.send("Selected Data", None)
self.send("Unselected Data", None)
def SetData(self, data=None):
if self.__runstate == OWKEGGPathwayBrowser.Initializing:
self.__initialize_finish()
self.Error.clear()
if data:
self.data = data
self.tax_id = str(self.data.attributes.get(TAX_ID, None))
self.use_attr_names = self.data.attributes.get(
GENE_AS_ATTRIBUTE_NAME, None)
self.gene_id_attribute = self.data.attributes.get(
GENE_ID_ATTRIBUTE, None)
self.gene_id_column = self.data.attributes.get(
GENE_ID_COLUMN, None)
if not (self.use_attr_names is not None and
((self.gene_id_attribute is None) ^
(self.gene_id_column is None))):
if self.tax_id is None:
self.Error.missing_annotation()
return
self.Error.missing_gene_id()
return
elif self.tax_id is None:
self.Error.missing_tax_id()
return
self.warning(0)
self.error(0)
self.information(0)
self.__invalidated = True
else:
self.clear()
def SetRefData(self, data=None):
self.information(1)
if data is not None and self.useReference:
self.ref_data = data
self.ref_tax_id = str(self.ref_data.attributes.get(TAX_ID, None))
self.ref_use_attr_names = self.ref_data.attributes.get(
GENE_AS_ATTRIBUTE_NAME, None)
self.ref_gene_id_attribute = self.ref_data.attributes.get(
GENE_ID_ATTRIBUTE, None)
self.ref_gene_id_column = self.ref_data.attributes.get(
GENE_ID_COLUMN, None)
if not (self.ref_use_attr_names is not None and
((self.ref_gene_id_attribute is None) ^
(self.ref_gene_id_column is None))):
if self.ref_tax_id is None:
self.Error.missing_annotation()
return
self.Error.missing_gene_id()
return
elif self.ref_tax_id is None:
self.Error.missing_tax_id()
return
self.__invalidated = True
def handleNewSignals(self):
if self.__invalidated:
self.Update()
self.__invalidated = False
def UpdateListView(self):
self.bestPValueItem = None
self.listView.clear()
if not self.data:
return
allPathways = self.org.pathways()
allRefPathways = kegg.pathways("map")
items = []
kegg_pathways = kegg.KEGGPathways()
org_code = self.org.org_code
if self.showOrthology:
self.koOrthology = kegg.KEGGBrite("ko00001")
self.listView.setRootIsDecorated(True)
path_ids = {s[-5:] for s in self.pathways.keys()}
def _walkCollect(koEntry):
num = koEntry.title[:5] if koEntry.title else None
if num in path_ids:
return [koEntry] + reduce(
lambda li, c: li + _walkCollect(c),
[child for child in koEntry.entries], [])
else:
c = reduce(lambda li, c: li + _walkCollect(c),
[child for child in koEntry.entries], [])
return c + (c and [koEntry] or [])
allClasses = reduce(lambda li1, li2: li1 + li2,
[_walkCollect(c) for c in self.koOrthology],
[])
def _walkCreate(koEntry, lvItem):
item = QTreeWidgetItem(lvItem)
id = "path:" + org_code + koEntry.title[:5]
if koEntry.title[:5] in path_ids:
p = kegg_pathways.get_entry(id)
if p is None:
# In case the genesets still have obsolete entries
name = koEntry.title
else:
name = p.name
genes, p_value, ref = self.pathways[id]
item.setText(0, name)
item.setText(1, "%.5f" % p_value)
item.setText(
2, "%i of %i" % (len(genes), len(self.input_genes)))
item.setText(3, "%i of %i" % (ref, len(self.ref_genes)))
item.pathway_id = id if p is not None else None
else:
if id in allPathways:
text = kegg_pathways.get_entry(id).name
else:
text = koEntry.title
item.setText(0, text)
if id in allPathways:
item.pathway_id = id
elif "path:map" + koEntry.title[:5] in allRefPathways:
item.pathway_id = "path:map" + koEntry.title[:5]
else:
item.pathway_id = None
for child in koEntry.entries:
if child in allClasses:
_walkCreate(child, item)
for koEntry in self.koOrthology:
if koEntry in allClasses:
_walkCreate(koEntry, self.listView)
self.listView.update()
else:
self.listView.setRootIsDecorated(False)
pathways = self.pathways.items()
pathways = sorted(pathways, key=lambda item: item[1][1])
for id, (genes, p_value, ref) in pathways:
item = QTreeWidgetItem(self.listView)
item.setText(0, kegg_pathways.get_entry(id).name)
item.setText(1, "%.5f" % p_value)
item.setText(2,
"%i of %i" % (len(genes), len(self.input_genes)))
item.setText(3, "%i of %i" % (ref, len(self.ref_genes)))
item.pathway_id = id
items.append(item)
self.bestPValueItem = items and items[0] or None
self.listView.expandAll()
for i in range(4):
self.listView.resizeColumnToContents(i)
if self.bestPValueItem:
index = self.listView.indexFromItem(self.bestPValueItem)
self.listView.selectionModel().select(
index, QItemSelectionModel.ClearAndSelect)
def UpdatePathwayView(self):
items = self.listView.selectedItems()
if len(items) > 0:
item = items[0]
else:
item = None
self.commit()
item = item or self.bestPValueItem
if not item or not item.pathway_id:
self.pathwayView.SetPathway(None)
return
def get_kgml_and_image(pathway_id):
"""Return an initialized KEGGPathway with pre-cached data"""
p = kegg.KEGGPathway(pathway_id)
p._get_kgml() # makes sure the kgml file is downloaded
p._get_image_filename() # makes sure the image is downloaded
return (pathway_id, p)
self.setEnabled(False)
self._pathwayTask = concurrent.Task(
function=lambda: get_kgml_and_image(item.pathway_id))
self._pathwayTask.finished.connect(self._onPathwayTaskFinshed)
self._executor.submit(self._pathwayTask)
def _onPathwayTaskFinshed(self):
self.setEnabled(True)
pathway_id, self.pathway = self._pathwayTask.result()
objects = self.pathways.get(pathway_id, [[]])[0] # [ncbi_gene_id]
# map ncbi_gene_id to keg_id for display
objects = flatten(relation_map(self.ncbi_gene_map, objects))
self.pathwayView.SetPathway(self.pathway, objects)
def UpdatePathwayViewTransform(self):
self.pathwayView.updateTransform()
def Update(self):
"""
Update (recompute enriched pathways) the widget state.
"""
if not self.data:
return
self.error(0)
self.information(0)
# XXX: Check data in setData, do not even allow this to be executed if
# data has no genes
try:
self.__get_input_genes()
self.input_genes = set(self.input_genes)
except ValueError:
self.error(0, "Cannot extract gene names from input.")
self.information(1)
self.org = kegg.KEGGOrganism(kegg.from_taxid(self.tax_id))
if self.useReference and self.ref_data:
self.__get_ref_genes()
self.ref_genes = set(self.ref_genes)
else:
self.ref_genes = self.org.get_ncbi_ids()
def run_enrichment(org_code, genes, reference, progress=None):
# We use the kegg pathway gene sets provided by 'geneset' for
# the enrichment calculation.
kegg_api = kegg.api.CachedKeggApi()
link_map = kegg_api.link(org_code,
"pathway") # [(pathway_id, kegg_gene_id)]
ncbi_gene_map = kegg_api.conv(
org_code, 'ncbi-geneid') # [(ncbi_gene_id, kegg_gene_id)]
ncbi_gene_map = [(_1.split(":", 1)[1], _2)
for _1, _2 in ncbi_gene_map]
link_map = relation_join(
link_map,
[(_2, _1)
for _1, _2 in ncbi_gene_map]) # [(pathway_id, ncbi_gene_id)]
kegg_sets = relation_list_to_multimap(
link_map) # {pathway_id -> [ncbi_gene_ids]}
# map kegg gene ids to ncbi_gene_ids.
kegg_sets = geneset.GeneSets(sets=[
geneset.GeneSet(gs_id=ddi, genes=set(genes))
for ddi, genes in kegg_sets.items()
])
pathways = pathway_enrichment(kegg_sets,
genes,
reference,
callback=progress)
# Ensure that pathway entries are pre-cached for later use in the
# list/tree view
kegg_pathways = kegg.KEGGPathways()
kegg_pathways.pre_cache(pathways.keys(),
progress_callback=progress)
return pathways, ncbi_gene_map
self.progressBarInit()
self.setEnabled(False)
self.infoLabel.setText("Retrieving...\n")
progress = concurrent.methodinvoke(self, "setProgress", (float, ))
run_func = partial(run_enrichment, self.org.org_code, self.input_genes,
self.ref_genes, progress)
self._enrichTask = concurrent.Task(function=run_func)
self._enrichTask.finished.connect(self._onEnrichTaskFinished)
self._executor.submit(self._enrichTask)
def _onEnrichTaskFinished(self):
self.setEnabled(True)
self.setBlocking(False)
try:
pathways, ncbi_gene_map = self._enrichTask.result()
except Exception:
raise
self.progressBarFinished()
self.pathways = pathways
self.ncbi_gene_map = ncbi_gene_map
if not self.pathways:
self.warning(0, "No enriched pathways found.")
else:
self.warning(0)
self.infoLabel.setText("{} unique gene names on input\n".format(
len(set(self.input_genes))))
self.UpdateListView()
@Slot(float)
def setProgress(self, value):
if self.__in_setProgress:
return
self.__in_setProgress = True
self.progressBarSet(value)
self.__in_setProgress = False
def __get_input_genes(self):
"""
Extract and return gene names from `data`.
"""
self.input_genes = []
if self.use_attr_names:
for variable in self.data.domain.attributes:
self.input_genes.append(
str(variable.attributes.get(self.gene_id_attribute, '?')))
else:
genes, _ = self.data.get_column_view(self.gene_id_column)
self.input_genes = [str(g) for g in genes]
if len(self.input_genes) <= 0:
raise ValueError("No gene names in data.")
def __get_ref_genes(self):
"""
Extract and return gene names from `data`.
"""
self.ref_genes = []
if self.ref_use_attr_names:
for variable in self.ref_data.domain.attributes:
self.ref_genes.append(
str(
variable.attributes.get(self.ref_gene_id_attribute,
'?')))
else:
genes, _ = self.ref_data.get_column_view(self.ref_gene_id_column)
self.ref_genes = [str(g) for g in genes]
def selectAll(self):
"""
Select all items in the pathway view.
"""
changed = False
scene = self.pathwayView.scene()
with disconnected(scene.selectionChanged, self._onSelectionChanged):
for item in scene.items():
if item.flags(
) & QGraphicsItem.ItemIsSelectable and not item.isSelected():
item.setSelected(True)
changed = True
if changed:
self._onSelectionChanged()
def _onSelectionChanged(self):
# Item selection in the pathwayView/scene has changed
self.commit()
def commit(self):
if self.data:
selectedItems = self.pathwayView.scene().selectedItems()
selectedGenes = reduce(
set.union, [item.marked_objects for item in selectedItems],
set())
# map kegg_ids back to ncbi_gene_id
backmap = [(_2, _1) for _1, _2 in self.ncbi_gene_map]
selectedGenes = set(flatten(relation_map(backmap, selectedGenes)))
if self.use_attr_names:
selected = [
column for column in self.data.domain.attributes
if self.gene_id_attribute in column.attributes
and str(column.attributes[
self.gene_id_attribute]) in selectedGenes
]
data = self.data[:, selected]
self.send("Selected Data", data)
else:
selected_indices = []
other_indices = []
for row_index, row in enumerate(self.data):
gene_in_row = str(row[self.gene_id_column])
if gene_in_row in self.input_genes and gene_in_row in selectedGenes:
selected_indices.append(row_index)
else:
other_indices.append(row_index)
if selected_indices:
selected = self.data[selected_indices]
else:
selected = None
if other_indices:
other = self.data[other_indices]
else:
other = None
self.send("Selected Data", selected)
self.send("Unselected Data", other)
else:
self.send("Selected Data", None)
self.send("Unselected Data", None)
def ClearCache(self):
kegg.caching.clear_cache()
def onDeleteWidget(self):
"""
Called before the widget is removed from the canvas.
"""
super().onDeleteWidget()
self.org = None
self._executor.shutdown(wait=False)
gc.collect() # Force collection (WHY?)
def sizeHint(self):
return QSize(1024, 720)
class OWKEGGPathwayBrowser(widget.OWWidget):
name = "KEGG Pathways"
description = "Browse KEGG pathways that include an input set of genes."
icon = "../widgets/icons/KEGGPathways.svg"
priority = 2030
inputs = [("Data", Orange.data.Table, "SetData", widget.Default), ("Reference", Orange.data.Table, "SetRefData")]
outputs = [("Selected Data", Orange.data.Table, widget.Default), ("Unselected Data", Orange.data.Table)]
settingsHandler = settings.DomainContextHandler()
organismIndex = settings.ContextSetting(0)
geneAttrIndex = settings.ContextSetting(0)
useAttrNames = settings.ContextSetting(False)
autoCommit = settings.Setting(False)
autoResize = settings.Setting(True)
useReference = settings.Setting(False)
showOrthology = settings.Setting(True)
Ready, Initializing, Running = 0, 1, 2
def __init__(self, parent=None):
super().__init__(parent)
self.organismCodes = []
self._changedFlag = False
self.__invalidated = False
self.__runstate = OWKEGGPathwayBrowser.Initializing
self.__in_setProgress = False
self.controlArea.setMaximumWidth(250)
box = gui.widgetBox(self.controlArea, "Info")
self.infoLabel = gui.widgetLabel(box, "No data on input\n")
# Organism selection.
box = gui.widgetBox(self.controlArea, "Organism")
self.organismComboBox = gui.comboBox(
box,
self,
"organismIndex",
items=[],
callback=self.Update,
addSpace=True,
tooltip="Select the organism of the input genes",
)
# Selection of genes attribute
box = gui.widgetBox(self.controlArea, "Gene attribute")
self.geneAttrCandidates = itemmodels.VariableListModel(parent=self)
self.geneAttrCombo = gui.comboBox(box, self, "geneAttrIndex", callback=self.Update)
self.geneAttrCombo.setModel(self.geneAttrCandidates)
gui.checkBox(
box, self, "useAttrNames", "Use variable names", disables=[(-1, self.geneAttrCombo)], callback=self.Update
)
self.geneAttrCombo.setDisabled(bool(self.useAttrNames))
gui.separator(self.controlArea)
gui.checkBox(self.controlArea, self, "useReference", "From signal", box="Reference", callback=self.Update)
gui.separator(self.controlArea)
gui.checkBox(
self.controlArea,
self,
"showOrthology",
"Show pathways in full orthology",
box="Orthology",
callback=self.UpdateListView,
)
gui.checkBox(
self.controlArea, self, "autoResize", "Resize to fit", box="Image", callback=self.UpdatePathwayViewTransform
)
box = gui.widgetBox(self.controlArea, "Cache Control")
gui.button(
box,
self,
"Clear cache",
callback=self.ClearCache,
tooltip="Clear all locally cached KEGG data.",
default=False,
autoDefault=False,
)
gui.separator(self.controlArea)
gui.auto_commit(self.controlArea, self, "autoCommit", "Commit")
gui.rubber(self.controlArea)
spliter = QSplitter(Qt.Vertical, self.mainArea)
self.pathwayView = PathwayView(self, spliter)
self.pathwayView.scene().selectionChanged.connect(self._onSelectionChanged)
self.mainArea.layout().addWidget(spliter)
self.listView = QTreeWidget(
allColumnsShowFocus=True, selectionMode=QTreeWidget.SingleSelection, sortingEnabled=True, maximumHeight=200
)
spliter.addWidget(self.listView)
self.listView.setColumnCount(4)
self.listView.setHeaderLabels(["Pathway", "P value", "Genes", "Reference"])
self.listView.itemSelectionChanged.connect(self.UpdatePathwayView)
select = QAction("Select All", self, shortcut=QKeySequence.SelectAll)
select.triggered.connect(self.selectAll)
self.addAction(select)
self.data = None
self.refData = None
self._executor = concurrent.ThreadExecutor()
self.setEnabled(False)
self.setBlocking(True)
progress = concurrent.methodinvoke(self, "setProgress", (float,))
def get_genome():
"""Return a KEGGGenome with the common org entries precached."""
genome = kegg.KEGGGenome()
essential = genome.essential_organisms()
common = genome.common_organisms()
# Remove duplicates of essential from common.
# (essential + common list as defined here will be used in the
# GUI.)
common = [c for c in common if c not in essential]
# TODO: Add option to specify additional organisms not
# in the common list.
keys = list(map(genome.org_code_to_entry_key, essential + common))
genome.pre_cache(keys, progress_callback=progress)
return (keys, genome)
self._genomeTask = task = concurrent.Task(function=get_genome)
task.finished.connect(self.__initialize_finish)
self.progressBarInit()
self.infoLabel.setText("Fetching organism definitions\n")
self._executor.submit(task)
def __initialize_finish(self):
if self.__runstate != OWKEGGPathwayBrowser.Initializing:
return
try:
keys, genome = self._genomeTask.result()
except Exception as err:
self.error(0, str(err))
raise
self.progressBarFinished()
self.setEnabled(True)
self.setBlocking(False)
entries = [genome[key] for key in keys]
items = [entry.definition for entry in entries]
codes = [entry.organism_code for entry in entries]
self.organismCodes = codes
self.organismComboBox.clear()
self.organismComboBox.addItems(items)
self.organismComboBox.setCurrentIndex(self.organismIndex)
self.infoLabel.setText("No data on input\n")
def Clear(self):
"""
Clear the widget state.
"""
self.queryGenes = []
self.referenceGenes = []
self.genes = {}
self.uniqueGenesDict = {}
self.revUniqueGenesDict = {}
self.pathways = {}
self.org = None
self.geneAttrCandidates[:] = []
self.infoLabel.setText("No data on input\n")
self.listView.clear()
self.pathwayView.SetPathway(None)
self.send("Selected Data", None)
self.send("Unselected Data", None)
def SetData(self, data=None):
if self.__runstate == OWKEGGPathwayBrowser.Initializing:
self.__initialize_finish()
self.closeContext()
self.data = data
self.warning(0)
self.error(0)
self.information(0)
if data is not None:
vars = data.domain.variables + data.domain.metas
vars = [var for var in vars if isinstance(var, Orange.data.StringVariable)]
self.geneAttrCandidates[:] = vars
# Try to guess the gene name variable
if vars:
names_lower = [v.name.lower() for v in vars]
scores = [(name == "gene", "gene" in name) for name in names_lower]
imax, _ = max(enumerate(scores), key=itemgetter(1))
else:
imax = -1
self.geneAttrIndex = imax
taxid = data_hints.get_hint(data, "taxid", None)
if taxid:
try:
code = kegg.from_taxid(taxid)
self.organismIndex = self.organismCodes.index(code)
except Exception as ex:
print(ex, taxid)
self.useAttrNames = data_hints.get_hint(data, "genesinrows", self.useAttrNames)
self.openContext(data)
if len(self.geneAttrCandidates) == 0:
self.useAttrNames = True
self.geneAttrIndex = -1
else:
self.geneAttrIndex = min(self.geneAttrIndex, len(self.geneAttrCandidates) - 1)
else:
self.Clear()
self.__invalidated = True
def SetRefData(self, data=None):
self.refData = data
self.information(1)
if data is not None and self.useReference:
self.__invalidated = True
def handleNewSignals(self):
if self.__invalidated:
self.Update()
self.__invalidated = False
def UpdateListView(self):
self.bestPValueItem = None
self.listView.clear()
if not self.data:
return
allPathways = self.org.pathways()
allRefPathways = kegg.pathways("map")
items = []
kegg_pathways = kegg.KEGGPathways()
org_code = self.organismCodes[min(self.organismIndex, len(self.organismCodes) - 1)]
if self.showOrthology:
self.koOrthology = kegg.KEGGBrite("ko00001")
self.listView.setRootIsDecorated(True)
path_ids = set([s[-5:] for s in self.pathways.keys()])
def _walkCollect(koEntry):
num = koEntry.title[:5] if koEntry.title else None
if num in path_ids:
return [koEntry] + reduce(
lambda li, c: li + _walkCollect(c), [child for child in koEntry.entries], []
)
else:
c = reduce(lambda li, c: li + _walkCollect(c), [child for child in koEntry.entries], [])
return c + (c and [koEntry] or [])
allClasses = reduce(lambda li1, li2: li1 + li2, [_walkCollect(c) for c in self.koOrthology], [])
def _walkCreate(koEntry, lvItem):
item = QTreeWidgetItem(lvItem)
id = "path:" + org_code + koEntry.title[:5]
if koEntry.title[:5] in path_ids:
p = kegg_pathways.get_entry(id)
if p is None:
# In case the genesets still have obsolete entries
name = koEntry.title
else:
name = p.name
genes, p_value, ref = self.pathways[id]
item.setText(0, name)
item.setText(1, "%.5f" % p_value)
item.setText(2, "%i of %i" % (len(genes), len(self.genes)))
item.setText(3, "%i of %i" % (ref, len(self.referenceGenes)))
item.pathway_id = id if p is not None else None
else:
if id in allPathways:
text = kegg_pathways.get_entry(id).name
else:
text = koEntry.title
item.setText(0, text)
if id in allPathways:
item.pathway_id = id
elif "path:map" + koEntry.title[:5] in allRefPathways:
item.pathway_id = "path:map" + koEntry.title[:5]
else:
item.pathway_id = None
for child in koEntry.entries:
if child in allClasses:
_walkCreate(child, item)
for koEntry in self.koOrthology:
if koEntry in allClasses:
_walkCreate(koEntry, self.listView)
self.listView.update()
else:
self.listView.setRootIsDecorated(False)
pathways = self.pathways.items()
pathways = sorted(pathways, key=lambda item: item[1][1])
for id, (genes, p_value, ref) in pathways:
item = QTreeWidgetItem(self.listView)
item.setText(0, kegg_pathways.get_entry(id).name)
item.setText(1, "%.5f" % p_value)
item.setText(2, "%i of %i" % (len(genes), len(self.genes)))
item.setText(3, "%i of %i" % (ref, len(self.referenceGenes)))
item.pathway_id = id
items.append(item)
self.bestPValueItem = items and items[0] or None
self.listView.expandAll()
for i in range(4):
self.listView.resizeColumnToContents(i)
if self.bestPValueItem:
index = self.listView.indexFromItem(self.bestPValueItem)
self.listView.selectionModel().select(index, QItemSelectionModel.ClearAndSelect)
def UpdatePathwayView(self):
items = self.listView.selectedItems()
if len(items) > 0:
item = items[0]
else:
item = None
self.commit()
item = item or self.bestPValueItem
if not item or not item.pathway_id:
self.pathwayView.SetPathway(None)
return
def get_kgml_and_image(pathway_id):
"""Return an initialized KEGGPathway with pre-cached data"""
p = kegg.KEGGPathway(pathway_id)
p._get_kgml() # makes sure the kgml file is downloaded
p._get_image_filename() # makes sure the image is downloaded
return (pathway_id, p)
self.setEnabled(False)
self._pathwayTask = concurrent.Task(function=lambda: get_kgml_and_image(item.pathway_id))
self._pathwayTask.finished.connect(self._onPathwayTaskFinshed)
self._executor.submit(self._pathwayTask)
def _onPathwayTaskFinshed(self):
self.setEnabled(True)
pathway_id, self.pathway = self._pathwayTask.result()
self.pathwayView.SetPathway(self.pathway, self.pathways.get(pathway_id, [[]])[0])
def UpdatePathwayViewTransform(self):
self.pathwayView.updateTransform()
def Update(self):
"""
Update (recompute enriched pathways) the widget state.
"""
if not self.data:
return
self.error(0)
self.information(0)
# XXX: Check data in setData, do not even alow this to be executed if
# data has no genes
try:
genes = self.GeneNamesFromData(self.data)
except ValueError:
self.error(0, "Cannot extract gene names from input.")
genes = []
if not self.useAttrNames and any("," in gene for gene in genes):
genes = reduce(add, (split_and_strip(gene, ",") for gene in genes), [])
self.information(0, "Separators detected in input gene names. " "Assuming multiple genes per instance.")
self.queryGenes = genes
self.information(1)
reference = None
if self.useReference and self.refData:
reference = self.GeneNamesFromData(self.refData)
if not self.useAttrNames and any("," in gene for gene in reference):
reference = reduce(add, (split_and_strip(gene, ",") for gene in reference), [])
self.information(
1, "Separators detected in reference gene " "names. Assuming multiple genes per " "instance."
)
org_code = self.SelectedOrganismCode()
def run_enrichment(org_code, genes, reference=None, progress=None):
org = kegg.KEGGOrganism(org_code)
if reference is None:
reference = org.get_genes()
# Map 'genes' and 'reference' sets to unique KEGG identifiers
unique_genes, _, _ = org.get_unique_gene_ids(set(genes))
unique_ref_genes, _, _ = org.get_unique_gene_ids(set(reference))
taxid = kegg.to_taxid(org.org_code)
# Map the taxid back to standard 'common' taxids
# (as used by 'geneset') if applicable
r_tax_map = dict((v, k) for k, v in kegg.KEGGGenome.TAXID_MAP.items())
if taxid in r_tax_map:
taxid = r_tax_map[taxid]
# We use the kegg pathway gene sets provided by 'geneset' for
# the enrichment calculation.
# Ensure we are using the latest genesets
# TODO: ?? Is updating the index enough?
serverfiles.update(geneset.sfdomain, "index.pck")
kegg_gs_collections = geneset.collections((("KEGG", "pathways"), taxid))
pathways = pathway_enrichment(
kegg_gs_collections, unique_genes.keys(), unique_ref_genes.keys(), callback=progress
)
# Ensure that pathway entries are pre-cached for later use in the
# list/tree view
kegg_pathways = kegg.KEGGPathways()
kegg_pathways.pre_cache(pathways.keys(), progress_callback=progress)
return pathways, org, unique_genes, unique_ref_genes
self.progressBarInit()
self.setEnabled(False)
self.infoLabel.setText("Retrieving...\n")
progress = concurrent.methodinvoke(self, "setProgress", (float,))
self._enrichTask = concurrent.Task(function=lambda: run_enrichment(org_code, genes, reference, progress))
self._enrichTask.finished.connect(self._onEnrichTaskFinished)
self._executor.submit(self._enrichTask)
def _onEnrichTaskFinished(self):
self.setEnabled(True)
self.setBlocking(False)
try:
pathways, org, unique_genes, unique_ref_genes = self._enrichTask.result()
except Exception:
raise
self.progressBarFinished()
self.org = org
self.genes = unique_genes.keys()
self.uniqueGenesDict = unique_genes
self.revUniqueGenesDict = dict([(val, key) for key, val in self.uniqueGenesDict.items()])
self.referenceGenes = unique_ref_genes.keys()
self.pathways = pathways
if not self.pathways:
self.warning(0, "No enriched pathways found.")
else:
self.warning(0)
count = len(set(self.queryGenes))
self.infoLabel.setText(
"%i unique gene names on input\n"
"%i (%.1f%%) genes names matched"
% (count, len(unique_genes), 100.0 * len(unique_genes) / count if count else 0.0)
)
self.UpdateListView()
@Slot(float)
def setProgress(self, value):
if self.__in_setProgress:
return
self.__in_setProgress = True
self.progressBarSet(value)
self.__in_setProgress = False
def GeneNamesFromData(self, data):
"""
Extract and return gene names from `data`.
"""
if self.useAttrNames:
genes = [str(v.name).strip() for v in data.domain.attributes]
elif self.geneAttrCandidates:
assert 0 <= self.geneAttrIndex < len(self.geneAttrCandidates)
geneAttr = self.geneAttrCandidates[self.geneAttrIndex]
genes = [str(e[geneAttr]) for e in data if not numpy.isnan(e[geneAttr])]
else:
raise ValueError("No gene names in data.")
return genes
def SelectedOrganismCode(self):
"""
Return the selected organism code.
"""
return self.organismCodes[min(self.organismIndex, len(self.organismCodes) - 1)]
def selectAll(self):
"""
Select all items in the pathway view.
"""
changed = False
scene = self.pathwayView.scene()
with disconnected(scene.selectionChanged, self._onSelectionChanged):
for item in scene.items():
if item.flags() & QGraphicsItem.ItemIsSelectable and not item.isSelected():
item.setSelected(True)
changed = True
if changed:
self._onSelectionChanged()
def _onSelectionChanged(self):
# Item selection in the pathwayView/scene has changed
self.commit()
def commit(self):
if self.data:
selectedItems = self.pathwayView.scene().selectedItems()
selectedGenes = reduce(set.union, [item.marked_objects for item in selectedItems], set())
if self.useAttrNames:
selected = [self.data.domain[self.uniqueGenesDict[gene]] for gene in selectedGenes]
# newDomain = Orange.data.Domain(selectedVars, 0)
data = self.data[:, selected]
# data = Orange.data.Table(newDomain, self.data)
self.send("Selected Data", data)
elif self.geneAttrCandidates:
assert 0 <= self.geneAttrIndex < len(self.geneAttrCandidates)
geneAttr = self.geneAttrCandidates[self.geneAttrIndex]
selectedIndices = []
otherIndices = []
for i, ex in enumerate(self.data):
names = [
self.revUniqueGenesDict.get(name, None) for name in split_and_strip(str(ex[geneAttr]), ",")
]
if any(name and name in selectedGenes for name in names):
selectedIndices.append(i)
else:
otherIndices.append(i)
if selectedIndices:
selected = self.data[selectedIndices]
else:
selected = None
if otherIndices:
other = self.data[otherIndices]
else:
other = None
self.send("Selected Data", selected)
self.send("Unselected Data", other)
else:
self.send("Selected Data", None)
self.send("Unselected Data", None)
def ClearCache(self):
kegg.caching.clear_cache()
def onDeleteWidget(self):
"""
Called before the widget is removed from the canvas.
"""
super().onDeleteWidget()
self.org = None
self._executor.shutdown(wait=False)
gc.collect() # Force collection (WHY?)
def sizeHint(self):
return QSize(1024, 720)
class OWKEGGPathwayBrowser(widget.OWWidget):
name = "KEGG Pathways"
description = "Browse KEGG pathways that include an input set of genes."
icon = "../widgets/icons/OWKEGGPathwayBrowser.svg"
priority = 8
inputs = [("Data", Orange.data.Table, "SetData", widget.Default),
("Reference", Orange.data.Table, "SetRefData")]
outputs = [("Selected Data", Orange.data.Table, widget.Default),
("Unselected Data", Orange.data.Table)]
settingsHandler = settings.DomainContextHandler()
organismIndex = settings.ContextSetting(0)
geneAttrIndex = settings.ContextSetting(0)
useAttrNames = settings.ContextSetting(False)
autoCommit = settings.Setting(False)
autoResize = settings.Setting(True)
useReference = settings.Setting(False)
showOrthology = settings.Setting(True)
Ready, Initializing, Running = 0, 1, 2
def __init__(self, parent=None):
super().__init__(parent)
self.organismCodes = []
self._changedFlag = False
self.__invalidated = False
self.__runstate = OWKEGGPathwayBrowser.Initializing
self.__in_setProgress = False
self.controlArea.setMaximumWidth(250)
box = gui.widgetBox(self.controlArea, "Info")
self.infoLabel = gui.widgetLabel(box, "No data on input\n")
# Organism selection.
box = gui.widgetBox(self.controlArea, "Organism")
self.organismComboBox = gui.comboBox(
box, self, "organismIndex",
items=[],
callback=self.Update,
addSpace=True,
tooltip="Select the organism of the input genes")
# Selection of genes attribute
box = gui.widgetBox(self.controlArea, "Gene attribute")
self.geneAttrCandidates = itemmodels.VariableListModel(parent=self)
self.geneAttrCombo = gui.comboBox(
box, self, "geneAttrIndex", callback=self.Update)
self.geneAttrCombo.setModel(self.geneAttrCandidates)
gui.checkBox(box, self, "useAttrNames",
"Use variable names", disables=[(-1, self.geneAttrCombo)],
callback=self.Update)
self.geneAttrCombo.setDisabled(bool(self.useAttrNames))
gui.separator(self.controlArea)
gui.checkBox(self.controlArea, self, "useReference",
"From signal", box="Reference", callback=self.Update)
gui.separator(self.controlArea)
gui.checkBox(self.controlArea, self, "showOrthology",
"Show pathways in full orthology", box="Orthology",
callback=self.UpdateListView)
gui.checkBox(self.controlArea, self, "autoResize",
"Resize to fit", box="Image",
callback=self.UpdatePathwayViewTransform)
box = gui.widgetBox(self.controlArea, "Cache Control")
gui.button(box, self, "Clear cache",
callback=self.ClearCache,
tooltip="Clear all locally cached KEGG data.",
default=False, autoDefault=False)
gui.separator(self.controlArea)
gui.auto_commit(self.controlArea, self, "autoCommit", "Commit")
gui.rubber(self.controlArea)
spliter = QSplitter(Qt.Vertical, self.mainArea)
self.pathwayView = PathwayView(self, spliter)
self.pathwayView.scene().selectionChanged.connect(
self._onSelectionChanged
)
self.mainArea.layout().addWidget(spliter)
self.listView = QTreeWidget(
allColumnsShowFocus=True,
selectionMode=QTreeWidget.SingleSelection,
sortingEnabled=True,
maximumHeight=200)
spliter.addWidget(self.listView)
self.listView.setColumnCount(4)
self.listView.setHeaderLabels(
["Pathway", "P value", "Genes", "Reference"])
self.listView.itemSelectionChanged.connect(self.UpdatePathwayView)
select = QAction(
"Select All", self,
shortcut=QKeySequence.SelectAll
)
select.triggered.connect(self.selectAll)
self.addAction(select)
self.data = None
self.refData = None
self._executor = concurrent.ThreadExecutor()
self.setEnabled(False)
self.setBlocking(True)
progress = concurrent.methodinvoke(self, "setProgress", (float,))
def get_genome():
"""Return a KEGGGenome with the common org entries precached."""
genome = kegg.KEGGGenome()
essential = genome.essential_organisms()
common = genome.common_organisms()
# Remove duplicates of essential from common.
# (essential + common list as defined here will be used in the
# GUI.)
common = [c for c in common if c not in essential]
# TODO: Add option to specify additional organisms not
# in the common list.
keys = list(map(genome.org_code_to_entry_key, essential + common))
genome.pre_cache(keys, progress_callback=progress)
return (keys, genome)
self._genomeTask = task = concurrent.Task(function=get_genome)
task.finished.connect(self.__initialize_finish)
self.progressBarInit()
self.infoLabel.setText("Fetching organism definitions\n")
self._executor.submit(task)
def __initialize_finish(self):
if self.__runstate != OWKEGGPathwayBrowser.Initializing:
return
try:
keys, genome = self._genomeTask.result()
except Exception as err:
self.error(0, str(err))
raise
self.progressBarFinished()
self.setEnabled(True)
self.setBlocking(False)
entries = [genome[key] for key in keys]
items = [entry.definition for entry in entries]
codes = [entry.organism_code for entry in entries]
self.organismCodes = codes
self.organismComboBox.clear()
self.organismComboBox.addItems(items)
self.organismComboBox.setCurrentIndex(self.organismIndex)
self.infoLabel.setText("No data on input\n")
def Clear(self):
"""
Clear the widget state.
"""
self.queryGenes = []
self.referenceGenes = []
self.genes = {}
self.uniqueGenesDict = {}
self.revUniqueGenesDict = {}
self.pathways = {}
self.org = None
self.geneAttrCandidates[:] = []
self.infoLabel.setText("No data on input\n")
self.listView.clear()
self.pathwayView.SetPathway(None)
self.send("Selected Data", None)
self.send("Unselected Data", None)
def SetData(self, data=None):
if self.__runstate == OWKEGGPathwayBrowser.Initializing:
self.__initialize_finish()
self.data = data
self.warning(0)
self.error(0)
self.information(0)
if data is not None:
vars = data.domain.variables + data.domain.metas
vars = [var for var in vars
if isinstance(var, Orange.data.StringVariable)]
self.geneAttrCandidates[:] = vars
# Try to guess the gene name variable
if vars:
names_lower = [v.name.lower() for v in vars]
scores = [(name == "gene", "gene" in name)
for name in names_lower]
imax, _ = max(enumerate(scores), key=itemgetter(1))
else:
imax = -1
self.geneAttrIndex = imax
taxid = data_hints.get_hint(data, TAX_ID, None)
if taxid:
try:
code = kegg.from_taxid(taxid)
self.organismIndex = self.organismCodes.index(code)
except Exception as ex:
print(ex, taxid)
self.useAttrNames = data_hints.get_hint(data, GENE_NAME, self.useAttrNames)
if len(self.geneAttrCandidates) == 0:
self.useAttrNames = True
self.geneAttrIndex = -1
else:
self.geneAttrIndex = min(self.geneAttrIndex,
len(self.geneAttrCandidates) - 1)
else:
self.Clear()
self.__invalidated = True
def SetRefData(self, data=None):
self.refData = data
self.information(1)
if data is not None and self.useReference:
self.__invalidated = True
def handleNewSignals(self):
if self.__invalidated:
self.Update()
self.__invalidated = False
def UpdateListView(self):
self.bestPValueItem = None
self.listView.clear()
if not self.data:
return
allPathways = self.org.pathways()
allRefPathways = kegg.pathways("map")
items = []
kegg_pathways = kegg.KEGGPathways()
org_code = self.organismCodes[min(self.organismIndex,
len(self.organismCodes) - 1)]
if self.showOrthology:
self.koOrthology = kegg.KEGGBrite("ko00001")
self.listView.setRootIsDecorated(True)
path_ids = set([s[-5:] for s in self.pathways.keys()])
def _walkCollect(koEntry):
num = koEntry.title[:5] if koEntry.title else None
if num in path_ids:
return ([koEntry] +
reduce(lambda li, c: li + _walkCollect(c),
[child for child in koEntry.entries],
[]))
else:
c = reduce(lambda li, c: li + _walkCollect(c),
[child for child in koEntry.entries],
[])
return c + (c and [koEntry] or [])
allClasses = reduce(lambda li1, li2: li1 + li2,
[_walkCollect(c) for c in self.koOrthology],
[])
def _walkCreate(koEntry, lvItem):
item = QTreeWidgetItem(lvItem)
id = "path:" + org_code + koEntry.title[:5]
if koEntry.title[:5] in path_ids:
p = kegg_pathways.get_entry(id)
if p is None:
# In case the genesets still have obsolete entries
name = koEntry.title
else:
name = p.name
genes, p_value, ref = self.pathways[id]
item.setText(0, name)
item.setText(1, "%.5f" % p_value)
item.setText(2, "%i of %i" % (len(genes), len(self.genes)))
item.setText(3, "%i of %i" % (ref, len(self.referenceGenes)))
item.pathway_id = id if p is not None else None
else:
if id in allPathways:
text = kegg_pathways.get_entry(id).name
else:
text = koEntry.title
item.setText(0, text)
if id in allPathways:
item.pathway_id = id
elif "path:map" + koEntry.title[:5] in allRefPathways:
item.pathway_id = "path:map" + koEntry.title[:5]
else:
item.pathway_id = None
for child in koEntry.entries:
if child in allClasses:
_walkCreate(child, item)
for koEntry in self.koOrthology:
if koEntry in allClasses:
_walkCreate(koEntry, self.listView)
self.listView.update()
else:
self.listView.setRootIsDecorated(False)
pathways = self.pathways.items()
pathways = sorted(pathways, key=lambda item: item[1][1])
for id, (genes, p_value, ref) in pathways:
item = QTreeWidgetItem(self.listView)
item.setText(0, kegg_pathways.get_entry(id).name)
item.setText(1, "%.5f" % p_value)
item.setText(2, "%i of %i" % (len(genes), len(self.genes)))
item.setText(3, "%i of %i" % (ref, len(self.referenceGenes)))
item.pathway_id = id
items.append(item)
self.bestPValueItem = items and items[0] or None
self.listView.expandAll()
for i in range(4):
self.listView.resizeColumnToContents(i)
if self.bestPValueItem:
index = self.listView.indexFromItem(self.bestPValueItem)
self.listView.selectionModel().select(
index, QItemSelectionModel.ClearAndSelect
)
def UpdatePathwayView(self):
items = self.listView.selectedItems()
if len(items) > 0:
item = items[0]
else:
item = None
self.commit()
item = item or self.bestPValueItem
if not item or not item.pathway_id:
self.pathwayView.SetPathway(None)
return
def get_kgml_and_image(pathway_id):
"""Return an initialized KEGGPathway with pre-cached data"""
p = kegg.KEGGPathway(pathway_id)
p._get_kgml() # makes sure the kgml file is downloaded
p._get_image_filename() # makes sure the image is downloaded
return (pathway_id, p)
self.setEnabled(False)
self._pathwayTask = concurrent.Task(
function=lambda: get_kgml_and_image(item.pathway_id)
)
self._pathwayTask.finished.connect(self._onPathwayTaskFinshed)
self._executor.submit(self._pathwayTask)
def _onPathwayTaskFinshed(self):
self.setEnabled(True)
pathway_id, self.pathway = self._pathwayTask.result()
self.pathwayView.SetPathway(
self.pathway,
self.pathways.get(pathway_id, [[]])[0]
)
def UpdatePathwayViewTransform(self):
self.pathwayView.updateTransform()
def Update(self):
"""
Update (recompute enriched pathways) the widget state.
"""
if not self.data:
return
self.error(0)
self.information(0)
# XXX: Check data in setData, do not even allow this to be executed if
# data has no genes
try:
genes = self.GeneNamesFromData(self.data)
except ValueError:
self.error(0, "Cannot extract gene names from input.")
genes = []
if not self.useAttrNames and any("," in gene for gene in genes):
genes = reduce(add, (split_and_strip(gene, ",")
for gene in genes),
[])
self.information(0,
"Separators detected in input gene names. "
"Assuming multiple genes per instance.")
self.queryGenes = genes
self.information(1)
reference = None
if self.useReference and self.refData:
reference = self.GeneNamesFromData(self.refData)
if not self.useAttrNames \
and any("," in gene for gene in reference):
reference = reduce(add, (split_and_strip(gene, ",")
for gene in reference),
[])
self.information(1,
"Separators detected in reference gene "
"names. Assuming multiple genes per "
"instance.")
org_code = self.SelectedOrganismCode()
from orangecontrib.bioinformatics.ncbi.gene import GeneMatcher
gm = GeneMatcher(kegg.to_taxid(org_code))
gm.genes = genes
gm.run_matcher()
mapped_genes = {gene: str(ncbi_id) for gene, ncbi_id in gm.map_input_to_ncbi().items()}
def run_enrichment(org_code, genes, reference=None, progress=None):
org = kegg.KEGGOrganism(org_code)
if reference is None:
reference = org.get_ncbi_ids()
# This is here just to keep widget working without any major changes.
# map not needed, geneMatcher will not work on widget level.
unique_genes = genes
unique_ref_genes = dict([(gene, gene) for gene in set(reference)])
taxid = kegg.to_taxid(org.org_code)
# Map the taxid back to standard 'common' taxids
# (as used by 'geneset') if applicable
r_tax_map = dict((v, k) for k, v in
kegg.KEGGGenome.TAXID_MAP.items())
if taxid in r_tax_map:
taxid = r_tax_map[taxid]
# We use the kegg pathway gene sets provided by 'geneset' for
# the enrichment calculation.
kegg_api = kegg.api.CachedKeggApi()
linkmap = kegg_api.link(org.org_code, "pathway")
converted_ids = kegg_api.conv(org.org_code, 'ncbi-geneid')
kegg_sets = relation_list_to_multimap(linkmap, dict((gene.upper(), ncbi.split(':')[-1])
for ncbi, gene in converted_ids))
kegg_sets = geneset.GeneSets(input=kegg_sets)
pathways = pathway_enrichment(
kegg_sets, unique_genes.values(),
unique_ref_genes.keys(),
callback=progress
)
# Ensure that pathway entries are pre-cached for later use in the
# list/tree view
kegg_pathways = kegg.KEGGPathways()
kegg_pathways.pre_cache(
pathways.keys(), progress_callback=progress
)
return pathways, org, unique_genes, unique_ref_genes
self.progressBarInit()
self.setEnabled(False)
self.infoLabel.setText("Retrieving...\n")
progress = concurrent.methodinvoke(self, "setProgress", (float,))
self._enrichTask = concurrent.Task(
function=lambda:
run_enrichment(org_code, mapped_genes, reference, progress)
)
self._enrichTask.finished.connect(self._onEnrichTaskFinished)
self._executor.submit(self._enrichTask)
def _onEnrichTaskFinished(self):
self.setEnabled(True)
self.setBlocking(False)
try:
pathways, org, unique_genes, unique_ref_genes = \
self._enrichTask.result()
except Exception:
raise
self.progressBarFinished()
self.org = org
self.genes = unique_genes.keys()
self.uniqueGenesDict = {ncbi_id: input_name for input_name, ncbi_id in unique_genes.items()}
self.revUniqueGenesDict = dict([(val, key) for key, val in
self.uniqueGenesDict.items()])
self.referenceGenes = unique_ref_genes.keys()
self.pathways = pathways
if not self.pathways:
self.warning(0, "No enriched pathways found.")
else:
self.warning(0)
count = len(set(self.queryGenes))
self.infoLabel.setText(
"%i unique gene names on input\n"
"%i (%.1f%%) genes names matched" %
(count, len(unique_genes),
100.0 * len(unique_genes) / count if count else 0.0)
)
self.UpdateListView()
@Slot(float)
def setProgress(self, value):
if self.__in_setProgress:
return
self.__in_setProgress = True
self.progressBarSet(value)
self.__in_setProgress = False
def GeneNamesFromData(self, data):
"""
Extract and return gene names from `data`.
"""
if self.useAttrNames:
genes = [str(v.name).strip() for v in data.domain.attributes]
elif self.geneAttrCandidates:
assert 0 <= self.geneAttrIndex < len(self.geneAttrCandidates)
geneAttr = self.geneAttrCandidates[self.geneAttrIndex]
genes = [str(e[geneAttr]) for e in data
if not numpy.isnan(e[geneAttr])]
else:
raise ValueError("No gene names in data.")
return genes
def SelectedOrganismCode(self):
"""
Return the selected organism code.
"""
return self.organismCodes[min(self.organismIndex,
len(self.organismCodes) - 1)]
def selectAll(self):
"""
Select all items in the pathway view.
"""
changed = False
scene = self.pathwayView.scene()
with disconnected(scene.selectionChanged, self._onSelectionChanged):
for item in scene.items():
if item.flags() & QGraphicsItem.ItemIsSelectable and \
not item.isSelected():
item.setSelected(True)
changed = True
if changed:
self._onSelectionChanged()
def _onSelectionChanged(self):
# Item selection in the pathwayView/scene has changed
self.commit()
def commit(self):
if self.data:
selectedItems = self.pathwayView.scene().selectedItems()
selectedGenes = reduce(set.union, [item.marked_objects
for item in selectedItems],
set())
if self.useAttrNames:
selected = [self.data.domain[self.uniqueGenesDict[gene]]
for gene in selectedGenes]
# newDomain = Orange.data.Domain(selectedVars, 0)
data = self.data[:, selected]
# data = Orange.data.Table(newDomain, self.data)
self.send("Selected Data", data)
elif self.geneAttrCandidates:
assert 0 <= self.geneAttrIndex < len(self.geneAttrCandidates)
geneAttr = self.geneAttrCandidates[self.geneAttrIndex]
selectedIndices = []
otherIndices = []
for i, ex in enumerate(self.data):
names = [self.revUniqueGenesDict.get(name, None)
for name in split_and_strip(str(ex[geneAttr]), ",")]
if any(name and name in selectedGenes for name in names):
selectedIndices.append(i)
else:
otherIndices.append(i)
if selectedIndices:
selected = self.data[selectedIndices]
else:
selected = None
if otherIndices:
other = self.data[otherIndices]
else:
other = None
self.send("Selected Data", selected)
self.send("Unselected Data", other)
else:
self.send("Selected Data", None)
self.send("Unselected Data", None)
def ClearCache(self):
kegg.caching.clear_cache()
def onDeleteWidget(self):
"""
Called before the widget is removed from the canvas.
"""
super().onDeleteWidget()
self.org = None
self._executor.shutdown(wait=False)
gc.collect() # Force collection (WHY?)
def sizeHint(self):
return QSize(1024, 720)
class OWWordEnrichment(OWWidget, ConcurrentWidgetMixin):
# Basic widget info
name = "Word Enrichment"
description = "Word enrichment analysis for selected documents."
icon = "icons/SetEnrichment.svg"
priority = 600
# Input/output
class Inputs:
selected_data = Input("Selected Data", Table)
data = Input("Data", Table)
want_main_area = True
class Error(OWWidget.Error):
no_bow_features = Msg('No bag-of-words features!')
no_words_overlap = Msg('No words overlap!')
empty_selection = Msg('Selected data is empty!')
all_selected = Msg('All examples can not be selected!')
# Settings
filter_by_p: bool = Setting(False)
filter_p_value: float = Setting(0.01)
filter_by_fdr: bool = Setting(True)
filter_fdr_value: float = Setting(0.2)
def __init__(self):
OWWidget.__init__(self)
ConcurrentWidgetMixin.__init__(self)
# Init data
self.data = None
self.selected_data = None
# used for transforming the 'selected data' into the 'data' domain
self.selected_data_transformed = None
self.results = Result()
# info box
fbox = gui.widgetBox(self.controlArea, "Info")
self.info_fil = gui.label(fbox, self, 'Words displayed: 0')
# Filtering settings
fbox = gui.widgetBox(self.controlArea, "Filter")
hbox = gui.widgetBox(fbox, orientation=0)
self.chb_p = gui.checkBox(hbox,
self,
"filter_by_p",
"p-value",
callback=self.filter_and_display,
tooltip="Filter by word p-value")
self.spin_p = gui.doubleSpin(hbox,
self,
'filter_p_value',
1e-4,
1,
step=1e-4,
labelWidth=15,
callback=self.filter_and_display,
tooltip="Max p-value for word")
self.spin_p.setEnabled(self.filter_by_p)
hbox = gui.widgetBox(fbox, orientation=0)
self.chb_fdr = gui.checkBox(hbox,
self,
"filter_by_fdr",
"FDR",
callback=self.filter_and_display,
tooltip="Filter by word FDR")
self.spin_fdr = gui.doubleSpin(hbox,
self,
'filter_fdr_value',
1e-4,
1,
step=1e-4,
labelWidth=15,
callback=self.filter_and_display,
tooltip="Max p-value for word")
self.spin_fdr.setEnabled(self.filter_by_fdr)
gui.rubber(self.controlArea)
# Word's list view
self.cols = ['Word', 'p-value', 'FDR']
self.sig_words = QTreeWidget()
self.sig_words.setColumnCount(len(self.cols))
self.sig_words.setHeaderLabels(self.cols)
self.sig_words.setSortingEnabled(True)
self.sig_words.setSelectionMode(QTreeView.NoSelection)
self.sig_words.sortByColumn(2, 0) # 0 is ascending order
for i in range(len(self.cols)):
self.sig_words.resizeColumnToContents(i)
self.mainArea.layout().addWidget(self.sig_words)
def sizeHint(self):
return QSize(450, 240)
@Inputs.data
def set_data(self, data=None):
self.data = data
# selected data transformed depends on data domain
self.selected_data_transformed = None
@Inputs.selected_data
def set_data_selected(self, data=None):
self.selected_data = data
def handleNewSignals(self):
self.check_data()
def get_bow_domain(self):
domain = self.data.domain
return Domain(attributes=[
a for a in domain.attributes
if a.attributes.get('bow-feature', False)
],
class_vars=domain.class_vars,
metas=domain.metas,
source=domain)
def check_data(self):
self.Error.clear()
if isinstance(self.data, Table) and \
isinstance(self.selected_data, Table):
if len(self.selected_data) == 0:
self.Error.empty_selection()
self.clear()
return
# keep only BoW features
bow_domain = self.get_bow_domain()
if len(bow_domain.attributes) == 0:
self.Error.no_bow_features()
self.clear()
return
self.data = Corpus.from_table(bow_domain, self.data)
self.selected_data_transformed = Corpus.from_table(
bow_domain, self.selected_data)
if np_sp_sum(self.selected_data_transformed.X) == 0:
self.Error.no_words_overlap()
self.clear()
elif len(self.data) == len(self.selected_data):
self.Error.all_selected()
self.clear()
else:
self.set_input_info()
self.apply()
else:
self.clear()
def clear(self):
self.sig_words.clear()
self.info.set_input_summary(self.info.NoInput)
self.set_displayed_info(0)
def filter_enabled(self, b):
self.chb_p.setEnabled(b)
self.chb_fdr.setEnabled(b)
self.spin_p.setEnabled(b)
self.spin_fdr.setEnabled(b)
def filter_and_display(self):
self.spin_p.setEnabled(self.filter_by_p)
self.spin_fdr.setEnabled(self.filter_by_fdr)
self.sig_words.clear()
if self.selected_data_transformed is None: # do nothing when no Data
return
if self.results.words:
count = self.build_tree()
else:
count = 0
for i in range(len(self.cols)):
self.sig_words.resizeColumnToContents(i)
self.set_displayed_info(count)
def build_tree(self) -> int:
count = 0
for word, pval, fval in zip(self.results.words, self.results.p_values,
self.results.fdr_values):
if ((not self.filter_by_p or pval <= self.filter_p_value) and
(not self.filter_by_fdr or fval <= self.filter_fdr_value)):
it = EATreeWidgetItem(word, pval, fval, self.sig_words)
self.sig_words.addTopLevelItem(it)
count += 1
return count
def set_input_info(self) -> None:
cluster_words = len(self.selected_data_transformed.domain.attributes)
selected_words = np.count_nonzero(
np_sp_sum(self.selected_data_transformed.X, axis=0))
self.info.set_input_summary(
f"{cluster_words}|{selected_words}",
f"Total words: {cluster_words}\n"
f"Words in subset: {selected_words}")
def set_displayed_info(self, count: int) -> None:
self.info_fil.setText(f"Words displayed: {count}")
def apply(self):
self.sig_words.clear()
self.filter_enabled(False)
self.start(Runner.run, self.selected_data_transformed, self.data,
self.results)
def on_done(self, result: Result) -> None:
self.filter_and_display()
self.filter_enabled(True)
def on_exception(self, ex: Exception) -> None:
self.filter_enabled(True)
def tree_to_table(self):
view = [self.cols]
items = self.sig_words.topLevelItemCount()
for i in range(items):
line = []
for j in range(3):
line.append(self.sig_words.topLevelItem(i).text(j))
view.append(line)
return view
def send_report(self):
if self.results.words:
self.report_table("Enriched words", self.tree_to_table())
