class OWGOEnrichmentAnalysis(widget.OWWidget): name = "GO Browser" description = "Enrichment analysis for Gene Ontology terms." icon = "../widgets/icons/GOBrowser.svg" priority = 2020 inputs = [("Cluster Data", Orange.data.Table, "setDataset", widget.Single + widget.Default), ("Reference Data", Orange.data.Table, "setReferenceDataset")] outputs = [("Data on Selected Genes", Orange.data.Table), ("Data on Unselected Genes", Orange.data.Table), ("Data on Unknown Genes", Orange.data.Table), ("Enrichment Report", Orange.data.Table)] settingsHandler = settings.DomainContextHandler() annotationIndex = settings.ContextSetting(0) geneAttrIndex = settings.ContextSetting(0) useAttrNames = settings.ContextSetting(False) geneMatcherSettings = settings.Setting([True, False, False, False]) useReferenceDataset = settings.Setting(False) aspectIndex = settings.Setting(0) useEvidenceType = settings.Setting( {et: True for et in go.evidenceTypesOrdered}) filterByNumOfInstances = settings.Setting(False) minNumOfInstances = settings.Setting(1) filterByPValue = settings.Setting(True) maxPValue = settings.Setting(0.2) filterByPValue_nofdr = settings.Setting(False) maxPValue_nofdr = settings.Setting(0.01) probFunc = settings.Setting(0) selectionDirectAnnotation = settings.Setting(0) selectionDisjoint = settings.Setting(0) selectionAddTermAsClass = settings.Setting(0) Ready, Initializing, Running = 0, 1, 2 def __init__(self, parent=None): super().__init__(self, parent) self.clusterDataset = None self.referenceDataset = None self.ontology = None self.annotations = None self.loadedAnnotationCode = "---" self.treeStructRootKey = None self.probFunctions = [stats.Binomial(), stats.Hypergeometric()] self.selectedTerms = [] self.selectionChanging = 0 self.__state = OWGOEnrichmentAnalysis.Initializing self.annotationCodes = [] ############# ## GUI ############# self.tabs = gui.tabWidget(self.controlArea) ## Input tab self.inputTab = gui.createTabPage(self.tabs, "Input") box = gui.widgetBox(self.inputTab, "Info") self.infoLabel = gui.widgetLabel(box, "No data on input\n") gui.button(box, self, "Ontology/Annotation Info", callback=self.ShowInfo, tooltip="Show information on loaded ontology and annotations") box = gui.widgetBox(self.inputTab, "Organism") self.annotationComboBox = gui.comboBox( box, self, "annotationIndex", items=self.annotationCodes, callback=self._updateEnrichment, tooltip="Select organism") genebox = gui.widgetBox(self.inputTab, "Gene Names") self.geneAttrIndexCombo = gui.comboBox( genebox, self, "geneAttrIndex", callback=self._updateEnrichment, tooltip="Use this attribute to extract gene names from input data") self.geneAttrIndexCombo.setDisabled(self.useAttrNames) cb = gui.checkBox(genebox, self, "useAttrNames", "Use column names", tooltip="Use column names for gene names", callback=self._updateEnrichment) cb.toggled[bool].connect(self.geneAttrIndexCombo.setDisabled) gui.button(genebox, self, "Gene matcher settings", callback=self.UpdateGeneMatcher, tooltip="Open gene matching settings dialog") self.referenceRadioBox = gui.radioButtonsInBox( self.inputTab, self, "useReferenceDataset", ["Entire genome", "Reference set (input)"], tooltips=["Use entire genome for reference", "Use genes from Referece Examples input signal as reference"], box="Reference", callback=self._updateEnrichment) self.referenceRadioBox.buttons[1].setDisabled(True) gui.radioButtonsInBox( self.inputTab, self, "aspectIndex", ["Biological process", "Cellular component", "Molecular function"], box="Aspect", callback=self._updateEnrichment) ## Filter tab self.filterTab = gui.createTabPage(self.tabs, "Filter") box = gui.widgetBox(self.filterTab, "Filter GO Term Nodes") gui.checkBox(box, self, "filterByNumOfInstances", "Genes", callback=self.FilterAndDisplayGraph, tooltip="Filter by number of input genes mapped to a term") ibox = gui.indentedBox(box) gui.spin(ibox, self, 'minNumOfInstances', 1, 100, step=1, label='#:', labelWidth=15, callback=self.FilterAndDisplayGraph, callbackOnReturn=True, tooltip="Min. number of input genes mapped to a term") gui.checkBox(box, self, "filterByPValue_nofdr", "p-value", callback=self.FilterAndDisplayGraph, tooltip="Filter by term p-value") gui.doubleSpin(gui.indentedBox(box), self, 'maxPValue_nofdr', 1e-8, 1, step=1e-8, label='p:', labelWidth=15, callback=self.FilterAndDisplayGraph, callbackOnReturn=True, tooltip="Max term p-value") #use filterByPValue for FDR, as it was the default in prior versions gui.checkBox(box, self, "filterByPValue", "FDR", callback=self.FilterAndDisplayGraph, tooltip="Filter by term FDR") gui.doubleSpin(gui.indentedBox(box), self, 'maxPValue', 1e-8, 1, step=1e-8, label='p:', labelWidth=15, callback=self.FilterAndDisplayGraph, callbackOnReturn=True, tooltip="Max term p-value") box = gui.widgetBox(box, "Significance test") gui.radioButtonsInBox(box, self, "probFunc", ["Binomial", "Hypergeometric"], tooltips=["Use binomial distribution test", "Use hypergeometric distribution test"], callback=self._updateEnrichment) box = gui.widgetBox(self.filterTab, "Evidence codes in annotation", addSpace=True) self.evidenceCheckBoxDict = {} for etype in go.evidenceTypesOrdered: ecb = QCheckBox( etype, toolTip=go.evidenceTypes[etype], checked=self.useEvidenceType[etype]) ecb.toggled.connect(self.__on_evidenceChanged) box.layout().addWidget(ecb) self.evidenceCheckBoxDict[etype] = ecb ## Select tab self.selectTab = gui.createTabPage(self.tabs, "Select") box = gui.radioButtonsInBox( self.selectTab, self, "selectionDirectAnnotation", ["Directly or Indirectly", "Directly"], box="Annotated genes", callback=self.ExampleSelection) box = gui.widgetBox(self.selectTab, "Output", addSpace=True) gui.radioButtonsInBox( box, self, "selectionDisjoint", btnLabels=["All selected genes", "Term-specific genes", "Common term genes"], tooltips=["Outputs genes annotated to all selected GO terms", "Outputs genes that appear in only one of selected GO terms", "Outputs genes common to all selected GO terms"], callback=[self.ExampleSelection, self.UpdateAddClassButton]) self.addClassCB = gui.checkBox( box, self, "selectionAddTermAsClass", "Add GO Term as class", callback=self.ExampleSelection) # ListView for DAG, and table for significant GOIDs self.DAGcolumns = ['GO term', 'Cluster', 'Reference', 'p-value', 'FDR', 'Genes', 'Enrichment'] self.splitter = QSplitter(Qt.Vertical, self.mainArea) self.mainArea.layout().addWidget(self.splitter) # list view self.listView = GOTreeWidget(self.splitter) self.listView.setSelectionMode(QTreeView.ExtendedSelection) self.listView.setAllColumnsShowFocus(1) self.listView.setColumnCount(len(self.DAGcolumns)) self.listView.setHeaderLabels(self.DAGcolumns) self.listView.header().setSectionsClickable(True) self.listView.header().setSortIndicatorShown(True) self.listView.setSortingEnabled(True) self.listView.setItemDelegateForColumn( 6, EnrichmentColumnItemDelegate(self)) self.listView.setRootIsDecorated(True) self.listView.itemSelectionChanged.connect(self.ViewSelectionChanged) # table of significant GO terms self.sigTerms = QTreeWidget(self.splitter) self.sigTerms.setColumnCount(len(self.DAGcolumns)) self.sigTerms.setHeaderLabels(self.DAGcolumns) self.sigTerms.setSortingEnabled(True) self.sigTerms.setSelectionMode(QTreeView.ExtendedSelection) self.sigTerms.setItemDelegateForColumn( 6, EnrichmentColumnItemDelegate(self)) self.sigTerms.itemSelectionChanged.connect(self.TableSelectionChanged) self.sigTableTermsSorted = [] self.graph = {} self.inputTab.layout().addStretch(1) self.filterTab.layout().addStretch(1) self.selectTab.layout().addStretch(1) self.setBlocking(True) self._executor = ThreadExecutor() self._init = EnsureDownloaded( [(taxonomy.Taxonomy.DOMAIN, taxonomy.Taxonomy.FILENAME), ("GO", "taxonomy.pickle")] ) self._init.finished.connect(self.__initialize_finish) self._executor.submit(self._init) def sizeHint(self): return QSize(1000, 700) def __initialize_finish(self): self.setBlocking(False) try: self.annotationFiles = listAvailable() except ConnectTimeout: self.error(2, "Internet connection error, unable to load data. " + \ "Check connection and create a new GO Browser widget.") self.filterTab.setEnabled(False) self.inputTab.setEnabled(False) self.selectTab.setEnabled(False) self.listView.setEnabled(False) self.sigTerms.setEnabled(False) else: self.annotationCodes = sorted(self.annotationFiles.keys()) self.annotationComboBox.clear() self.annotationComboBox.addItems(self.annotationCodes) self.annotationComboBox.setCurrentIndex(self.annotationIndex) self.__state = OWGOEnrichmentAnalysis.Ready def __on_evidenceChanged(self): for etype, cb in self.evidenceCheckBoxDict.items(): self.useEvidenceType[etype] = cb.isChecked() self._updateEnrichment() def UpdateGeneMatcher(self): """Open the Gene matcher settings dialog.""" dialog = GeneMatcherDialog(self, defaults=self.geneMatcherSettings, modal=True) if dialog.exec_() != QDialog.Rejected: self.geneMatcherSettings = [getattr(dialog, item[0]) for item in dialog.items] if self.annotations: self.SetGeneMatcher() self._updateEnrichment() def clear(self): self.infoLabel.setText("No data on input\n") self.warning(0) self.warning(1) self.geneAttrIndexCombo.clear() self.ClearGraph() self.send("Data on Selected Genes", None) self.send("Data on Unselected Genes", None) self.send("Data on Unknown Genes", None) self.send("Enrichment Report", None) def setDataset(self, data=None): if self.__state == OWGOEnrichmentAnalysis.Initializing: self.__initialize_finish() self.closeContext() self.clear() self.clusterDataset = data if data is not None: domain = data.domain allvars = domain.variables + domain.metas self.candidateGeneAttrs = [var for var in allvars if isstring(var)] self.geneAttrIndexCombo.clear() for var in self.candidateGeneAttrs: self.geneAttrIndexCombo.addItem(*gui.attributeItem(var)) taxid = data_hints.get_hint(data, "taxid", "") code = None try: code = go.from_taxid(taxid) except KeyError: pass except Exception as ex: print(ex) if code is not None: filename = "gene_association.%s.tar.gz" % code if filename in self.annotationFiles.values(): self.annotationIndex = \ [i for i, name in enumerate(self.annotationCodes) \ if self.annotationFiles[name] == filename].pop() self.useAttrNames = data_hints.get_hint(data, "genesinrows", self.useAttrNames) self.openContext(data) self.geneAttrIndex = min(self.geneAttrIndex, len(self.candidateGeneAttrs) - 1) if len(self.candidateGeneAttrs) == 0: self.useAttrNames = True self.geneAttrIndex = -1 elif self.geneAttrIndex < len(self.candidateGeneAttrs): self.geneAttrIndex = len(self.candidateGeneAttrs) - 1 self._updateEnrichment() def setReferenceDataset(self, data=None): self.referenceDataset = data self.referenceRadioBox.buttons[1].setDisabled(not bool(data)) self.referenceRadioBox.buttons[1].setText("Reference set") if self.clusterDataset is not None and self.useReferenceDataset: self.useReferenceDataset = 0 if not data else 1 graph = self.Enrichment() self.SetGraph(graph) elif self.clusterDataset: self.__updateReferenceSetButton() def handleNewSignals(self): super().handleNewSignals() def _updateEnrichment(self): if self.clusterDataset is not None and \ self.__state == OWGOEnrichmentAnalysis.Ready: pb = gui.ProgressBar(self, 100) self.Load(pb=pb) graph = self.Enrichment(pb=pb) self.FilterUnknownGenes() self.SetGraph(graph) def __updateReferenceSetButton(self): allgenes, refgenes = None, None if self.referenceDataset: try: allgenes = self.genesFromTable(self.referenceDataset) except Exception: allgenes = [] refgenes, unknown = self.FilterAnnotatedGenes(allgenes) self.referenceRadioBox.buttons[1].setDisabled(not bool(allgenes)) self.referenceRadioBox.buttons[1].setText("Reference set " + ("(%i genes, %i matched)" % (len(allgenes), len(refgenes)) if allgenes and refgenes else "")) def genesFromTable(self, data): if self.useAttrNames: genes = [v.name for v in data.domain.variables] else: attr = self.candidateGeneAttrs[min(self.geneAttrIndex, len(self.candidateGeneAttrs) - 1)] genes = [str(ex[attr]) for ex in data if not numpy.isnan(ex[attr])] if any("," in gene for gene in genes): self.information(0, "Separators detected in gene names. Assuming multiple genes per example.") genes = reduce(operator.iadd, (genes.split(",") for genes in genes), []) return genes def FilterAnnotatedGenes(self, genes): matchedgenes = self.annotations.get_gene_names_translator(genes).values() return matchedgenes, [gene for gene in genes if gene not in matchedgenes] def FilterUnknownGenes(self): if not self.useAttrNames and self.candidateGeneAttrs: geneAttr = self.candidateGeneAttrs[min(self.geneAttrIndex, len(self.candidateGeneAttrs)-1)] indices = [] for i, ex in enumerate(self.clusterDataset): if not any(self.annotations.genematcher.match(n.strip()) for n in str(ex[geneAttr]).split(",")): indices.append(i) if indices: data = self.clusterDataset[indices] else: data = None self.send("Data on Unknown Genes", data) else: self.send("Data on Unknown Genes", None) def Load(self, pb=None): if self.__state == OWGOEnrichmentAnalysis.Ready: go_files, tax_files = serverfiles.listfiles("GO"), serverfiles.listfiles("Taxonomy") calls = [] pb, finish = (gui.ProgressBar(self, 0), True) if pb is None else (pb, False) count = 0 if not tax_files: calls.append(("Taxonomy", "ncbi_taxnomy.tar.gz")) count += 1 org = self.annotationCodes[min(self.annotationIndex, len(self.annotationCodes)-1)] if org != self.loadedAnnotationCode: count += 1 if self.annotationFiles[org] not in go_files: calls.append(("GO", self.annotationFiles[org])) count += 1 if "gene_ontology_edit.obo.tar.gz" not in go_files: calls.append(("GO", "gene_ontology_edit.obo.tar.gz")) count += 1 if not self.ontology: count += 1 pb.iter += count * 100 for args in calls: serverfiles.localpath_download(*args, **dict(callback=pb.advance)) i = len(calls) if not self.ontology: self.ontology = go.Ontology(progress_callback=lambda value: pb.advance()) i += 1 if org != self.loadedAnnotationCode: self.annotations = None gc.collect() # Force run garbage collection code = self.annotationFiles[org].split(".")[-3] self.annotations = go.Annotations(code, genematcher=gene.GMDirect(), progress_callback=lambda value: pb.advance()) i += 1 self.loadedAnnotationCode = org count = defaultdict(int) geneSets = defaultdict(set) for anno in self.annotations.annotations: count[anno.evidence] += 1 geneSets[anno.evidence].add(anno.geneName) for etype in go.evidenceTypesOrdered: ecb = self.evidenceCheckBoxDict[etype] ecb.setEnabled(bool(count[etype])) ecb.setText(etype + ": %i annots(%i genes)" % (count[etype], len(geneSets[etype]))) if finish: pb.finish() def SetGeneMatcher(self): if self.annotations: taxid = self.annotations.taxid matchers = [] for matcher, use in zip([gene.GMGO, gene.GMKEGG, gene.GMNCBI, gene.GMAffy], self.geneMatcherSettings): if use: try: if taxid == "352472": matchers.extend([matcher(taxid), gene.GMDicty(), [matcher(taxid), gene.GMDicty()]]) # The reason machers are duplicated is that we want `matcher` or `GMDicty` to # match genes by them self if possible. Only use the joint matcher if they fail. else: matchers.append(matcher(taxid)) except Exception as ex: print(ex) self.annotations.genematcher = gene.matcher(matchers) self.annotations.genematcher.set_targets(self.annotations.gene_names) def Enrichment(self, pb=None): assert self.clusterDataset is not None pb = gui.ProgressBar(self, 100) if pb is None else pb if not self.annotations.ontology: self.annotations.ontology = self.ontology if isinstance(self.annotations.genematcher, gene.GMDirect): self.SetGeneMatcher() self.error(1) self.warning([0, 1]) if self.useAttrNames: clusterGenes = [v.name for v in self.clusterDataset.domain.attributes] self.information(0) elif 0 <= self.geneAttrIndex < len(self.candidateGeneAttrs): geneAttr = self.candidateGeneAttrs[self.geneAttrIndex] clusterGenes = [str(ex[geneAttr]) for ex in self.clusterDataset if not numpy.isnan(ex[geneAttr])] if any("," in gene for gene in clusterGenes): self.information(0, "Separators detected in cluster gene names. Assuming multiple genes per example.") clusterGenes = reduce(operator.iadd, (genes.split(",") for genes in clusterGenes), []) else: self.information(0) else: self.error(1, "Failed to extract gene names from input dataset!") return {} genesSetCount = len(set(clusterGenes)) self.clusterGenes = clusterGenes = self.annotations.get_gene_names_translator(clusterGenes).values() self.infoLabel.setText("%i unique genes on input\n%i (%.1f%%) genes with known annotations" % (genesSetCount, len(clusterGenes), 100.0*len(clusterGenes)/genesSetCount if genesSetCount else 0.0)) referenceGenes = None if not self.useReferenceDataset or self.referenceDataset is None: self.information(2) self.information(1) referenceGenes = self.annotations.gene_names elif self.referenceDataset is not None: if self.useAttrNames: referenceGenes = [v.name for v in self.referenceDataset.domain.attributes] self.information(1) elif geneAttr in (self.referenceDataset.domain.variables + self.referenceDataset.domain.metas): referenceGenes = [str(ex[geneAttr]) for ex in self.referenceDataset if not numpy.isnan(ex[geneAttr])] if any("," in gene for gene in clusterGenes): self.information(1, "Separators detected in reference gene names. Assuming multiple genes per example.") referenceGenes = reduce(operator.iadd, (genes.split(",") for genes in referenceGenes), []) else: self.information(1) else: self.information(1) referenceGenes = None if referenceGenes is None: referenceGenes = list(self.annotations.gene_names) self.referenceRadioBox.buttons[1].setText("Reference set") self.referenceRadioBox.buttons[1].setDisabled(True) self.information(2, "Unable to extract gene names from reference dataset. Using entire genome for reference") self.useReferenceDataset = 0 else: refc = len(referenceGenes) referenceGenes = self.annotations.get_gene_names_translator(referenceGenes).values() self.referenceRadioBox.buttons[1].setText("Reference set (%i genes, %i matched)" % (refc, len(referenceGenes))) self.referenceRadioBox.buttons[1].setDisabled(False) self.information(2) else: self.useReferenceDataset = 0 if not referenceGenes: self.error(1, "No valid reference set") return {} self.referenceGenes = referenceGenes evidences = [] for etype in go.evidenceTypesOrdered: if self.useEvidenceType[etype]: evidences.append(etype) aspect = ["P", "C", "F"][self.aspectIndex] if clusterGenes: self.terms = terms = self.annotations.get_enriched_terms( clusterGenes, referenceGenes, evidences, aspect=aspect, prob=self.probFunctions[self.probFunc], use_fdr=False, progress_callback=lambda value: pb.advance()) ids = [] pvals = [] for i, d in self.terms.items(): ids.append(i) pvals.append(d[1]) for i, fdr in zip(ids, stats.FDR(pvals)): # save FDR as the last part of the tuple terms[i] = tuple(list(terms[i]) + [ fdr ]) else: self.terms = terms = {} if not self.terms: self.warning(0, "No enriched terms found.") else: self.warning(0) pb.finish() self.treeStructDict = {} ids = self.terms.keys() self.treeStructRootKey = None parents = {} for id in ids: parents[id] = set([term for _, term in self.ontology[id].related]) children = {} for term in self.terms: children[term] = set([id for id in ids if term in parents[id]]) for term in self.terms: self.treeStructDict[term] = TreeNode(self.terms[term], children[term]) if not self.ontology[term].related and not getattr(self.ontology[term], "is_obsolete", False): self.treeStructRootKey = term return terms def FilterGraph(self, graph): if self.filterByPValue_nofdr: graph = go.filterByPValue(graph, self.maxPValue_nofdr) if self.filterByPValue: #FDR graph = dict(filter(lambda item: item[1][3] <= self.maxPValue, graph.items())) if self.filterByNumOfInstances: graph = dict(filter(lambda item: len(item[1][0]) >= self.minNumOfInstances, graph.items())) return graph def FilterAndDisplayGraph(self): if self.clusterDataset: self.graph = self.FilterGraph(self.originalGraph) if self.originalGraph and not self.graph: self.warning(1, "All found terms were filtered out.") else: self.warning(1) self.ClearGraph() self.DisplayGraph() def SetGraph(self, graph=None): self.originalGraph = graph if graph: self.FilterAndDisplayGraph() else: self.graph = {} self.ClearGraph() def ClearGraph(self): self.listView.clear() self.listViewItems=[] self.sigTerms.clear() def DisplayGraph(self): fromParentDict = {} self.termListViewItemDict = {} self.listViewItems = [] enrichment = lambda t: len(t[0]) / t[2] * (len(self.referenceGenes) / len(self.clusterGenes)) maxFoldEnrichment = max([enrichment(term) for term in self.graph.values()] or [1]) def addNode(term, parent, parentDisplayNode): if (parent, term) in fromParentDict: return if term in self.graph: displayNode = GOTreeWidgetItem(self.ontology[term], self.graph[term], len(self.clusterGenes), len(self.referenceGenes), maxFoldEnrichment, parentDisplayNode) displayNode.goId = term self.listViewItems.append(displayNode) if term in self.termListViewItemDict: self.termListViewItemDict[term].append(displayNode) else: self.termListViewItemDict[term] = [displayNode] fromParentDict[(parent, term)] = True parent = term else: displayNode = parentDisplayNode for c in self.treeStructDict[term].children: addNode(c, parent, displayNode) if self.treeStructDict: addNode(self.treeStructRootKey, None, self.listView) terms = self.graph.items() terms = sorted(terms, key=lambda item: item[1][1]) self.sigTableTermsSorted = [t[0] for t in terms] self.sigTerms.clear() for i, (t_id, (genes, p_value, refCount, fdr)) in enumerate(terms): item = GOTreeWidgetItem(self.ontology[t_id], (genes, p_value, refCount, fdr), len(self.clusterGenes), len(self.referenceGenes), maxFoldEnrichment, self.sigTerms) item.goId = t_id self.listView.expandAll() for i in range(5): self.listView.resizeColumnToContents(i) self.sigTerms.resizeColumnToContents(i) self.sigTerms.resizeColumnToContents(6) width = min(self.listView.columnWidth(0), 350) self.listView.setColumnWidth(0, width) self.sigTerms.setColumnWidth(0, width) # Create and send the enrichemnt report table. termsDomain = Orange.data.Domain( [], [], # All is meta! [Orange.data.StringVariable("GO Term Id"), Orange.data.StringVariable("GO Term Name"), Orange.data.ContinuousVariable("Cluster Frequency"), Orange.data.ContinuousVariable("Genes in Cluster", number_of_decimals=0), Orange.data.ContinuousVariable("Reference Frequency"), Orange.data.ContinuousVariable("Genes in Reference", number_of_decimals=0), Orange.data.ContinuousVariable("p-value"), Orange.data.ContinuousVariable("FDR"), Orange.data.ContinuousVariable("Enrichment"), Orange.data.StringVariable("Genes")]) terms = [[t_id, self.ontology[t_id].name, len(genes) / len(self.clusterGenes), len(genes), r_count / len(self.referenceGenes), r_count, p_value, fdr, len(genes) / len(self.clusterGenes) * \ len(self.referenceGenes) / r_count, ",".join(genes) ] for t_id, (genes, p_value, r_count, fdr) in terms] if terms: X = numpy.empty((len(terms), 0)) M = numpy.array(terms, dtype=object) termsTable = Orange.data.Table.from_numpy(termsDomain, X, metas=M) else: termsTable = Orange.data.Table(termsDomain) self.send("Enrichment Report", termsTable) def ViewSelectionChanged(self): if self.selectionChanging: return self.selectionChanging = 1 self.selectedTerms = [] selected = self.listView.selectedItems() self.selectedTerms = list(set([lvi.term.id for lvi in selected])) self.ExampleSelection() self.selectionChanging = 0 def TableSelectionChanged(self): if self.selectionChanging: return self.selectionChanging = 1 self.selectedTerms = [] selectedIds = set([self.sigTerms.itemFromIndex(index).goId for index in self.sigTerms.selectedIndexes()]) for i in range(self.sigTerms.topLevelItemCount()): item = self.sigTerms.topLevelItem(i) selected = item.goId in selectedIds term = item.goId if selected: self.selectedTerms.append(term) for lvi in self.termListViewItemDict[term]: try: lvi.setSelected(selected) if selected: lvi.setExpanded(True) except RuntimeError: # Underlying C/C++ object deleted pass self.ExampleSelection() self.selectionChanging = 0 def UpdateAddClassButton(self): self.addClassCB.setEnabled(self.selectionDisjoint == 1) def ExampleSelection(self): self.commit() def commit(self): if self.clusterDataset is None: return terms = set(self.selectedTerms) genes = reduce(operator.ior, (set(self.graph[term][0]) for term in terms), set()) evidences = [] for etype in go.evidenceTypesOrdered: if self.useEvidenceType[etype]: # if getattr(self, "useEvidence" + etype): evidences.append(etype) allTerms = self.annotations.get_annotated_terms( genes, direct_annotation_only=self.selectionDirectAnnotation, evidence_codes=evidences) if self.selectionDisjoint > 0: count = defaultdict(int) for term in self.selectedTerms: for g in allTerms.get(term, []): count[g] += 1 ccount = 1 if self.selectionDisjoint == 1 else len(self.selectedTerms) selectedGenes = [gene for gene, c in count.items() if c == ccount and gene in genes] else: selectedGenes = reduce( operator.ior, (set(allTerms.get(term, [])) for term in self.selectedTerms), set()) if self.useAttrNames: vars = [self.clusterDataset.domain[gene] for gene in set(selectedGenes)] domain = Orange.data.Domain( vars, self.clusterDataset.domain.class_vars, self.clusterDataset.domain.metas) newdata = self.clusterDataset.from_table(domain, self.clusterDataset) self.send("Data on Selected Genes", newdata) self.send("Data on Unselected Genes", None) elif self.candidateGeneAttrs: selectedExamples = [] unselectedExamples = [] geneAttr = self.candidateGeneAttrs[min(self.geneAttrIndex, len(self.candidateGeneAttrs)-1)] if self.selectionDisjoint == 1: goVar = Orange.data.DiscreteVariable( "GO Term", values=list(self.selectedTerms)) newDomain = Orange.data.Domain( self.clusterDataset.domain.variables, goVar, self.clusterDataset.domain.metas) goColumn = [] for i, ex in enumerate(self.clusterDataset): if not numpy.isnan(ex[geneAttr]) and any(gene in selectedGenes for gene in str(ex[geneAttr]).split(",")): if self.selectionDisjoint == 1 and self.selectionAddTermAsClass: terms = filter(lambda term: any(gene in self.graph[term][0] for gene in str(ex[geneAttr]).split(",")) , self.selectedTerms) term = sorted(terms)[0] goColumn.append(goVar.values.index(term)) selectedExamples.append(i) else: unselectedExamples.append(i) if selectedExamples: selectedExamples = self.clusterDataset[selectedExamples] if self.selectionDisjoint == 1 and self.selectionAddTermAsClass: selectedExamples = Orange.data.Table.from_table(newDomain, selectedExamples) view, issparse = selectedExamples.get_column_view(goVar) assert not issparse view[:] = goColumn else: selectedExamples = None if unselectedExamples: unselectedExamples = self.clusterDataset[unselectedExamples] else: unselectedExamples = None self.send("Data on Selected Genes", selectedExamples) self.send("Data on Unselected Genes", unselectedExamples) def ShowInfo(self): dialog = QDialog(self) dialog.setModal(False) dialog.setLayout(QVBoxLayout()) label = QLabel(dialog) label.setText("Ontology:\n" + self.ontology.header if self.ontology else "Ontology not loaded!") dialog.layout().addWidget(label) label = QLabel(dialog) label.setText("Annotations:\n" + self.annotations.header.replace("!", "") if self.annotations else "Annotations not loaded!") dialog.layout().addWidget(label) dialog.show() def onDeleteWidget(self): """Called before the widget is removed from the canvas. """ self.annotations = None self.ontology = None gc.collect() # Force collection
class OWWordEnrichment(OWWidget): # Basic widget info name = "Word Enrichment" description = "Word enrichment analysis for selected documents." icon = "icons/SetEnrichment.svg" priority = 600 # Input/output class Inputs: selected_data = Input("Selected Data", Table) data = Input("Data", Table) want_main_area = True class Error(OWWidget.Error): no_bow_features = Msg('No bag-of-words features!') no_words_overlap = Msg('No words overlap!') empty_selection = Msg('Selected data is empty!') all_selected = Msg('All examples can not be selected!') # Settings filter_by_p = Setting(False) filter_p_value = Setting(0.01) filter_by_fdr = Setting(True) filter_fdr_value = Setting(0.2) def __init__(self): super().__init__() # Init data self.data = None self.selected_data = None self.selected_data_transformed = None # used for transforming the 'selected data' into the 'data' domain self.words = [] self.p_values = [] self.fdr_values = [] # Info section fbox = gui.widgetBox(self.controlArea, "Info") self.info_all = gui.label(fbox, self, 'Cluster words:') self.info_sel = gui.label(fbox, self, 'Selected words:') self.info_fil = gui.label(fbox, self, 'After filtering:') # Filtering settings fbox = gui.widgetBox(self.controlArea, "Filter") hbox = gui.widgetBox(fbox, orientation=0) self.chb_p = gui.checkBox(hbox, self, "filter_by_p", "p-value", callback=self.filter_and_display, tooltip="Filter by word p-value") self.spin_p = gui.doubleSpin(hbox, self, 'filter_p_value', 1e-4, 1, step=1e-4, labelWidth=15, callback=self.filter_and_display, callbackOnReturn=True, tooltip="Max p-value for word") self.spin_p.setEnabled(self.filter_by_p) hbox = gui.widgetBox(fbox, orientation=0) self.chb_fdr = gui.checkBox(hbox, self, "filter_by_fdr", "FDR", callback=self.filter_and_display, tooltip="Filter by word FDR") self.spin_fdr = gui.doubleSpin(hbox, self, 'filter_fdr_value', 1e-4, 1, step=1e-4, labelWidth=15, callback=self.filter_and_display, callbackOnReturn=True, tooltip="Max p-value for word") self.spin_fdr.setEnabled(self.filter_by_fdr) gui.rubber(self.controlArea) # Word's list view self.cols = ['Word', 'p-value', 'FDR'] self.sig_words = QTreeWidget() self.sig_words.setColumnCount(len(self.cols)) self.sig_words.setHeaderLabels(self.cols) self.sig_words.setSortingEnabled(True) self.sig_words.setSelectionMode(QTreeView.ExtendedSelection) self.sig_words.sortByColumn(2, 0) # 0 is ascending order for i in range(len(self.cols)): self.sig_words.resizeColumnToContents(i) self.mainArea.layout().addWidget(self.sig_words) @Inputs.data def set_data(self, data=None): self.data = data @Inputs.selected_data def set_data_selected(self, data=None): self.selected_data = data def handleNewSignals(self): self.check_data() def get_bow_domain(self): domain = self.data.domain return Domain(attributes=[ a for a in domain.attributes if a.attributes.get('bow-feature', False) ], class_vars=domain.class_vars, metas=domain.metas, source=domain) def check_data(self): self.Error.clear() if isinstance(self.data, Table) and \ isinstance(self.selected_data, Table): if len(self.selected_data) == 0: self.Error.empty_selection() self.clear() return # keep only BoW features bow_domain = self.get_bow_domain() if len(bow_domain.attributes) == 0: self.Error.no_bow_features() self.clear() return self.data = Corpus.from_table(bow_domain, self.data) self.selected_data_transformed = Corpus.from_table( bow_domain, self.selected_data) if np_sp_sum(self.selected_data_transformed.X) == 0: self.Error.no_words_overlap() self.clear() elif len(self.data) == len(self.selected_data): self.Error.all_selected() self.clear() else: self.apply() else: self.clear() def clear(self): self.sig_words.clear() self.info_all.setText('Cluster words:') self.info_sel.setText('Selected words:') self.info_fil.setText('After filtering:') def filter_enabled(self, b): self.chb_p.setEnabled(b) self.chb_fdr.setEnabled(b) self.spin_p.setEnabled(b) self.spin_fdr.setEnabled(b) def filter_and_display(self): self.spin_p.setEnabled(self.filter_by_p) self.spin_fdr.setEnabled(self.filter_by_fdr) self.sig_words.clear() if self.selected_data_transformed is None: # do nothing when no Data return count = 0 if self.words: for word, pval, fval in zip(self.words, self.p_values, self.fdr_values): if (not self.filter_by_p or pval <= self.filter_p_value) and \ (not self.filter_by_fdr or fval <= self.filter_fdr_value): it = EATreeWidgetItem(word, pval, fval, self.sig_words) self.sig_words.addTopLevelItem(it) count += 1 for i in range(len(self.cols)): self.sig_words.resizeColumnToContents(i) self.info_all.setText('Cluster words: {}'.format( len(self.selected_data_transformed.domain.attributes))) self.info_sel.setText('Selected words: {}'.format( np.count_nonzero( np_sp_sum(self.selected_data_transformed.X, axis=0)))) if not self.filter_by_p and not self.filter_by_fdr: self.info_fil.setText('After filtering:') self.info_fil.setEnabled(False) else: self.info_fil.setEnabled(True) self.info_fil.setText('After filtering: {}'.format(count)) def progress(self, p): self.progressBarSet(p) def apply(self): self.clear() self.progressBarInit() self.filter_enabled(False) self.words = [ i.name for i in self.selected_data_transformed.domain.attributes ] self.p_values = hypergeom_p_values(self.data.X, self.selected_data_transformed.X, callback=self.progress) self.fdr_values = false_discovery_rate(self.p_values) self.filter_and_display() self.filter_enabled(True) self.progressBarFinished()
class OWPIPAx(widget.OWWidget): name = "PIPAx" description = "Access data from PIPA RNA-Seq database." icon = "../widgets/icons/PIPA.svg" priority = 35 inputs = [] outputs = [("Data", Orange.data.Table)] username = settings.Setting("") password = settings.Setting("") log2 = settings.Setting(False) rtypei = settings.Setting(5) # hardcoded rpkm mapability polya excludeconstant = settings.Setting(False) joinreplicates = settings.Setting(False) #: The stored current selection (in experiments view) #: SelectionByKey | None currentSelection = settings.Setting(None) #: Stored selections (presets) #: list of SelectionByKey storedSelections = settings.Setting([]) #: Stored column sort keys (from Sort view) #: list of strings storedSortingOrder = settings.Setting( ["Strain", "Experiment", "Genotype", "Timepoint"]) experimentsHeaderState = settings.Setting( {name: False for _, name in HEADER[:ID_INDEX + 1]} ) def __init__(self, parent=None, signalManager=None, name="PIPAx"): super().__init__(parent) self.selectedExperiments = [] self.buffer = dicty.CacheSQLite(bufferfile) self.searchString = "" self.result_types = [] self.mappings = {} self.controlArea.setMaximumWidth(250) self.controlArea.setMinimumWidth(250) gui.button(self.controlArea, self, "Reload", callback=self.Reload) gui.button(self.controlArea, self, "Clear cache", callback=self.clear_cache) b = gui.widgetBox(self.controlArea, "Experiment Sets") self.selectionSetsWidget = SelectionSetsWidget(self) self.selectionSetsWidget.setSizePolicy( QSizePolicy.Preferred, QSizePolicy.Maximum) def store_selections(modified): if not modified: self.storedSelections = self.selectionSetsWidget.selections self.selectionSetsWidget.selectionModified.connect(store_selections) b.layout().addWidget(self.selectionSetsWidget) gui.separator(self.controlArea) b = gui.widgetBox(self.controlArea, "Sort output columns") self.columnsSortingWidget = SortedListWidget(self) self.columnsSortingWidget.setSizePolicy( QSizePolicy.Preferred, QSizePolicy.Maximum) def store_sort_order(): self.storedSortingOrder = self.columnsSortingWidget.sortingOrder self.columnsSortingWidget.sortingOrderChanged.connect(store_sort_order) b.layout().addWidget(self.columnsSortingWidget) sorting_model = QStringListModel(SORTING_MODEL_LIST) self.columnsSortingWidget.setModel(sorting_model) gui.separator(self.controlArea) box = gui.widgetBox(self.controlArea, 'Expression Type') self.expressionTypesCB = gui.comboBox( box, self, "rtypei", items=[], callback=self.UpdateResultsList) gui.checkBox(self.controlArea, self, "excludeconstant", "Exclude labels with constant values") gui.checkBox(self.controlArea, self, "joinreplicates", "Average replicates (use median)") gui.checkBox(self.controlArea, self, "log2", "Logarithmic (base 2) transformation") self.commit_button = gui.button(self.controlArea, self, "&Commit", callback=self.Commit) self.commit_button.setDisabled(True) gui.rubber(self.controlArea) box = gui.widgetBox(self.controlArea, "Authentication") gui.lineEdit(box, self, "username", "Username:"******"password", "Password:"******"searchString", "Search", callbackOnType=True, callback=self.SearchUpdate) self.headerLabels = [t[1] for t in HEADER] self.experimentsWidget = QTreeWidget() self.experimentsWidget.setHeaderLabels(self.headerLabels) self.experimentsWidget.setSelectionMode(QTreeWidget.ExtendedSelection) self.experimentsWidget.setRootIsDecorated(False) self.experimentsWidget.setSortingEnabled(True) contextEventFilter = gui.VisibleHeaderSectionContextEventFilter( self.experimentsWidget, self.experimentsWidget ) self.experimentsWidget.header().installEventFilter(contextEventFilter) self.experimentsWidget.setItemDelegateForColumn( 0, gui.IndicatorItemDelegate(self, role=Qt.DisplayRole)) self.experimentsWidget.setAlternatingRowColors(True) self.experimentsWidget.selectionModel().selectionChanged.connect( self.onSelectionChanged) self.selectionSetsWidget.setSelectionModel( self.experimentsWidget.selectionModel() ) self.mainArea.layout().addWidget(self.experimentsWidget) # Restore the selection states from the stored settings self.selectionSetsWidget.selections = self.storedSelections self.columnsSortingWidget.sortingOrder = self.storedSortingOrder self.restoreHeaderState() self.experimentsWidget.header().geometriesChanged.connect( self.saveHeaderState) self.dbc = None self.AuthSet() QTimer.singleShot(100, self.UpdateExperiments) def sizeHint(self): return QSize(800, 600) def AuthSet(self): if len(self.username): self.passf.setDisabled(False) else: self.passf.setDisabled(True) def AuthChanged(self): self.AuthSet() self.ConnectAndUpdate() def ConnectAndUpdate(self): self.Connect() self.UpdateExperiments(reload=True) def Connect(self): self.error(1) self.warning(1) def en(x): return x if len(x) else None self.dbc = dicty.PIPAx(cache=self.buffer, username=en(self.username), password=self.password) # check password if en(self.username) != None: try: self.dbc.mappings(reload=True) except dicty.AuthenticationError: self.error(1, "Wrong username or password") self.dbc = None except Exception as ex: print("Error when contacting the PIPA database", ex) sys.excepthook(*sys.exc_info()) try: # maybe cached? self.dbc.mappings() self.warning(1, "Can not access database - using cached data.") except Exception as ex: self.dbc = None self.error(1, "Can not access database.") def Reload(self): self.UpdateExperiments(reload=True) def clear_cache(self): self.buffer.clear() self.Reload() def rtype(self): """Return selected result template type """ if self.result_types: return self.result_types[self.rtypei][0] else: return "-1" def UpdateExperimentTypes(self): self.expressionTypesCB.clear() items = [desc for _, desc in self.result_types] self.expressionTypesCB.addItems(items) self.rtypei = max(0, min(self.rtypei, len(self.result_types) - 1)) def UpdateExperiments(self, reload=False): self.experimentsWidget.clear() self.items = [] self.progressBarInit() if not self.dbc: self.Connect() mappings = {} result_types = [] sucind = False # success indicator for database index try: mappings = self.dbc.mappings(reload=reload) result_types = self.dbc.result_types(reload=reload) sucind = True except Exception as ex: try: mappings = self.dbc.mappings() result_types = self.dbc.result_types() self.warning(0, "Can not access database - using cached data.") sucind = True except Exception as ex: self.error(0, "Can not access database.") if sucind: self.warning(0) self.error(0) self.mappings = mappings self.result_types = result_types self.UpdateExperimentTypes() self.UpdateResultsList(reload=reload) self.progressBarFinished() if self.currentSelection: self.currentSelection.select( self.experimentsWidget.selectionModel()) self.handle_commit_button() def UpdateResultsList(self, reload=False): results_list = {} try: results_list = self.dbc.results_list(self.rtype(), reload=reload) except Exception as ex: try: results_list = self.dbc.results_list(self.rtype()) except Exception as ex: self.error(0, "Can not access database.") self.results_list = results_list mappings_key_dict = dict(((m["data_id"], m["id"]), key) \ for key, m in self.mappings.items()) def mapping_unique_id(annot): """Map annotations dict from results_list to unique `mappings` ids. """ data_id, mappings_id = annot["data_id"], annot["mappings_id"] return mappings_key_dict[data_id, mappings_id] elements = [] # softly change the view so that the selection stays the same items_shown = {} for i, item in enumerate(self.items): c = str(item.text(10)) items_shown[c] = i items_to_show = dict((mapping_unique_id(annot), annot) for annot in self.results_list.values()) add_items = set(items_to_show) - set(items_shown) delete_items = set(items_shown) - set(items_to_show) i = 0 while i < self.experimentsWidget.topLevelItemCount(): it = self.experimentsWidget.topLevelItem(i) if str(it.text(10)) in delete_items: self.experimentsWidget.takeTopLevelItem(i) else: i += 1 delete_ind = set([items_shown[i] for i in delete_items]) self.items = [it for i, it in enumerate(self.items) if i not in delete_ind] for r_annot in [items_to_show[i] for i in add_items]: d = defaultdict(lambda: "?", r_annot) row_items = [""] + [d.get(key, "?") for key, _ in HEADER[1:]] try: time_dict = literal_eval(row_items[DATE_INDEX]) date_rna = date(time_dict["fullYearUTC"], time_dict["monthUTC"] + 1, # Why is month 0 based? time_dict["dateUTC"]) row_items[DATE_INDEX] = date_rna.strftime("%x") except Exception: row_items[DATE_INDEX] = '' row_items[ID_INDEX] = mapping_unique_id(r_annot) elements.append(row_items) ci = MyTreeWidgetItem(self.experimentsWidget, row_items) self.items.append(ci) for i in range(len(self.headerLabels)): self.experimentsWidget.resizeColumnToContents(i) # which is the ok buffer version # FIXME: what attribute to use for version? self.wantbufver = \ lambda x, ad=self.results_list: \ defaultdict(lambda: "?", ad[x])["date"] self.wantbufver = lambda x: "0" self.UpdateCached() def UpdateCached(self): if self.wantbufver and self.dbc: fn = self.dbc.download_key_function() result_id_key = dict(((m["data_id"], m["mappings_id"]), key) \ for key, m in self.results_list.items()) for item in self.items: c = str(item.text(10)) mapping = self.mappings[c] data_id, mappings_id = mapping["data_id"], mapping["id"] r_id = result_id_key[data_id, mappings_id] # Get the buffered version buffered = self.dbc.inBuffer(fn(r_id)) value = " " if buffered == self.wantbufver(r_id) else "" item.setData(0, Qt.DisplayRole, value) def SearchUpdate(self, string=""): for item in self.items: item.setHidden(not all(s in item \ for s in self.searchString.split()) ) def Commit(self): if not self.dbc: self.Connect() pb = gui.ProgressBar(self, iterations=100) table = None ids = [] for item in self.experimentsWidget.selectedItems(): unique_id = str(item.text(10)) annots = self.mappings[unique_id] ids.append((annots["data_id"], annots["id"])) transfn = None if self.log2: transfn = lambda x: math.log(x + 1.0, 2) reverse_header_dict = dict((name, key) for key, name in HEADER) hview = self.experimentsWidget.header() shownHeaders = [label for i, label in \ list(enumerate(self.headerLabels))[1:] \ if not hview.isSectionHidden(i) ] allowed_labels = [reverse_header_dict.get(label, label) \ for label in shownHeaders] if self.joinreplicates and "id" not in allowed_labels: # need 'id' labels in join_replicates for attribute names allowed_labels.append("id") if len(ids): table = self.dbc.get_data(ids=ids, result_type=self.rtype(), callback=pb.advance, exclude_constant_labels=self.excludeconstant, # bufver=self.wantbufver, transform=transfn, allowed_labels=allowed_labels) if self.joinreplicates: table = dicty.join_replicates(table, ignorenames=["replicate", "data_id", "mappings_id", "data_name", "id", "unique_id"], namefn=None, avg=dicty.median ) # Sort attributes sortOrder = self.columnsSortingWidget.sortingOrder all_values = defaultdict(set) for at in table.domain.attributes: atts = at.attributes for name in sortOrder: all_values[name].add(atts.get(reverse_header_dict[name], "")) isnum = {} for at, vals in all_values.items(): vals = filter(None, vals) try: for a in vals: float(a) isnum[at] = True except: isnum[at] = False def optfloat(x, at): if x == "": return "" else: return float(x) if isnum[at] else x def sorting_key(attr): atts = attr.attributes return tuple([optfloat(atts.get(reverse_header_dict[name], ""), name) \ for name in sortOrder]) attributes = sorted(table.domain.attributes, key=sorting_key) domain = Orange.data.Domain( attributes, table.domain.class_var, table.domain.metas) table = table.from_table(domain, table) data_hints.set_hint(table, "taxid", "352472") data_hints.set_hint(table, "genesinrows", False) self.send("Data", table) self.UpdateCached() pb.finish() def onSelectionChanged(self, selected, deselected): self.handle_commit_button() def handle_commit_button(self): self.currentSelection = \ SelectionByKey(self.experimentsWidget.selectionModel().selection(), key=(1, 2, 3, 10)) self.commit_button.setDisabled(not len(self.currentSelection)) def saveHeaderState(self): hview = self.experimentsWidget.header() for i, label in enumerate(self.headerLabels): self.experimentsHeaderState[label] = hview.isSectionHidden(i) def restoreHeaderState(self): hview = self.experimentsWidget.header() state = self.experimentsHeaderState for i, label in enumerate(self.headerLabels): hview.setSectionHidden(i, state.get(label, True)) self.experimentsWidget.resizeColumnToContents(i)
class OWDatabasesUpdate(OWWidget): name = "Databases Update" description = "Update local systems biology databases." icon = "../widgets/icons/Databases.svg" priority = 10 inputs = [] outputs = [] want_main_area = False def __init__(self, parent=None, signalManager=None, name="Databases update"): OWWidget.__init__(self, parent, signalManager, name, wantMainArea=False) self.searchString = "" fbox = gui.widgetBox(self.controlArea, "Filter") self.completer = TokenListCompleter( self, caseSensitivity=Qt.CaseInsensitive) self.lineEditFilter = QLineEdit(textChanged=self.SearchUpdate) self.lineEditFilter.setCompleter(self.completer) fbox.layout().addWidget(self.lineEditFilter) box = gui.widgetBox(self.controlArea, "Files") self.filesView = QTreeWidget(self) self.filesView.setHeaderLabels( ["", "Data Source", "Update", "Last Updated", "Size"]) self.filesView.setRootIsDecorated(False) self.filesView.setUniformRowHeights(True) self.filesView.setSelectionMode(QAbstractItemView.NoSelection) self.filesView.setSortingEnabled(True) self.filesView.sortItems(1, Qt.AscendingOrder) self.filesView.setItemDelegateForColumn( 0, UpdateOptionsItemDelegate(self.filesView)) self.filesView.model().layoutChanged.connect(self.SearchUpdate) box.layout().addWidget(self.filesView) box = gui.widgetBox(self.controlArea, orientation="horizontal") self.updateButton = gui.button( box, self, "Update all", callback=self.UpdateAll, tooltip="Update all updatable files", ) self.downloadButton = gui.button( box, self, "Download all", callback=self.DownloadFiltered, tooltip="Download all filtered files shown" ) self.cancelButton = gui.button( box, self, "Cancel", callback=self.Cancel, tooltip="Cancel scheduled downloads/updates." ) self.retryButton = gui.button( box, self, "Reconnect", callback=self.RetrieveFilesList ) self.retryButton.hide() gui.rubber(box) self.warning(0) box = gui.widgetBox(self.controlArea, orientation="horizontal") gui.rubber(box) self.infoLabel = QLabel() self.infoLabel.setAlignment(Qt.AlignCenter) self.controlArea.layout().addWidget(self.infoLabel) self.infoLabel.setSizePolicy(QSizePolicy.Expanding, QSizePolicy.Fixed) self.updateItems = [] self.resize(800, 600) self.progress = ProgressState(self, maximum=3) self.progress.valueChanged.connect(self._updateProgress) self.progress.rangeChanged.connect(self._updateProgress) self.executor = ThreadExecutor( threadPool=QThreadPool(maxThreadCount=2) ) task = Task(self, function=self.RetrieveFilesList) task.exceptionReady.connect(self.HandleError) task.start() self._tasks = [] self._haveProgress = False def RetrieveFilesList(self): self.retryButton.hide() self.warning(0) self.progress.setRange(0, 3) task = Task(function=partial(retrieveFilesList, methodinvoke(self.progress, "advance"))) task.resultReady.connect(self.SetFilesList) task.exceptionReady.connect(self.HandleError) self.executor.submit(task) self.setEnabled(False) def SetFilesList(self, serverInfo): """ Set the files to show. """ self.setEnabled(True) localInfo = serverfiles.allinfo() all_tags = set() self.filesView.clear() self.updateItems = [] for item in join_info_dict(localInfo, serverInfo): tree_item = UpdateTreeWidgetItem(item) options_widget = UpdateOptionsWidget(item.state) options_widget.item = item options_widget.installClicked.connect( partial(self.SubmitDownloadTask, item.domain, item.filename) ) options_widget.removeClicked.connect( partial(self.SubmitRemoveTask, item.domain, item.filename) ) self.updateItems.append((item, tree_item, options_widget)) all_tags.update(item.tags) self.filesView.addTopLevelItems( [tree_item for _, tree_item, _ in self.updateItems] ) for item, tree_item, options_widget in self.updateItems: self.filesView.setItemWidget(tree_item, 0, options_widget) # Add an update button if the file is updateable if item.state == OUTDATED: button = QToolButton( None, text="Update", maximumWidth=120, minimumHeight=20, maximumHeight=20 ) if sys.platform == "darwin": button.setAttribute(Qt.WA_MacSmallSize) button.clicked.connect( partial(self.SubmitDownloadTask, item.domain, item.filename) ) self.filesView.setItemWidget(tree_item, 2, button) self.progress.advance() self.filesView.setColumnWidth(0, self.filesView.sizeHintForColumn(0)) for column in range(1, 4): contents_hint = self.filesView.sizeHintForColumn(column) header_hint = self.filesView.header().sectionSizeHint(column) width = max(min(contents_hint, 400), header_hint) self.filesView.setColumnWidth(column, width) hints = [hint for hint in sorted(all_tags) if not hint.startswith("#")] self.completer.setTokenList(hints) self.SearchUpdate() self.UpdateInfoLabel() self.toggleButtons() self.cancelButton.setEnabled(False) self.progress.setRange(0, 0) def buttonCheck(self, selected_items, state, button): for item in selected_items: if item.state != state: button.setEnabled(False) else: button.setEnabled(True) break def toggleButtons(self): selected_items = [item for item, tree_item, _ in self.updateItems if not tree_item.isHidden()] self.buttonCheck(selected_items, OUTDATED, self.updateButton) self.buttonCheck(selected_items, AVAILABLE, self.downloadButton) def HandleError(self, exception): if isinstance(exception, ConnectionError): self.warning(0, "Could not connect to server! Check your connection " "and try to reconnect.") self.SetFilesList({}) self.retryButton.show() else: sys.excepthook(type(exception), exception, None) self.progress.setRange(0, 0) self.setEnabled(True) def UpdateInfoLabel(self): local = [item for item, tree_item, _ in self.updateItems if item.state != AVAILABLE and not tree_item.isHidden()] size = sum(float(item.size) for item in local) onServer = [item for item, tree_item, _ in self.updateItems if not tree_item.isHidden()] sizeOnServer = sum(float(item.size) for item in onServer) text = ("%i items, %s (on server: %i items, %s)" % (len(local), sizeof_fmt(size), len(onServer), sizeof_fmt(sizeOnServer))) self.infoLabel.setText(text) def UpdateAll(self): self.warning(0) for item, tree_item, _ in self.updateItems: if item.state == OUTDATED and not tree_item.isHidden(): self.SubmitDownloadTask(item.domain, item.filename) def DownloadFiltered(self): # TODO: submit items in the order shown. for item, tree_item, _ in self.updateItems: if not tree_item.isHidden() and item.state in \ [AVAILABLE, OUTDATED]: self.SubmitDownloadTask(item.domain, item.filename) def SearchUpdate(self, searchString=None): strings = str(self.lineEditFilter.text()).split() for item, tree_item, _ in self.updateItems: hide = not all(UpdateItem_match(item, string) for string in strings) tree_item.setHidden(hide) self.UpdateInfoLabel() self.toggleButtons() def SubmitDownloadTask(self, domain, filename): """ Submit the (domain, filename) to be downloaded/updated. """ self.cancelButton.setEnabled(True) index = self.updateItemIndex(domain, filename) _, tree_item, opt_widget = self.updateItems[index] sf = LocalFiles(serverfiles.PATH, serverfiles.ServerFiles()) task = DownloadTask(domain, filename, sf) self.progress.adjustRange(0, 100) pb = ItemProgressBar(self.filesView) pb.setRange(0, 100) pb.setTextVisible(False) task.advanced.connect(pb.advance) task.advanced.connect(self.progress.advance) task.finished.connect(pb.hide) task.finished.connect(self.onDownloadFinished, Qt.QueuedConnection) task.exception.connect(self.onDownloadError, Qt.QueuedConnection) self.filesView.setItemWidget(tree_item, 2, pb) # Clear the text so it does not show behind the progress bar. tree_item.setData(2, Qt.DisplayRole, "") pb.show() # Disable the options widget opt_widget.setEnabled(False) self._tasks.append(task) self.executor.submit(task) def EndDownloadTask(self, task): future = task.future() index = self.updateItemIndex(task.domain, task.filename) item, tree_item, opt_widget = self.updateItems[index] self.filesView.removeItemWidget(tree_item, 2) opt_widget.setEnabled(True) if future.cancelled(): # Restore the previous state tree_item.setUpdateItem(item) opt_widget.setState(item.state) elif future.exception(): tree_item.setUpdateItem(item) opt_widget.setState(item.state) # Show the exception string in the size column. self.warning(0, "Error while downloading. Check your connection " "and retry.") # recreate button for download button = QToolButton( None, text="Retry", maximumWidth=120, minimumHeight=20, maximumHeight=20 ) if sys.platform == "darwin": button.setAttribute(Qt.WA_MacSmallSize) button.clicked.connect( partial(self.SubmitDownloadTask, item.domain, item.filename) ) self.filesView.setItemWidget(tree_item, 2, button) else: # get the new updated info dict and replace the the old item self.warning(0) info = serverfiles.info(item.domain, item.filename) new_item = update_item_from_info(item.domain, item.filename, info, info) self.updateItems[index] = (new_item, tree_item, opt_widget) tree_item.setUpdateItem(new_item) opt_widget.setState(new_item.state) self.UpdateInfoLabel() def SubmitRemoveTask(self, domain, filename): serverfiles.LOCALFILES.remove(domain, filename) index = self.updateItemIndex(domain, filename) item, tree_item, opt_widget = self.updateItems[index] if item.info_server: new_item = item._replace(state=AVAILABLE, local=None, info_local=None) else: new_item = item._replace(local=None, info_local=None) # Disable the options widget. No more actions can be performed # for the item. opt_widget.setEnabled(False) tree_item.setUpdateItem(new_item) opt_widget.setState(new_item.state) self.updateItems[index] = (new_item, tree_item, opt_widget) self.UpdateInfoLabel() def Cancel(self): """ Cancel all pending update/download tasks (that have not yet started). """ for task in self._tasks: task.future().cancel() def onDeleteWidget(self): self.Cancel() self.executor.shutdown(wait=False) OWWidget.onDeleteWidget(self) def onDownloadFinished(self): # on download completed/canceled/error assert QThread.currentThread() is self.thread() for task in list(self._tasks): future = task.future() if future.done(): self.EndDownloadTask(task) self._tasks.remove(task) if not self._tasks: # Clear/reset the overall progress self.progress.setRange(0, 0) self.cancelButton.setEnabled(False) def onDownloadError(self, exc_info): sys.excepthook(*exc_info) self.warning(0, "Error while downloading. Check your connection and " "retry.") def updateItemIndex(self, domain, filename): for i, (item, _, _) in enumerate(self.updateItems): if item.domain == domain and item.filename == filename: return i raise ValueError("%r, %r not in update list" % (domain, filename)) def _updateProgress(self, *args): rmin, rmax = self.progress.range() if rmin != rmax: if not self._haveProgress: self._haveProgress = True self.progressBarInit() self.progressBarSet(self.progress.ratioCompleted() * 100, processEvents=None) if rmin == rmax: self._haveProgress = False self.progressBarFinished()
class OWGOEnrichmentAnalysis(widget.OWWidget): name = "GO Browser" description = "Enrichment analysis for Gene Ontology terms." icon = "../widgets/icons/GOBrowser.svg" priority = 2020 inputs = [("Cluster Data", Orange.data.Table, "setDataset", widget.Single + widget.Default), ("Reference Data", Orange.data.Table, "setReferenceDataset")] outputs = [("Data on Selected Genes", Orange.data.Table), ("Data on Unselected Genes", Orange.data.Table), ("Data on Unknown Genes", Orange.data.Table), ("Enrichment Report", Orange.data.Table)] settingsHandler = settings.DomainContextHandler() annotationIndex = settings.ContextSetting(0) geneAttrIndex = settings.ContextSetting(0) useAttrNames = settings.ContextSetting(False) geneMatcherSettings = settings.Setting([True, False, False, False]) useReferenceDataset = settings.Setting(False) aspectIndex = settings.Setting(0) useEvidenceType = settings.Setting( {et: True for et in go.evidenceTypesOrdered}) filterByNumOfInstances = settings.Setting(False) minNumOfInstances = settings.Setting(1) filterByPValue = settings.Setting(True) maxPValue = settings.Setting(0.2) filterByPValue_nofdr = settings.Setting(False) maxPValue_nofdr = settings.Setting(0.01) probFunc = settings.Setting(0) selectionDirectAnnotation = settings.Setting(0) selectionDisjoint = settings.Setting(0) selectionAddTermAsClass = settings.Setting(0) Ready, Initializing, Running = 0, 1, 2 def __init__(self, parent=None): super().__init__(self, parent) self.clusterDataset = None self.referenceDataset = None self.ontology = None self.annotations = None self.loadedAnnotationCode = "---" self.treeStructRootKey = None self.probFunctions = [stats.Binomial(), stats.Hypergeometric()] self.selectedTerms = [] self.selectionChanging = 0 self.__state = OWGOEnrichmentAnalysis.Initializing self.annotationCodes = [] ############# ## GUI ############# self.tabs = gui.tabWidget(self.controlArea) ## Input tab self.inputTab = gui.createTabPage(self.tabs, "Input") box = gui.widgetBox(self.inputTab, "Info") self.infoLabel = gui.widgetLabel(box, "No data on input\n") gui.button( box, self, "Ontology/Annotation Info", callback=self.ShowInfo, tooltip="Show information on loaded ontology and annotations") box = gui.widgetBox(self.inputTab, "Organism") self.annotationComboBox = gui.comboBox(box, self, "annotationIndex", items=self.annotationCodes, callback=self._updateEnrichment, tooltip="Select organism") genebox = gui.widgetBox(self.inputTab, "Gene Names") self.geneAttrIndexCombo = gui.comboBox( genebox, self, "geneAttrIndex", callback=self._updateEnrichment, tooltip="Use this attribute to extract gene names from input data") self.geneAttrIndexCombo.setDisabled(self.useAttrNames) cb = gui.checkBox(genebox, self, "useAttrNames", "Use column names", tooltip="Use column names for gene names", callback=self._updateEnrichment) cb.toggled[bool].connect(self.geneAttrIndexCombo.setDisabled) gui.button(genebox, self, "Gene matcher settings", callback=self.UpdateGeneMatcher, tooltip="Open gene matching settings dialog") self.referenceRadioBox = gui.radioButtonsInBox( self.inputTab, self, "useReferenceDataset", ["Entire genome", "Reference set (input)"], tooltips=[ "Use entire genome for reference", "Use genes from Referece Examples input signal as reference" ], box="Reference", callback=self._updateEnrichment) self.referenceRadioBox.buttons[1].setDisabled(True) gui.radioButtonsInBox( self.inputTab, self, "aspectIndex", ["Biological process", "Cellular component", "Molecular function"], box="Aspect", callback=self._updateEnrichment) ## Filter tab self.filterTab = gui.createTabPage(self.tabs, "Filter") box = gui.widgetBox(self.filterTab, "Filter GO Term Nodes") gui.checkBox( box, self, "filterByNumOfInstances", "Genes", callback=self.FilterAndDisplayGraph, tooltip="Filter by number of input genes mapped to a term") ibox = gui.indentedBox(box) gui.spin(ibox, self, 'minNumOfInstances', 1, 100, step=1, label='#:', labelWidth=15, callback=self.FilterAndDisplayGraph, callbackOnReturn=True, tooltip="Min. number of input genes mapped to a term") gui.checkBox(box, self, "filterByPValue_nofdr", "p-value", callback=self.FilterAndDisplayGraph, tooltip="Filter by term p-value") gui.doubleSpin(gui.indentedBox(box), self, 'maxPValue_nofdr', 1e-8, 1, step=1e-8, label='p:', labelWidth=15, callback=self.FilterAndDisplayGraph, callbackOnReturn=True, tooltip="Max term p-value") #use filterByPValue for FDR, as it was the default in prior versions gui.checkBox(box, self, "filterByPValue", "FDR", callback=self.FilterAndDisplayGraph, tooltip="Filter by term FDR") gui.doubleSpin(gui.indentedBox(box), self, 'maxPValue', 1e-8, 1, step=1e-8, label='p:', labelWidth=15, callback=self.FilterAndDisplayGraph, callbackOnReturn=True, tooltip="Max term p-value") box = gui.widgetBox(box, "Significance test") gui.radioButtonsInBox(box, self, "probFunc", ["Binomial", "Hypergeometric"], tooltips=[ "Use binomial distribution test", "Use hypergeometric distribution test" ], callback=self._updateEnrichment) box = gui.widgetBox(self.filterTab, "Evidence codes in annotation", addSpace=True) self.evidenceCheckBoxDict = {} for etype in go.evidenceTypesOrdered: ecb = QCheckBox(etype, toolTip=go.evidenceTypes[etype], checked=self.useEvidenceType[etype]) ecb.toggled.connect(self.__on_evidenceChanged) box.layout().addWidget(ecb) self.evidenceCheckBoxDict[etype] = ecb ## Select tab self.selectTab = gui.createTabPage(self.tabs, "Select") box = gui.radioButtonsInBox(self.selectTab, self, "selectionDirectAnnotation", ["Directly or Indirectly", "Directly"], box="Annotated genes", callback=self.ExampleSelection) box = gui.widgetBox(self.selectTab, "Output", addSpace=True) gui.radioButtonsInBox( box, self, "selectionDisjoint", btnLabels=[ "All selected genes", "Term-specific genes", "Common term genes" ], tooltips=[ "Outputs genes annotated to all selected GO terms", "Outputs genes that appear in only one of selected GO terms", "Outputs genes common to all selected GO terms" ], callback=[self.ExampleSelection, self.UpdateAddClassButton]) self.addClassCB = gui.checkBox(box, self, "selectionAddTermAsClass", "Add GO Term as class", callback=self.ExampleSelection) # ListView for DAG, and table for significant GOIDs self.DAGcolumns = [ 'GO term', 'Cluster', 'Reference', 'p-value', 'FDR', 'Genes', 'Enrichment' ] self.splitter = QSplitter(Qt.Vertical, self.mainArea) self.mainArea.layout().addWidget(self.splitter) # list view self.listView = GOTreeWidget(self.splitter) self.listView.setSelectionMode(QTreeView.ExtendedSelection) self.listView.setAllColumnsShowFocus(1) self.listView.setColumnCount(len(self.DAGcolumns)) self.listView.setHeaderLabels(self.DAGcolumns) self.listView.header().setSectionsClickable(True) self.listView.header().setSortIndicatorShown(True) self.listView.setSortingEnabled(True) self.listView.setItemDelegateForColumn( 6, EnrichmentColumnItemDelegate(self)) self.listView.setRootIsDecorated(True) self.listView.itemSelectionChanged.connect(self.ViewSelectionChanged) # table of significant GO terms self.sigTerms = QTreeWidget(self.splitter) self.sigTerms.setColumnCount(len(self.DAGcolumns)) self.sigTerms.setHeaderLabels(self.DAGcolumns) self.sigTerms.setSortingEnabled(True) self.sigTerms.setSelectionMode(QTreeView.ExtendedSelection) self.sigTerms.setItemDelegateForColumn( 6, EnrichmentColumnItemDelegate(self)) self.sigTerms.itemSelectionChanged.connect(self.TableSelectionChanged) self.sigTableTermsSorted = [] self.graph = {} self.inputTab.layout().addStretch(1) self.filterTab.layout().addStretch(1) self.selectTab.layout().addStretch(1) self.setBlocking(True) self._executor = ThreadExecutor() self._init = EnsureDownloaded([(taxonomy.Taxonomy.DOMAIN, taxonomy.Taxonomy.FILENAME), ("GO", "taxonomy.pickle")]) self._init.finished.connect(self.__initialize_finish) self._executor.submit(self._init) def sizeHint(self): return QSize(1000, 700) def __initialize_finish(self): self.setBlocking(False) try: self.annotationFiles = listAvailable() except ConnectTimeout: self.error(2, "Internet connection error, unable to load data. " + \ "Check connection and create a new GO Browser widget.") self.filterTab.setEnabled(False) self.inputTab.setEnabled(False) self.selectTab.setEnabled(False) self.listView.setEnabled(False) self.sigTerms.setEnabled(False) else: self.annotationCodes = sorted(self.annotationFiles.keys()) self.annotationComboBox.clear() self.annotationComboBox.addItems(self.annotationCodes) self.annotationComboBox.setCurrentIndex(self.annotationIndex) self.__state = OWGOEnrichmentAnalysis.Ready def __on_evidenceChanged(self): for etype, cb in self.evidenceCheckBoxDict.items(): self.useEvidenceType[etype] = cb.isChecked() self._updateEnrichment() def UpdateGeneMatcher(self): """Open the Gene matcher settings dialog.""" dialog = GeneMatcherDialog(self, defaults=self.geneMatcherSettings, modal=True) if dialog.exec_() != QDialog.Rejected: self.geneMatcherSettings = [ getattr(dialog, item[0]) for item in dialog.items ] if self.annotations: self.SetGeneMatcher() self._updateEnrichment() def clear(self): self.infoLabel.setText("No data on input\n") self.warning(0) self.warning(1) self.geneAttrIndexCombo.clear() self.ClearGraph() self.send("Data on Selected Genes", None) self.send("Data on Unselected Genes", None) self.send("Data on Unknown Genes", None) self.send("Enrichment Report", None) def setDataset(self, data=None): if self.__state == OWGOEnrichmentAnalysis.Initializing: self.__initialize_finish() self.closeContext() self.clear() self.clusterDataset = data if data is not None: domain = data.domain allvars = domain.variables + domain.metas self.candidateGeneAttrs = [var for var in allvars if isstring(var)] self.geneAttrIndexCombo.clear() for var in self.candidateGeneAttrs: self.geneAttrIndexCombo.addItem(*gui.attributeItem(var)) taxid = data_hints.get_hint(data, "taxid", "") code = None try: code = go.from_taxid(taxid) except KeyError: pass except Exception as ex: print(ex) if code is not None: filename = "gene_association.%s.tar.gz" % code if filename in self.annotationFiles.values(): self.annotationIndex = \ [i for i, name in enumerate(self.annotationCodes) \ if self.annotationFiles[name] == filename].pop() self.useAttrNames = data_hints.get_hint(data, "genesinrows", self.useAttrNames) self.openContext(data) self.geneAttrIndex = min(self.geneAttrIndex, len(self.candidateGeneAttrs) - 1) if len(self.candidateGeneAttrs) == 0: self.useAttrNames = True self.geneAttrIndex = -1 elif self.geneAttrIndex < len(self.candidateGeneAttrs): self.geneAttrIndex = len(self.candidateGeneAttrs) - 1 self._updateEnrichment() def setReferenceDataset(self, data=None): self.referenceDataset = data self.referenceRadioBox.buttons[1].setDisabled(not bool(data)) self.referenceRadioBox.buttons[1].setText("Reference set") if self.clusterDataset is not None and self.useReferenceDataset: self.useReferenceDataset = 0 if not data else 1 graph = self.Enrichment() self.SetGraph(graph) elif self.clusterDataset: self.__updateReferenceSetButton() def handleNewSignals(self): super().handleNewSignals() def _updateEnrichment(self): if self.clusterDataset is not None and \ self.__state == OWGOEnrichmentAnalysis.Ready: pb = gui.ProgressBar(self, 100) self.Load(pb=pb) graph = self.Enrichment(pb=pb) self.FilterUnknownGenes() self.SetGraph(graph) def __updateReferenceSetButton(self): allgenes, refgenes = None, None if self.referenceDataset: try: allgenes = self.genesFromTable(self.referenceDataset) except Exception: allgenes = [] refgenes, unknown = self.FilterAnnotatedGenes(allgenes) self.referenceRadioBox.buttons[1].setDisabled(not bool(allgenes)) self.referenceRadioBox.buttons[1].setText("Reference set " + ( "(%i genes, %i matched)" % (len(allgenes), len(refgenes)) if allgenes and refgenes else "")) def genesFromTable(self, data): if self.useAttrNames: genes = [v.name for v in data.domain.variables] else: attr = self.candidateGeneAttrs[min( self.geneAttrIndex, len(self.candidateGeneAttrs) - 1)] genes = [str(ex[attr]) for ex in data if not numpy.isnan(ex[attr])] if any("," in gene for gene in genes): self.information( 0, "Separators detected in gene names. Assuming multiple genes per example." ) genes = reduce(operator.iadd, (genes.split(",") for genes in genes), []) return genes def FilterAnnotatedGenes(self, genes): matchedgenes = self.annotations.get_gene_names_translator( genes).values() return matchedgenes, [ gene for gene in genes if gene not in matchedgenes ] def FilterUnknownGenes(self): if not self.useAttrNames and self.candidateGeneAttrs: geneAttr = self.candidateGeneAttrs[min( self.geneAttrIndex, len(self.candidateGeneAttrs) - 1)] indices = [] for i, ex in enumerate(self.clusterDataset): if not any( self.annotations.genematcher.match(n.strip()) for n in str(ex[geneAttr]).split(",")): indices.append(i) if indices: data = self.clusterDataset[indices] else: data = None self.send("Data on Unknown Genes", data) else: self.send("Data on Unknown Genes", None) def Load(self, pb=None): if self.__state == OWGOEnrichmentAnalysis.Ready: go_files, tax_files = serverfiles.listfiles( "GO"), serverfiles.listfiles("Taxonomy") calls = [] pb, finish = (gui.ProgressBar(self, 0), True) if pb is None else (pb, False) count = 0 if not tax_files: calls.append(("Taxonomy", "ncbi_taxnomy.tar.gz")) count += 1 org = self.annotationCodes[min(self.annotationIndex, len(self.annotationCodes) - 1)] if org != self.loadedAnnotationCode: count += 1 if self.annotationFiles[org] not in go_files: calls.append(("GO", self.annotationFiles[org])) count += 1 if "gene_ontology_edit.obo.tar.gz" not in go_files: calls.append(("GO", "gene_ontology_edit.obo.tar.gz")) count += 1 if not self.ontology: count += 1 pb.iter += count * 100 for args in calls: serverfiles.localpath_download(*args, **dict(callback=pb.advance)) i = len(calls) if not self.ontology: self.ontology = go.Ontology( progress_callback=lambda value: pb.advance()) i += 1 if org != self.loadedAnnotationCode: self.annotations = None gc.collect() # Force run garbage collection code = self.annotationFiles[org].split(".")[-3] self.annotations = go.Annotations( code, genematcher=gene.GMDirect(), progress_callback=lambda value: pb.advance()) i += 1 self.loadedAnnotationCode = org count = defaultdict(int) geneSets = defaultdict(set) for anno in self.annotations.annotations: count[anno.evidence] += 1 geneSets[anno.evidence].add(anno.geneName) for etype in go.evidenceTypesOrdered: ecb = self.evidenceCheckBoxDict[etype] ecb.setEnabled(bool(count[etype])) ecb.setText(etype + ": %i annots(%i genes)" % (count[etype], len(geneSets[etype]))) if finish: pb.finish() def SetGeneMatcher(self): if self.annotations: taxid = self.annotations.taxid matchers = [] for matcher, use in zip( [gene.GMGO, gene.GMKEGG, gene.GMNCBI, gene.GMAffy], self.geneMatcherSettings): if use: try: if taxid == "352472": matchers.extend([ matcher(taxid), gene.GMDicty(), [matcher(taxid), gene.GMDicty()] ]) # The reason machers are duplicated is that we want `matcher` or `GMDicty` to # match genes by them self if possible. Only use the joint matcher if they fail. else: matchers.append(matcher(taxid)) except Exception as ex: print(ex) self.annotations.genematcher = gene.matcher(matchers) self.annotations.genematcher.set_targets( self.annotations.gene_names) def Enrichment(self, pb=None): assert self.clusterDataset is not None pb = gui.ProgressBar(self, 100) if pb is None else pb if not self.annotations.ontology: self.annotations.ontology = self.ontology if isinstance(self.annotations.genematcher, gene.GMDirect): self.SetGeneMatcher() self.error(1) self.warning([0, 1]) if self.useAttrNames: clusterGenes = [ v.name for v in self.clusterDataset.domain.attributes ] self.information(0) elif 0 <= self.geneAttrIndex < len(self.candidateGeneAttrs): geneAttr = self.candidateGeneAttrs[self.geneAttrIndex] clusterGenes = [ str(ex[geneAttr]) for ex in self.clusterDataset if not numpy.isnan(ex[geneAttr]) ] if any("," in gene for gene in clusterGenes): self.information( 0, "Separators detected in cluster gene names. Assuming multiple genes per example." ) clusterGenes = reduce(operator.iadd, (genes.split(",") for genes in clusterGenes), []) else: self.information(0) else: self.error(1, "Failed to extract gene names from input dataset!") return {} genesSetCount = len(set(clusterGenes)) self.clusterGenes = clusterGenes = self.annotations.get_gene_names_translator( clusterGenes).values() self.infoLabel.setText( "%i unique genes on input\n%i (%.1f%%) genes with known annotations" % (genesSetCount, len(clusterGenes), 100.0 * len(clusterGenes) / genesSetCount if genesSetCount else 0.0)) referenceGenes = None if not self.useReferenceDataset or self.referenceDataset is None: self.information(2) self.information(1) referenceGenes = self.annotations.gene_names elif self.referenceDataset is not None: if self.useAttrNames: referenceGenes = [ v.name for v in self.referenceDataset.domain.attributes ] self.information(1) elif geneAttr in (self.referenceDataset.domain.variables + self.referenceDataset.domain.metas): referenceGenes = [ str(ex[geneAttr]) for ex in self.referenceDataset if not numpy.isnan(ex[geneAttr]) ] if any("," in gene for gene in clusterGenes): self.information( 1, "Separators detected in reference gene names. Assuming multiple genes per example." ) referenceGenes = reduce(operator.iadd, (genes.split(",") for genes in referenceGenes), []) else: self.information(1) else: self.information(1) referenceGenes = None if referenceGenes is None: referenceGenes = list(self.annotations.gene_names) self.referenceRadioBox.buttons[1].setText("Reference set") self.referenceRadioBox.buttons[1].setDisabled(True) self.information( 2, "Unable to extract gene names from reference dataset. Using entire genome for reference" ) self.useReferenceDataset = 0 else: refc = len(referenceGenes) referenceGenes = self.annotations.get_gene_names_translator( referenceGenes).values() self.referenceRadioBox.buttons[1].setText( "Reference set (%i genes, %i matched)" % (refc, len(referenceGenes))) self.referenceRadioBox.buttons[1].setDisabled(False) self.information(2) else: self.useReferenceDataset = 0 if not referenceGenes: self.error(1, "No valid reference set") return {} self.referenceGenes = referenceGenes evidences = [] for etype in go.evidenceTypesOrdered: if self.useEvidenceType[etype]: evidences.append(etype) aspect = ["P", "C", "F"][self.aspectIndex] if clusterGenes: self.terms = terms = self.annotations.get_enriched_terms( clusterGenes, referenceGenes, evidences, aspect=aspect, prob=self.probFunctions[self.probFunc], use_fdr=False, progress_callback=lambda value: pb.advance()) ids = [] pvals = [] for i, d in self.terms.items(): ids.append(i) pvals.append(d[1]) for i, fdr in zip(ids, stats.FDR( pvals)): # save FDR as the last part of the tuple terms[i] = tuple(list(terms[i]) + [fdr]) else: self.terms = terms = {} if not self.terms: self.warning(0, "No enriched terms found.") else: self.warning(0) pb.finish() self.treeStructDict = {} ids = self.terms.keys() self.treeStructRootKey = None parents = {} for id in ids: parents[id] = set([term for _, term in self.ontology[id].related]) children = {} for term in self.terms: children[term] = set([id for id in ids if term in parents[id]]) for term in self.terms: self.treeStructDict[term] = TreeNode(self.terms[term], children[term]) if not self.ontology[term].related and not getattr( self.ontology[term], "is_obsolete", False): self.treeStructRootKey = term return terms def FilterGraph(self, graph): if self.filterByPValue_nofdr: graph = go.filterByPValue(graph, self.maxPValue_nofdr) if self.filterByPValue: #FDR graph = dict( filter(lambda item: item[1][3] <= self.maxPValue, graph.items())) if self.filterByNumOfInstances: graph = dict( filter(lambda item: len(item[1][0]) >= self.minNumOfInstances, graph.items())) return graph def FilterAndDisplayGraph(self): if self.clusterDataset: self.graph = self.FilterGraph(self.originalGraph) if self.originalGraph and not self.graph: self.warning(1, "All found terms were filtered out.") else: self.warning(1) self.ClearGraph() self.DisplayGraph() def SetGraph(self, graph=None): self.originalGraph = graph if graph: self.FilterAndDisplayGraph() else: self.graph = {} self.ClearGraph() def ClearGraph(self): self.listView.clear() self.listViewItems = [] self.sigTerms.clear() def DisplayGraph(self): fromParentDict = {} self.termListViewItemDict = {} self.listViewItems = [] enrichment = lambda t: len(t[0]) / t[2] * (len(self.referenceGenes) / len(self.clusterGenes)) maxFoldEnrichment = max( [enrichment(term) for term in self.graph.values()] or [1]) def addNode(term, parent, parentDisplayNode): if (parent, term) in fromParentDict: return if term in self.graph: displayNode = GOTreeWidgetItem(self.ontology[term], self.graph[term], len(self.clusterGenes), len(self.referenceGenes), maxFoldEnrichment, parentDisplayNode) displayNode.goId = term self.listViewItems.append(displayNode) if term in self.termListViewItemDict: self.termListViewItemDict[term].append(displayNode) else: self.termListViewItemDict[term] = [displayNode] fromParentDict[(parent, term)] = True parent = term else: displayNode = parentDisplayNode for c in self.treeStructDict[term].children: addNode(c, parent, displayNode) if self.treeStructDict: addNode(self.treeStructRootKey, None, self.listView) terms = self.graph.items() terms = sorted(terms, key=lambda item: item[1][1]) self.sigTableTermsSorted = [t[0] for t in terms] self.sigTerms.clear() for i, (t_id, (genes, p_value, refCount, fdr)) in enumerate(terms): item = GOTreeWidgetItem(self.ontology[t_id], (genes, p_value, refCount, fdr), len(self.clusterGenes), len(self.referenceGenes), maxFoldEnrichment, self.sigTerms) item.goId = t_id self.listView.expandAll() for i in range(5): self.listView.resizeColumnToContents(i) self.sigTerms.resizeColumnToContents(i) self.sigTerms.resizeColumnToContents(6) width = min(self.listView.columnWidth(0), 350) self.listView.setColumnWidth(0, width) self.sigTerms.setColumnWidth(0, width) # Create and send the enrichemnt report table. termsDomain = Orange.data.Domain( [], [], # All is meta! [ Orange.data.StringVariable("GO Term Id"), Orange.data.StringVariable("GO Term Name"), Orange.data.ContinuousVariable("Cluster Frequency"), Orange.data.ContinuousVariable("Genes in Cluster", number_of_decimals=0), Orange.data.ContinuousVariable("Reference Frequency"), Orange.data.ContinuousVariable("Genes in Reference", number_of_decimals=0), Orange.data.ContinuousVariable("p-value"), Orange.data.ContinuousVariable("FDR"), Orange.data.ContinuousVariable("Enrichment"), Orange.data.StringVariable("Genes") ]) terms = [[t_id, self.ontology[t_id].name, len(genes) / len(self.clusterGenes), len(genes), r_count / len(self.referenceGenes), r_count, p_value, fdr, len(genes) / len(self.clusterGenes) * \ len(self.referenceGenes) / r_count, ",".join(genes) ] for t_id, (genes, p_value, r_count, fdr) in terms] if terms: X = numpy.empty((len(terms), 0)) M = numpy.array(terms, dtype=object) termsTable = Orange.data.Table.from_numpy(termsDomain, X, metas=M) else: termsTable = Orange.data.Table(termsDomain) self.send("Enrichment Report", termsTable) def ViewSelectionChanged(self): if self.selectionChanging: return self.selectionChanging = 1 self.selectedTerms = [] selected = self.listView.selectedItems() self.selectedTerms = list(set([lvi.term.id for lvi in selected])) self.ExampleSelection() self.selectionChanging = 0 def TableSelectionChanged(self): if self.selectionChanging: return self.selectionChanging = 1 self.selectedTerms = [] selectedIds = set([ self.sigTerms.itemFromIndex(index).goId for index in self.sigTerms.selectedIndexes() ]) for i in range(self.sigTerms.topLevelItemCount()): item = self.sigTerms.topLevelItem(i) selected = item.goId in selectedIds term = item.goId if selected: self.selectedTerms.append(term) for lvi in self.termListViewItemDict[term]: try: lvi.setSelected(selected) if selected: lvi.setExpanded(True) except RuntimeError: # Underlying C/C++ object deleted pass self.ExampleSelection() self.selectionChanging = 0 def UpdateAddClassButton(self): self.addClassCB.setEnabled(self.selectionDisjoint == 1) def ExampleSelection(self): self.commit() def commit(self): if self.clusterDataset is None: return terms = set(self.selectedTerms) genes = reduce(operator.ior, (set(self.graph[term][0]) for term in terms), set()) evidences = [] for etype in go.evidenceTypesOrdered: if self.useEvidenceType[etype]: # if getattr(self, "useEvidence" + etype): evidences.append(etype) allTerms = self.annotations.get_annotated_terms( genes, direct_annotation_only=self.selectionDirectAnnotation, evidence_codes=evidences) if self.selectionDisjoint > 0: count = defaultdict(int) for term in self.selectedTerms: for g in allTerms.get(term, []): count[g] += 1 ccount = 1 if self.selectionDisjoint == 1 else len( self.selectedTerms) selectedGenes = [ gene for gene, c in count.items() if c == ccount and gene in genes ] else: selectedGenes = reduce(operator.ior, (set(allTerms.get(term, [])) for term in self.selectedTerms), set()) if self.useAttrNames: vars = [ self.clusterDataset.domain[gene] for gene in set(selectedGenes) ] domain = Orange.data.Domain(vars, self.clusterDataset.domain.class_vars, self.clusterDataset.domain.metas) newdata = self.clusterDataset.from_table(domain, self.clusterDataset) self.send("Data on Selected Genes", newdata) self.send("Data on Unselected Genes", None) elif self.candidateGeneAttrs: selectedExamples = [] unselectedExamples = [] geneAttr = self.candidateGeneAttrs[min( self.geneAttrIndex, len(self.candidateGeneAttrs) - 1)] if self.selectionDisjoint == 1: goVar = Orange.data.DiscreteVariable("GO Term", values=list( self.selectedTerms)) newDomain = Orange.data.Domain( self.clusterDataset.domain.variables, goVar, self.clusterDataset.domain.metas) goColumn = [] for i, ex in enumerate(self.clusterDataset): if not numpy.isnan(ex[geneAttr]) and any( gene in selectedGenes for gene in str(ex[geneAttr]).split(",")): if self.selectionDisjoint == 1 and self.selectionAddTermAsClass: terms = filter( lambda term: any(gene in self.graph[term][0] for gene in str(ex[geneAttr]). split(",")), self.selectedTerms) term = sorted(terms)[0] goColumn.append(goVar.values.index(term)) selectedExamples.append(i) else: unselectedExamples.append(i) if selectedExamples: selectedExamples = self.clusterDataset[selectedExamples] if self.selectionDisjoint == 1 and self.selectionAddTermAsClass: selectedExamples = Orange.data.Table.from_table( newDomain, selectedExamples) view, issparse = selectedExamples.get_column_view(goVar) assert not issparse view[:] = goColumn else: selectedExamples = None if unselectedExamples: unselectedExamples = self.clusterDataset[unselectedExamples] else: unselectedExamples = None self.send("Data on Selected Genes", selectedExamples) self.send("Data on Unselected Genes", unselectedExamples) def ShowInfo(self): dialog = QDialog(self) dialog.setModal(False) dialog.setLayout(QVBoxLayout()) label = QLabel(dialog) label.setText( "Ontology:\n" + self.ontology.header if self.ontology else "Ontology not loaded!") dialog.layout().addWidget(label) label = QLabel(dialog) label.setText("Annotations:\n" + self.annotations.header.replace("!", "") if self. annotations else "Annotations not loaded!") dialog.layout().addWidget(label) dialog.show() def onDeleteWidget(self): """Called before the widget is removed from the canvas. """ self.annotations = None self.ontology = None gc.collect() # Force collection
class OWDatabasesUpdate(OWWidget): name = "Databases Update" description = "Update local systems biology databases." icon = "../widgets/icons/OWDatabasesUpdate.svg" priority = 1 inputs = [] outputs = [] want_main_area = False def __init__(self, parent=None, signalManager=None, name="Databases update"): OWWidget.__init__(self, parent, signalManager, name, wantMainArea=False) self.searchString = "" fbox = gui.widgetBox(self.controlArea, "Filter") self.completer = TokenListCompleter(self, caseSensitivity=Qt.CaseInsensitive) self.lineEditFilter = QLineEdit(textChanged=self.search_update) self.lineEditFilter.setCompleter(self.completer) fbox.layout().addWidget(self.lineEditFilter) box = gui.widgetBox(self.controlArea, "Files") self.filesView = QTreeWidget(self) self.filesView.setHeaderLabels(header_labels) self.filesView.setRootIsDecorated(False) self.filesView.setUniformRowHeights(True) self.filesView.setSelectionMode(QAbstractItemView.NoSelection) self.filesView.setSortingEnabled(True) self.filesView.sortItems(header.Title, Qt.AscendingOrder) self.filesView.setItemDelegateForColumn( 0, UpdateOptionsItemDelegate(self.filesView)) self.filesView.model().layoutChanged.connect(self.search_update) box.layout().addWidget(self.filesView) layout = QHBoxLayout() gui.widgetBox(self.controlArea, margin=0, orientation=layout) self.updateButton = gui.button( box, self, "Update all", callback=self.update_all, tooltip="Update all updatable files", ) self.downloadButton = gui.button( box, self, "Download all", callback=self.download_filtered, tooltip="Download all filtered files shown") self.cancelButton = gui.button( box, self, "Cancel", callback=self.cancel_active_threads, tooltip="Cancel scheduled downloads/updates.") self.addButton = gui.button(box, self, "Add ...", callback=self.__handle_dialog, tooltip="Add files for personal use.") layout.addWidget(self.updateButton) layout.addWidget(self.downloadButton) layout.addWidget(self.cancelButton) layout.addStretch() layout.addWidget(self.addButton) # Enable retryButton once connection is established # self.retryButton = gui.button( # box, self, "Reconnect", callback=self.initialize_files_view # ) # self.retryButton.hide() self.resize(800, 600) self.update_items = [] self._dialog = None self.progress_bar = None # threads self.threadpool = QThreadPool(self) #self.threadpool.setMaxThreadCount(1) self.workers = list() self.initialize_files_view() def __handle_dialog(self): if not self._dialog: self._dialog = FileUploadHelper(self) self._dialog.show() def __progress_advance(self): # GUI should be updated in main thread. That's why we are calling advance method here if self.progress_bar: self.progress_bar.advance() def handle_worker_exception(self, ex): self.progress_bar.finish() self.setStatusMessage('') if isinstance(ex, ConnectionError): # TODO: set warning messages pass print(ex) def initialize_files_view(self): # self.retryButton.hide() # clear view self.filesView.clear() # init progress bar self.progress_bar = gui.ProgressBar(self, iterations=3) # status message self.setStatusMessage('initializing') worker = Worker(evaluate_files_state, progress_callback=True) worker.signals.progress.connect(self.__progress_advance) worker.signals.result.connect(self.set_files_list) worker.signals.error.connect(self.handle_worker_exception) # move download process to worker thread self.threadpool.start(worker) self.setEnabled(False) def __create_action_button(self, fs, retry=None): if not fs.state not in [OUTDATED, USER_FILE] or not retry: self.filesView.setItemWidget(fs.tree_item, header.Update, None) button = QToolButton(None) if not retry: if fs.state == OUTDATED: button.setText('Update') button.clicked.connect( partial(self.submit_download_task, fs.domain, fs.filename, True)) elif fs.state == USER_FILE: if not fs.info_server: button.setText('Remove') button.clicked.connect( partial(self.submit_remove_task, fs.domain, fs.filename)) else: button.setText('Use server version') button.clicked.connect( partial(self.submit_download_task, fs.domain, fs.filename, True)) else: button.setText('Retry') button.clicked.connect( partial(self.submit_download_task, fs.domain, fs.filename, True)) button.setMaximumWidth(120) button.setMaximumHeight(20) button.setMinimumHeight(20) if sys.platform == "darwin": button.setAttribute(Qt.WA_MacSmallSize) self.filesView.setItemWidget(fs.tree_item, header.Update, button) def set_files_list(self, result): """ Set the files to show. """ assert threading.current_thread() == threading.main_thread() self.progress_bar.finish() self.setStatusMessage('') self.setEnabled(True) self.update_items = result all_tags = set() for fs in self.update_items: fs.tree_item = FileStateItem(fs) fs.download_option = DownloadOption(state=fs.state) fs.download_option.download_clicked.connect( partial(self.submit_download_task, fs.domain, fs.filename)) fs.download_option.remove_clicked.connect( partial(self.submit_remove_task, fs.domain, fs.filename)) # add widget items to the QTreeWidget self.filesView.addTopLevelItems( [fs.tree_item for fs in self.update_items]) # add action widgets to tree items for fs in self.update_items: self.filesView.setItemWidget(fs.tree_item, header.Download, fs.download_option) if fs.state in [USER_FILE, OUTDATED]: self.__create_action_button(fs) all_tags.update(fs.tags) self.filesView.setColumnWidth( header.Download, self.filesView.sizeHintForColumn(header.Download)) for column in range(1, len(header_labels)): self.filesView.resizeColumnToContents(column) hints = [hint for hint in sorted(all_tags) if not hint.startswith("#")] self.completer.setTokenList(hints) self.search_update() self.toggle_action_buttons() self.cancelButton.setEnabled(False) def toggle_action_buttons(self): selected_items = [ fs for fs in self.update_items if not fs.tree_item.isHidden() ] def button_check(sel_items, state, button): for item in sel_items: if item.state != state: button.setEnabled(False) else: button.setEnabled(True) break button_check(selected_items, OUTDATED, self.updateButton) button_check(selected_items, AVAILABLE, self.downloadButton) def search_update(self, searchString=None): strings = str(self.lineEditFilter.text()).split() for fs in self.update_items: hide = not all(UpdateItem_match(fs, string) for string in strings) fs.tree_item.setHidden(hide) self.toggle_action_buttons() def update_all(self): for fs in self.update_items: if fs.state == OUTDATED and not fs.tree_item.isHidden(): self.submit_download_task(fs.domain, fs.filename) def download_filtered(self): for fs in self.update_items: if not fs.tree_item.isHidden() and fs.state in [ AVAILABLE, OUTDATED ]: self.submit_download_task(fs.domain, fs.filename, start=False) self.run_download_tasks() def submit_download_task(self, domain, filename, start=True): """ Submit the (domain, filename) to be downloaded/updated. """ # get selected tree item index = self.tree_item_index(domain, filename) fs = self.update_items[index] worker = Worker(download_server_file, fs, index, progress_callback=True) worker.signals.progress.connect(self.__progress_advance) worker.signals.result.connect(self.on_download_finished) worker.signals.error.connect(self.on_download_exception) self.workers.append(worker) if start: self.run_download_tasks() def run_download_tasks(self): self.cancelButton.setEnabled(True) # init progress bar self.progress_bar = gui.ProgressBar(self, iterations=len(self.workers) * 100) # status message self.setStatusMessage('downloading') # move workers to threadpool [self.threadpool.start(worker) for worker in self.workers] self.filesView.setDisabled(True) # reset list of workers self.workers = list() def on_download_exception(self, ex): assert threading.current_thread() == threading.main_thread() self.progress_bar.finish() self.setStatusMessage('') print(ex) if isinstance(ex, ValueError): fs, index = ex.args # restore state and retry fs.refresh_state() fs.tree_item.update_data(fs) fs.download_option.state = fs.state self.__create_action_button(fs, retry=True) def on_download_finished(self, result): assert threading.current_thread() == threading.main_thread() # We check if all workers have completed. If not, continue if self.progress_bar.count == 100 or self.threadpool.activeThreadCount( ) == 0: self.filesView.setDisabled(False) self.progress_bar.finish() self.setStatusMessage('') fs, index = result # re-evaluate File State info = serverfiles.info(fs.domain, fs.filename) fs.refresh_state(info_local=info, info_server=info) # reinitialize treeWidgetItem fs.tree_item.update_data(fs) # reinitialize OptionWidget fs.download_option.state = fs.state self.filesView.setItemWidget(fs.tree_item, header.Update, None) self.toggle_action_buttons() for column in range(1, len(header_labels)): self.filesView.resizeColumnToContents(column) def submit_remove_task(self, domain, filename): serverfiles.LOCALFILES.remove(domain, filename) index = self.tree_item_index(domain, filename) fs = self.update_items[index] if fs.state == USER_FILE: self.filesView.takeTopLevelItem( self.filesView.indexOfTopLevelItem(fs.tree_item)) self.update_items.remove(fs) # self.filesView.removeItemWidget(index) else: # refresh item state fs.info_local = None fs.refresh_state() # reinitialize treeWidgetItem fs.tree_item.update_data(fs) # reinitialize OptionWidget fs.download_option.state = fs.state self.toggle_action_buttons() def cancel_active_threads(self): """ Cancel all pending update/download tasks (that have not yet started). """ if self.threadpool: self.threadpool.clear() def tree_item_index(self, domain, filename): for i, fs in enumerate(self.update_items): if fs.domain == domain and fs.filename == filename: return i raise ValueError("%r, %r not in update list" % (domain, filename)) def onDeleteWidget(self): self.cancel_active_threads() OWWidget.onDeleteWidget(self)
class OWGOBrowser(widget.OWWidget): name = "GO Browser" description = "Enrichment analysis for Gene Ontology terms." icon = "../widgets/icons/OWGOBrowser.svg" priority = 7 inputs = [("Cluster Data", Orange.data.Table, "setDataset", widget.Single + widget.Default), ("Reference Data", Orange.data.Table, "setReferenceDataset")] outputs = [("Data on Selected Genes", Orange.data.Table), ("Enrichment Report", Orange.data.Table)] settingsHandler = settings.DomainContextHandler() geneAttrIndex = settings.ContextSetting(0) useAttrNames = settings.ContextSetting(False) useReferenceDataset = settings.Setting(False) aspectIndex = settings.Setting(0) useEvidenceType = settings.Setting( {et: True for et in go.evidenceTypesOrdered}) filterByNumOfInstances = settings.Setting(False) minNumOfInstances = settings.Setting(1) filterByPValue = settings.Setting(True) maxPValue = settings.Setting(0.2) filterByPValue_nofdr = settings.Setting(False) maxPValue_nofdr = settings.Setting(0.01) probFunc = settings.Setting(0) selectionDirectAnnotation = settings.Setting(0) selectionDisjoint = settings.Setting(0) class Error(widget.OWWidget.Error): serverfiles_unavailable = widget.Msg('Can not locate annotation files, ' 'please check your connection and try again.') missing_annotation = widget.Msg(ERROR_ON_MISSING_ANNOTATION) missing_gene_id = widget.Msg(ERROR_ON_MISSING_GENE_ID) missing_tax_id = widget.Msg(ERROR_ON_MISSING_TAX_ID) def __init__(self, parent=None): super().__init__(self, parent) self.input_data = None self.ref_data = None self.ontology = None self.annotations = None self.loaded_annotation_code = None self.treeStructRootKey = None self.probFunctions = [statistics.Binomial(), statistics.Hypergeometric()] self.selectedTerms = [] self.selectionChanging = 0 self.__state = State.Ready self.__scheduletimer = QTimer(self, singleShot=True) self.__scheduletimer.timeout.connect(self.__update) ############# # GUI ############# self.tabs = gui.tabWidget(self.controlArea) # Input tab self.inputTab = gui.createTabPage(self.tabs, "Input") box = gui.widgetBox(self.inputTab, "Info") self.infoLabel = gui.widgetLabel(box, "No data on input\n") gui.button(box, self, "Ontology/Annotation Info", callback=self.ShowInfo, tooltip="Show information on loaded ontology and annotations") self.referenceRadioBox = gui.radioButtonsInBox( self.inputTab, self, "useReferenceDataset", ["Entire genome", "Reference set (input)"], tooltips=["Use entire genome for reference", "Use genes from Referece Examples input signal as reference"], box="Reference", callback=self.__invalidate) self.referenceRadioBox.buttons[1].setDisabled(True) gui.radioButtonsInBox( self.inputTab, self, "aspectIndex", ["Biological process", "Cellular component", "Molecular function"], box="Aspect", callback=self.__invalidate) # Filter tab self.filterTab = gui.createTabPage(self.tabs, "Filter") box = gui.widgetBox(self.filterTab, "Filter GO Term Nodes") gui.checkBox(box, self, "filterByNumOfInstances", "Genes", callback=self.FilterAndDisplayGraph, tooltip="Filter by number of input genes mapped to a term") ibox = gui.indentedBox(box) gui.spin(ibox, self, 'minNumOfInstances', 1, 100, step=1, label='#:', labelWidth=15, callback=self.FilterAndDisplayGraph, callbackOnReturn=True, tooltip="Min. number of input genes mapped to a term") gui.checkBox(box, self, "filterByPValue_nofdr", "p-value", callback=self.FilterAndDisplayGraph, tooltip="Filter by term p-value") gui.doubleSpin(gui.indentedBox(box), self, 'maxPValue_nofdr', 1e-8, 1, step=1e-8, label='p:', labelWidth=15, callback=self.FilterAndDisplayGraph, callbackOnReturn=True, tooltip="Max term p-value") # use filterByPValue for FDR, as it was the default in prior versions gui.checkBox(box, self, "filterByPValue", "FDR", callback=self.FilterAndDisplayGraph, tooltip="Filter by term FDR") gui.doubleSpin(gui.indentedBox(box), self, 'maxPValue', 1e-8, 1, step=1e-8, label='p:', labelWidth=15, callback=self.FilterAndDisplayGraph, callbackOnReturn=True, tooltip="Max term p-value") box = gui.widgetBox(box, "Significance test") gui.radioButtonsInBox(box, self, "probFunc", ["Binomial", "Hypergeometric"], tooltips=["Use binomial distribution test", "Use hypergeometric distribution test"], callback=self.__invalidate) # TODO: only update the p values box = gui.widgetBox(self.filterTab, "Evidence codes in annotation", addSpace=True) self.evidenceCheckBoxDict = {} for etype in go.evidenceTypesOrdered: ecb = QCheckBox( etype, toolTip=go.evidenceTypes[etype], checked=self.useEvidenceType[etype]) ecb.toggled.connect(self.__on_evidenceChanged) box.layout().addWidget(ecb) self.evidenceCheckBoxDict[etype] = ecb # Select tab self.selectTab = gui.createTabPage(self.tabs, "Select") box = gui.radioButtonsInBox( self.selectTab, self, "selectionDirectAnnotation", ["Directly or Indirectly", "Directly"], box="Annotated genes", callback=self.ExampleSelection) box = gui.widgetBox(self.selectTab, "Output", addSpace=True) gui.radioButtonsInBox( box, self, "selectionDisjoint", btnLabels=["All selected genes", "Term-specific genes", "Common term genes"], tooltips=["Outputs genes annotated to all selected GO terms", "Outputs genes that appear in only one of selected GO terms", "Outputs genes common to all selected GO terms"], callback=self.ExampleSelection) # ListView for DAG, and table for significant GOIDs self.DAGcolumns = ['GO term', 'Cluster', 'Reference', 'p-value', 'FDR', 'Genes', 'Enrichment'] self.splitter = QSplitter(Qt.Vertical, self.mainArea) self.mainArea.layout().addWidget(self.splitter) # list view self.listView = GOTreeWidget(self.splitter) self.listView.setSelectionMode(QTreeView.ExtendedSelection) self.listView.setAllColumnsShowFocus(1) self.listView.setColumnCount(len(self.DAGcolumns)) self.listView.setHeaderLabels(self.DAGcolumns) self.listView.header().setSectionsClickable(True) self.listView.header().setSortIndicatorShown(True) self.listView.header().setSortIndicator(self.DAGcolumns.index('p-value'), Qt.AscendingOrder) self.listView.setSortingEnabled(True) self.listView.setItemDelegateForColumn( 6, EnrichmentColumnItemDelegate(self)) self.listView.setRootIsDecorated(True) self.listView.itemSelectionChanged.connect(self.ViewSelectionChanged) # table of significant GO terms self.sigTerms = QTreeWidget(self.splitter) self.sigTerms.setColumnCount(len(self.DAGcolumns)) self.sigTerms.setHeaderLabels(self.DAGcolumns) self.sigTerms.setSortingEnabled(True) self.sigTerms.setSelectionMode(QTreeView.ExtendedSelection) self.sigTerms.header().setSortIndicator(self.DAGcolumns.index('p-value'), Qt.AscendingOrder) self.sigTerms.setItemDelegateForColumn( 6, EnrichmentColumnItemDelegate(self)) self.sigTerms.itemSelectionChanged.connect(self.TableSelectionChanged) self.sigTableTermsSorted = [] self.graph = {} self.originalGraph = None self.inputTab.layout().addStretch(1) self.filterTab.layout().addStretch(1) self.selectTab.layout().addStretch(1) class AnnotationSlot(SimpleNamespace): taxid = ... # type: str name = ... # type: str filename = ... # type:str @staticmethod def parse_tax_id(f_name): return f_name.split('.')[1] try: remote_files = serverfiles.ServerFiles().listfiles(DOMAIN) except (ConnectTimeout, RequestException, ConnectionError): # TODO: Warn user about failed connection to the remote server remote_files = [] self.available_annotations = [ AnnotationSlot( taxid=AnnotationSlot.parse_tax_id(annotation_file), name=taxonomy.common_taxid_to_name(AnnotationSlot.parse_tax_id(annotation_file)), filename=FILENAME_ANNOTATION.format(AnnotationSlot.parse_tax_id(annotation_file)) ) for _, annotation_file in set(remote_files + serverfiles.listfiles(DOMAIN)) if annotation_file != FILENAME_ONTOLOGY ] self._executor = ThreadExecutor() def sizeHint(self): return QSize(1000, 700) def __on_evidenceChanged(self): for etype, cb in self.evidenceCheckBoxDict.items(): self.useEvidenceType[etype] = cb.isChecked() self.__invalidate() def clear(self): self.infoLabel.setText("No data on input\n") self.warning(0) self.warning(1) self.ClearGraph() self.send("Data on Selected Genes", None) self.send("Enrichment Report", None) def setDataset(self, data=None): self.closeContext() self.clear() self.Error.clear() if data: self.input_data = data self.tax_id = str(self.input_data.attributes.get(TAX_ID, None)) self.use_attr_names = self.input_data.attributes.get(GENE_AS_ATTRIBUTE_NAME, None) self.gene_id_attribute = self.input_data.attributes.get(GENE_ID_ATTRIBUTE, None) self.gene_id_column = self.input_data.attributes.get(GENE_ID_COLUMN, None) self.annotation_index = None if not(self.use_attr_names is not None and ((self.gene_id_attribute is None) ^ (self.gene_id_column is None))): if self.tax_id is None: self.Error.missing_annotation() return self.Error.missing_gene_id() return elif self.tax_id is None: self.Error.missing_tax_id() return _c2i = {a.taxid: i for i, a in enumerate(self.available_annotations)} try: self.annotation_index = _c2i[self.tax_id] except KeyError: self.Error.serverfiles_unavailable() # raise ValueError('Taxonomy {} not supported.'.format(self.tax_id)) return self.__invalidate() def setReferenceDataset(self, data=None): self.Error.clear() if data: self.ref_data = data self.ref_tax_id = str(self.ref_data.attributes.get(TAX_ID, None)) self.ref_use_attr_names = self.ref_data.attributes.get(GENE_AS_ATTRIBUTE_NAME, None) self.ref_gene_id_attribute = self.ref_data.attributes.get(GENE_ID_ATTRIBUTE, None) self.ref_gene_id_column = self.ref_data.attributes.get(GENE_ID_COLUMN, None) if not (self.ref_use_attr_names is not None and ((self.ref_gene_id_attribute is None) ^ (self.ref_gene_id_column is None))): if self.ref_tax_id is None: self.Error.missing_annotation() return self.Error.missing_gene_id() return elif self.ref_tax_id is None: self.Error.missing_tax_id() return self.referenceRadioBox.buttons[1].setDisabled(not bool(data)) self.referenceRadioBox.buttons[1].setText("Reference set") if self.input_data is not None and self.useReferenceDataset: self.useReferenceDataset = 0 if not data else 1 self.__invalidate() @Slot() def __invalidate(self): # Invalidate the current results or pending task and schedule an # update. self.__scheduletimer.start() if self.__state != State.Ready: self.__state |= State.Stale self.SetGraph({}) self.ref_genes = None self.input_genes = None def __invalidateAnnotations(self): self.annotations = None self.loaded_annotation_code = None if self.input_data: self.infoLabel.setText("...\n") self.__invalidate() @Slot() def __update(self): self.__scheduletimer.stop() if self.input_data is None: return if self.__state & State.Running: self.__state |= State.Stale elif self.__state & State.Downloading: self.__state |= State.Stale elif self.__state & State.Ready: if self.__ensure_data(): self.Load() self.Enrichment() else: assert self.__state & State.Downloading assert self.isBlocking() def __get_ref_genes(self): self.ref_genes = [] if self.ref_use_attr_names: for variable in self.input_data.domain.attributes: self.ref_genes.append(str(variable.attributes.get(self.ref_gene_id_attribute, '?'))) else: genes, _ = self.ref_data.get_column_view(self.ref_gene_id_column) self.ref_genes = [str(g) for g in genes] def __get_input_genes(self): self.input_genes = [] if self.use_attr_names: for variable in self.input_data.domain.attributes: self.input_genes .append(str(variable.attributes.get(self.gene_id_attribute, '?'))) else: genes, _ = self.input_data.get_column_view(self.gene_id_column) self.input_genes = [str(g) for g in genes] def FilterAnnotatedGenes(self, genes): matchedgenes = self.annotations.get_gene_names_translator(genes).values() return matchedgenes, [gene for gene in genes if gene not in matchedgenes] def __start_download(self, files_list): # type: (List[Tuple[str, str]]) -> None task = EnsureDownloaded(files_list) task.progress.connect(self._progressBarSet) f = self._executor.submit(task) fw = FutureWatcher(f, self) fw.finished.connect(self.__download_finish) fw.finished.connect(fw.deleteLater) fw.resultReady.connect(self.__invalidate) self.progressBarInit(processEvents=None) self.setBlocking(True) self.setStatusMessage("Downloading") self.__state = State.Downloading @Slot(Future) def __download_finish(self, result): # type: (Future[None]) -> None assert QThread.currentThread() is self.thread() assert result.done() self.setBlocking(False) self.setStatusMessage("") self.progressBarFinished(processEvents=False) try: result.result() except ConnectTimeout: logging.getLogger(__name__).error("Error:") self.error(2, "Internet connection error, unable to load data. " + "Check connection and create a new GO Browser widget.") except RequestException as err: logging.getLogger(__name__).error("Error:") self.error(2, "Internet error:\n" + str(err)) except BaseException as err: logging.getLogger(__name__).error("Error:") self.error(2, "Error:\n" + str(err)) raise else: self.error(2) finally: self.__state = State.Ready def __ensure_data(self): # Ensure that all required database (ontology and annotations for # the current selected organism are present. If not start a download in # the background. Return True if all dbs are present and false # otherwise assert self.__state == State.Ready annotation = self.available_annotations[self.annotation_index] go_files = [fname for domain, fname in serverfiles.listfiles(DOMAIN)] files = [] if annotation.filename not in go_files: files.append(("GO", annotation.filename)) if FILENAME_ONTOLOGY not in go_files: files.append((DOMAIN, FILENAME_ONTOLOGY)) if files: self.__start_download(files) assert self.__state == State.Downloading return False else: return True def Load(self): a = self.available_annotations[self.annotation_index] if self.ontology is None: self.ontology = go.Ontology() if a.taxid != self.loaded_annotation_code: self.annotations = None gc.collect() # Force run garbage collection self.annotations = go.Annotations(a.taxid) self.loaded_annotation_code = a.taxid count = defaultdict(int) geneSets = defaultdict(set) for anno in self.annotations.annotations: count[anno.evidence] += 1 geneSets[anno.evidence].add(anno.gene_id) for etype in go.evidenceTypesOrdered: ecb = self.evidenceCheckBoxDict[etype] ecb.setEnabled(bool(count[etype])) ecb.setText(etype + ": %i annots(%i genes)" % (count[etype], len(geneSets[etype]))) def Enrichment(self): assert self.input_data is not None assert self.__state == State.Ready if not self.annotations.ontology: self.annotations.ontology = self.ontology self.error(1) self.warning([0, 1]) self.__get_input_genes() self.input_genes = set(self.input_genes) self.known_input_genes = self.annotations.get_genes_with_known_annotation(self.input_genes) # self.clusterGenes = clusterGenes = self.annotations.map_to_ncbi_id(self.input_genes).values() self.infoLabel.setText("%i unique genes on input\n%i (%.1f%%) genes with known annotations" % (len(self.input_genes), len(self.known_input_genes), 100.0*len(self.known_input_genes)/len(self.input_genes) if len(self.input_genes) else 0.0)) if not self.useReferenceDataset or self.ref_data is None: self.information(2) self.information(1) self.ref_genes = self.annotations.genes() self.ref_genes = set(self.ref_genes) elif self.ref_data is not None: self.__get_ref_genes() self.ref_genes = set(self.ref_genes) ref_count = len(self.ref_genes) if ref_count == 0: self.ref_genes = self.annotations.genes() self.referenceRadioBox.buttons[1].setText("Reference set") self.referenceRadioBox.buttons[1].setDisabled(True) self.information(2, "Unable to extract gene names from reference dataset. " "Using entire genome for reference") self.useReferenceDataset = 0 else: self.referenceRadioBox.buttons[1].setText("Reference set ({} genes)".format(ref_count)) self.referenceRadioBox.buttons[1].setDisabled(False) self.information(2) else: self.useReferenceDataset = 0 self.ref_genes = [] if not self.ref_genes: self.error(1, "No valid reference set") return {} evidences = [] for etype in go.evidenceTypesOrdered: if self.useEvidenceType[etype]: evidences.append(etype) aspect = ['Process', 'Component', 'Function'][self.aspectIndex] self.progressBarInit(processEvents=False) self.setBlocking(True) self.__state = State.Running if self.input_genes: f = self._executor.submit( self.annotations.get_enriched_terms, self.input_genes, self.ref_genes, evidences, aspect=aspect, prob=self.probFunctions[self.probFunc], use_fdr=False, progress_callback=methodinvoke( self, "_progressBarSet", (float,)) ) fw = FutureWatcher(f, parent=self) fw.done.connect(self.__on_enrichment_done) fw.done.connect(fw.deleteLater) return else: f = Future() f.set_result({}) self.__on_enrichment_done(f) def __on_enrichment_done(self, results): # type: (Future[Dict[str, tuple]]) -> None self.progressBarFinished(processEvents=False) self.setBlocking(False) self.setStatusMessage("") if self.__state & State.Stale: self.__state = State.Ready self.__invalidate() return self.__state = State.Ready try: results = results.result() # type: Dict[str, tuple] except Exception as ex: results = {} error = str(ex) self.error(1, error) if results: terms = list(results.items()) fdr_vals = statistics.FDR([d[1] for _, d in terms]) terms = [(key, d + (fdr,)) for (key, d), fdr in zip(terms, fdr_vals)] terms = dict(terms) else: terms = {} self.terms = terms if not self.terms: self.warning(0, "No enriched terms found.") else: self.warning(0) self.treeStructDict = {} ids = self.terms.keys() self.treeStructRootKey = None parents = {} for id in ids: parents[id] = set([term for _, term in self.ontology[id].related]) children = {} for term in self.terms: children[term] = set([id for id in ids if term in parents[id]]) for term in self.terms: self.treeStructDict[term] = TreeNode(self.terms[term], children[term]) if not self.ontology[term].related and not getattr(self.ontology[term], "is_obsolete", False): self.treeStructRootKey = term self.SetGraph(terms) self._updateEnrichmentReportOutput() self.commit() def _updateEnrichmentReportOutput(self): terms = sorted(self.terms.items(), key=lambda item: item[1][1]) # Create and send the enrichemnt report table. termsDomain = Orange.data.Domain( [], [], # All is meta! [Orange.data.StringVariable("GO Term Id"), Orange.data.StringVariable("GO Term Name"), Orange.data.ContinuousVariable("Cluster Frequency"), Orange.data.ContinuousVariable("Genes in Cluster", number_of_decimals=0), Orange.data.ContinuousVariable("Reference Frequency"), Orange.data.ContinuousVariable("Genes in Reference", number_of_decimals=0), Orange.data.ContinuousVariable("p-value"), Orange.data.ContinuousVariable("FDR"), Orange.data.ContinuousVariable("Enrichment"), Orange.data.StringVariable("Genes")]) terms = [[t_id, self.ontology[t_id].name, len(genes) / len(self.input_genes), len(genes), r_count / len(self.ref_genes), r_count, p_value, fdr, len(genes) / len(self.input_genes) * \ len(self.ref_genes) / r_count, ",".join(genes) ] for t_id, (genes, p_value, r_count, fdr) in terms if genes and r_count] if terms: X = numpy.empty((len(terms), 0)) M = numpy.array(terms, dtype=object) termsTable = Orange.data.Table.from_numpy(termsDomain, X, metas=M) else: termsTable = None self.send("Enrichment Report", termsTable) @Slot(float) def _progressBarSet(self, value): assert QThread.currentThread() is self.thread() self.progressBarSet(value, processEvents=None) @Slot() def _progressBarFinish(self): assert QThread.currentThread() is self.thread() self.progressBarFinished(processEvents=None) def FilterGraph(self, graph): if self.filterByPValue_nofdr: graph = go.filterByPValue(graph, self.maxPValue_nofdr) if self.filterByPValue: # FDR graph = dict(filter(lambda item: item[1][3] <= self.maxPValue, graph.items())) if self.filterByNumOfInstances: graph = dict(filter(lambda item: len(item[1][0]) >= self.minNumOfInstances, graph.items())) return graph def FilterAndDisplayGraph(self): if self.input_data and self.originalGraph is not None: self.graph = self.FilterGraph(self.originalGraph) if self.originalGraph and not self.graph: self.warning(1, "All found terms were filtered out.") else: self.warning(1) self.ClearGraph() self.DisplayGraph() def SetGraph(self, graph=None): self.originalGraph = graph if graph: self.FilterAndDisplayGraph() else: self.graph = {} self.ClearGraph() def ClearGraph(self): self.listView.clear() self.listViewItems=[] self.sigTerms.clear() def DisplayGraph(self): fromParentDict = {} self.termListViewItemDict = {} self.listViewItems = [] def enrichment(t): try: return len(t[0]) / t[2] * (len(self.ref_genes) / len(self.input_genes)) except ZeroDivisionError: # TODO: find out why this happens return 0 maxFoldEnrichment = max([enrichment(term) for term in self.graph.values()] or [1]) def addNode(term, parent, parentDisplayNode): if (parent, term) in fromParentDict: return if term in self.graph: displayNode = GOTreeWidgetItem(self.ontology[term], self.graph[term], len(self.input_genes), len(self.ref_genes), maxFoldEnrichment, parentDisplayNode) displayNode.goId = term self.listViewItems.append(displayNode) if term in self.termListViewItemDict: self.termListViewItemDict[term].append(displayNode) else: self.termListViewItemDict[term] = [displayNode] fromParentDict[(parent, term)] = True parent = term else: displayNode = parentDisplayNode for c in self.treeStructDict[term].children: addNode(c, parent, displayNode) if self.treeStructDict: addNode(self.treeStructRootKey, None, self.listView) terms = self.graph.items() terms = sorted(terms, key=lambda item: item[1][1]) self.sigTableTermsSorted = [t[0] for t in terms] self.sigTerms.clear() for i, (t_id, (genes, p_value, refCount, fdr)) in enumerate(terms): item = GOTreeWidgetItem(self.ontology[t_id], (genes, p_value, refCount, fdr), len(self.input_genes), len(self.ref_genes), maxFoldEnrichment, self.sigTerms) item.goId = t_id self.listView.expandAll() for i in range(5): self.listView.resizeColumnToContents(i) self.sigTerms.resizeColumnToContents(i) self.sigTerms.resizeColumnToContents(6) width = min(self.listView.columnWidth(0), 350) self.listView.setColumnWidth(0, width) self.sigTerms.setColumnWidth(0, width) def ViewSelectionChanged(self): if self.selectionChanging: return self.selectionChanging = 1 self.selectedTerms = [] selected = self.listView.selectedItems() self.selectedTerms = list(set([lvi.term.id for lvi in selected])) self.ExampleSelection() self.selectionChanging = 0 def TableSelectionChanged(self): if self.selectionChanging: return self.selectionChanging = 1 self.selectedTerms = [] selectedIds = set([self.sigTerms.itemFromIndex(index).goId for index in self.sigTerms.selectedIndexes()]) for i in range(self.sigTerms.topLevelItemCount()): item = self.sigTerms.topLevelItem(i) selected = item.goId in selectedIds term = item.goId if selected: self.selectedTerms.append(term) for lvi in self.termListViewItemDict[term]: try: lvi.setSelected(selected) if selected: lvi.setExpanded(True) except RuntimeError: # Underlying C/C++ object deleted pass self.selectionChanging = 0 self.ExampleSelection() def ExampleSelection(self): self.commit() def commit(self): if self.input_data is None or self.originalGraph is None or \ self.annotations is None: return if self.__state & State.Stale: return terms = set(self.selectedTerms) genes = reduce(operator.ior, (set(self.graph[term][0]) for term in terms), set()) evidences = [] for etype in go.evidenceTypesOrdered: if self.useEvidenceType[etype]: evidences.append(etype) allTerms = self.annotations.get_annotated_terms( genes, direct_annotation_only=self.selectionDirectAnnotation, evidence_codes=evidences) if self.selectionDisjoint > 0: count = defaultdict(int) for term in self.selectedTerms: for g in allTerms.get(term, []): count[g] += 1 ccount = 1 if self.selectionDisjoint == 1 else len(self.selectedTerms) selected_genes = [gene for gene, c in count.items() if c == ccount and gene in genes] else: selected_genes = reduce( operator.ior, (set(allTerms.get(term, [])) for term in self.selectedTerms), set()) if self.use_attr_names: selected = [column for column in self.input_data.domain.attributes if self.gene_id_attribute in column.attributes and str(column.attributes[self.gene_id_attribute]) in set(selected_genes)] domain = Orange.data.Domain(selected, self.input_data.domain.class_vars, self.input_data.domain.metas) new_data = self.input_data.from_table(domain, self.input_data) self.send("Data on Selected Genes", new_data) else: selected_rows = [] for row_index, row in enumerate(self.input_data): gene_in_row = str(row[self.gene_id_column]) if gene_in_row in self.input_genes and gene_in_row in selected_genes: selected_rows.append(row_index) if selected_rows: selected = self.input_data[selected_rows] else: selected = None self.send("Data on Selected Genes", selected) def ShowInfo(self): dialog = QDialog(self) dialog.setModal(False) dialog.setLayout(QVBoxLayout()) label = QLabel(dialog) label.setText("Ontology:\n" + self.ontology.header if self.ontology else "Ontology not loaded!") dialog.layout().addWidget(label) label = QLabel(dialog) label.setText("Annotations:\n" + self.annotations.header.replace("!", "") if self.annotations else "Annotations not loaded!") dialog.layout().addWidget(label) dialog.show() def onDeleteWidget(self): """Called before the widget is removed from the canvas. """ self.annotations = None self.ontology = None gc.collect() # Force collection
class OWWordEnrichment(OWWidget): # Basic widget info name = "Word Enrichment" description = "Word enrichment analysis for selected documents." icon = "icons/SetEnrichment.svg" priority = 600 # Input/output class Inputs: selected_data = Input("Selected Data", Table) data = Input("Data", Table) want_main_area = True class Error(OWWidget.Error): no_bow_features = Msg('No bag-of-words features!') no_words_overlap = Msg('No words overlap!') empty_selection = Msg('Selected data is empty!') all_selected = Msg('All examples can not be selected!') # Settings filter_by_p = Setting(False) filter_p_value = Setting(0.01) filter_by_fdr = Setting(True) filter_fdr_value = Setting(0.2) def __init__(self): super().__init__() # Init data self.data = None self.selected_data = None self.selected_data_transformed = None # used for transforming the 'selected data' into the 'data' domain self.words = [] self.p_values = [] self.fdr_values = [] # Info section fbox = gui.widgetBox(self.controlArea, "Info") self.info_all = gui.label(fbox, self, 'Cluster words:') self.info_sel = gui.label(fbox, self, 'Selected words:') self.info_fil = gui.label(fbox, self, 'After filtering:') # Filtering settings fbox = gui.widgetBox(self.controlArea, "Filter") hbox = gui.widgetBox(fbox, orientation=0) self.chb_p = gui.checkBox(hbox, self, "filter_by_p", "p-value", callback=self.filter_and_display, tooltip="Filter by word p-value") self.spin_p = gui.doubleSpin(hbox, self, 'filter_p_value', 1e-4, 1, step=1e-4, labelWidth=15, callback=self.filter_and_display, callbackOnReturn=True, tooltip="Max p-value for word") self.spin_p.setEnabled(self.filter_by_p) hbox = gui.widgetBox(fbox, orientation=0) self.chb_fdr = gui.checkBox(hbox, self, "filter_by_fdr", "FDR", callback=self.filter_and_display, tooltip="Filter by word FDR") self.spin_fdr = gui.doubleSpin(hbox, self, 'filter_fdr_value', 1e-4, 1, step=1e-4, labelWidth=15, callback=self.filter_and_display, callbackOnReturn=True, tooltip="Max p-value for word") self.spin_fdr.setEnabled(self.filter_by_fdr) gui.rubber(self.controlArea) # Word's list view self.cols = ['Word', 'p-value', 'FDR'] self.sig_words = QTreeWidget() self.sig_words.setColumnCount(len(self.cols)) self.sig_words.setHeaderLabels(self.cols) self.sig_words.setSortingEnabled(True) self.sig_words.setSelectionMode(QTreeView.ExtendedSelection) self.sig_words.sortByColumn(2, 0) # 0 is ascending order for i in range(len(self.cols)): self.sig_words.resizeColumnToContents(i) self.mainArea.layout().addWidget(self.sig_words) def sizeHint(self): return QSize(450, 240) @Inputs.data def set_data(self, data=None): self.data = data @Inputs.selected_data def set_data_selected(self, data=None): self.selected_data = data def handleNewSignals(self): self.check_data() def get_bow_domain(self): domain = self.data.domain return Domain( attributes=[a for a in domain.attributes if a.attributes.get('bow-feature', False)], class_vars=domain.class_vars, metas=domain.metas, source=domain) def check_data(self): self.Error.clear() if isinstance(self.data, Table) and \ isinstance(self.selected_data, Table): if len(self.selected_data) == 0: self.Error.empty_selection() self.clear() return # keep only BoW features bow_domain = self.get_bow_domain() if len(bow_domain.attributes) == 0: self.Error.no_bow_features() self.clear() return self.data = Corpus.from_table(bow_domain, self.data) self.selected_data_transformed = Corpus.from_table(bow_domain, self.selected_data) if np_sp_sum(self.selected_data_transformed.X) == 0: self.Error.no_words_overlap() self.clear() elif len(self.data) == len(self.selected_data): self.Error.all_selected() self.clear() else: self.apply() else: self.clear() def clear(self): self.sig_words.clear() self.info_all.setText('Cluster words:') self.info_sel.setText('Selected words:') self.info_fil.setText('After filtering:') def filter_enabled(self, b): self.chb_p.setEnabled(b) self.chb_fdr.setEnabled(b) self.spin_p.setEnabled(b) self.spin_fdr.setEnabled(b) def filter_and_display(self): self.spin_p.setEnabled(self.filter_by_p) self.spin_fdr.setEnabled(self.filter_by_fdr) self.sig_words.clear() if self.selected_data_transformed is None: # do nothing when no Data return count = 0 if self.words: for word, pval, fval in zip(self.words, self.p_values, self.fdr_values): if (not self.filter_by_p or pval <= self.filter_p_value) and \ (not self.filter_by_fdr or fval <= self.filter_fdr_value): it = EATreeWidgetItem(word, pval, fval, self.sig_words) self.sig_words.addTopLevelItem(it) count += 1 for i in range(len(self.cols)): self.sig_words.resizeColumnToContents(i) self.info_all.setText('Cluster words: {}'.format(len(self.selected_data_transformed.domain.attributes))) self.info_sel.setText('Selected words: {}'.format(np.count_nonzero(np_sp_sum(self.selected_data_transformed.X, axis=0)))) if not self.filter_by_p and not self.filter_by_fdr: self.info_fil.setText('After filtering:') self.info_fil.setEnabled(False) else: self.info_fil.setEnabled(True) self.info_fil.setText('After filtering: {}'.format(count)) def progress(self, p): self.progressBarSet(p) def apply(self): self.clear() self.progressBarInit() self.filter_enabled(False) self.words = [i.name for i in self.selected_data_transformed.domain.attributes] self.p_values = hypergeom_p_values(self.data.X, self.selected_data_transformed.X, callback=self.progress) self.fdr_values = false_discovery_rate(self.p_values) self.filter_and_display() self.filter_enabled(True) self.progressBarFinished() def tree_to_table(self): view = [self.cols] items = self.sig_words.topLevelItemCount() for i in range(items): line = [] for j in range(3): line.append(self.sig_words.topLevelItem(i).text(j)) view.append(line) return(view) def send_report(self): if self.words: self.report_table("Enriched words", self.tree_to_table())
class OWGenExpress(widget.OWWidget): name = "GenExpress" description = "Expression data from GenExpress." icon = "../widgets/icons/GenCloud.svg" priority = 36 inputs = [] outputs = [("Data", Orange.data.Table)] username = settings.Setting("anonymous") password = settings.Setting("") log2 = settings.Setting(False) transpose = settings.Setting(False) rtypei = settings.Setting(0) projecti = settings.Setting(0) serveri = settings.Setting(0) exnamei = settings.Setting(6) excludeconstant = settings.Setting(False) joinreplicates = settings.Setting(False) currentSelection = settings.Setting(None) experimentsHeaderState = settings.Setting( {name: False for _, name in HEADER[:ID_INDEX + 1]}) storedSortOrder = settings.Setting([]) storedSelections = settings.Setting([]) def __init__(self, parent=None): super().__init__(parent) self.servers = [ ('https://dictyexpress.research.bcm.edu/', 'dictyExpress'), ('https://cloud.genialis.com/', 'Genialis'), ] self.selectedExperiments = [] self.buffer = dicty.CacheSQLite(bufferfile) self.searchString = "" self.items = [] self.result_types = [] self.controlArea.setMaximumWidth(250) self.controlArea.setMinimumWidth(250) box = gui.widgetBox(self.controlArea, 'Project') self.projectCB = gui.comboBox(box, self, "projecti", items=[], callback=self.ProjectChosen) self.projects = [] b = gui.widgetBox(self.controlArea, "Selection bookmarks") self.selectionSetsWidget = SelectionSetsWidget(self) self.selectionSetsWidget.setSizePolicy(QSizePolicy.Preferred, QSizePolicy.Maximum) def store_selections(modified): if not modified: self.storedSelections = self.selectionSetsWidget.selections self.selectionSetsWidget.selectionModified.connect(store_selections) b.layout().addWidget(self.selectionSetsWidget) gui.separator(self.controlArea) b = gui.widgetBox(self.controlArea, "Sort output columns") self.columnsSortingWidget = SortedListWidget(self) self.columnsSortingWidget.setSizePolicy(QSizePolicy.Preferred, QSizePolicy.Maximum) box = gui.widgetBox(self.controlArea, 'Experiment name') self.experimentNameCB = gui.comboBox(box, self, "exnamei", items=SORTING_MODEL_LIST) b.layout().addWidget(self.columnsSortingWidget) sorting_model = QStringListModel(SORTING_MODEL_LIST) self.columnsSortingWidget.setModel(sorting_model) self.columnsSortingWidget.sortingOrder = self.storedSortOrder def store_sort_order(): self.storedSortOrder = self.columnsSortingWidget.sortingOrder self.columnsSortingWidget.sortingOrderChanged.connect(store_sort_order) gui.separator(self.controlArea) box = gui.widgetBox(self.controlArea, 'Expression Type') self.expressionTypesCB = gui.comboBox(box, self, "rtypei", items=[], callback=self.UpdateResultsList) gui.checkBox(self.controlArea, self, "excludeconstant", "Exclude labels with constant values") gui.checkBox(self.controlArea, self, "joinreplicates", "Average replicates (use median)") gui.checkBox(self.controlArea, self, "log2", "Logarithmic (base 2) transformation") gui.checkBox(self.controlArea, self, "transpose", "Genes as columns") self.commit_button = gui.button(self.controlArea, self, "&Commit", callback=self.Commit) self.commit_button.setDisabled(True) gui.rubber(self.controlArea) box = gui.widgetBox(self.controlArea, 'Server') gui.comboBox(box, self, "serveri", items=[title for url, title in self.servers], callback=self.ServerChosen) gui.lineEdit(box, self, "username", "Username:"******"password", "Password:"******"Clear cache", callback=self.clear_cache) gui.lineEdit(self.mainArea, self, "searchString", "Search", callbackOnType=True, callback=self.SearchUpdate) self.headerLabels = [t[1] for t in HEADER] self.experimentsWidget = QTreeWidget() self.experimentsWidget.setHeaderLabels(self.headerLabels) self.experimentsWidget.setSelectionMode(QTreeWidget.ExtendedSelection) self.experimentsWidget.setRootIsDecorated(False) self.experimentsWidget.setSortingEnabled(True) contextEventFilter = gui.VisibleHeaderSectionContextEventFilter( self.experimentsWidget, self.experimentsWidget) self.experimentsWidget.header().installEventFilter(contextEventFilter) self.experimentsWidget.setItemDelegateForColumn( 0, gui.IndicatorItemDelegate(self, role=Qt.DisplayRole)) self.experimentsWidget.setAlternatingRowColors(True) self.experimentsWidget.selectionModel().selectionChanged.connect( self.onSelectionChanged) self.selectionSetsWidget.setSelectionModel( self.experimentsWidget.selectionModel()) self.selectionSetsWidget.setSelections(self.storedSelections) self.mainArea.layout().addWidget(self.experimentsWidget) self.restoreHeaderState() self.experimentsWidget.header().geometriesChanged.connect( self.saveHeaderState) self.dbc = None self.AuthSet() QTimer.singleShot(100, self.ConnectAndUpdate) def sizeHint(self): return QSize(800, 600) def AuthSet(self): if len(self.username): self.passf.setDisabled(False) else: self.passf.setDisabled(True) def AuthChanged(self): self.AuthSet() self.ConnectAndUpdate() def ConnectAndUpdate(self): self.Connect() if self.dbc: def get_data_count(project_id): # XXX: is there a better way? # Note: limit 0 would return all objects return self.dbc.gen.api.data.get( case_ids__contains=project_id, type__startswith='data:expression:', limit=1)['meta']['total_count'] self.projects = sorted([ p for p in self.dbc.projects().items() if get_data_count(p[0]) > 0 ], key=lambda x: x[1]) self.UpdateProjects() self.ProjectChosen() self.UpdateExperimentTypes() def Connect(self): self.error(1) self.warning(1) username = '******' password = '******' url = self.servers[self.serveri][0] if self.username: username = self.username password = self.password if username.lower() in ['*****@*****.**', 'anonymous']: username = '******' password = '******' self.dbc = None self.projects = [] self.result_types = [] try: self.dbc = Genesis(address=url, username=username, password=password, cache=self.buffer) except requests.exceptions.ConnectionError: self.dbc = Genesis(address=url, username=username, password=password, connect=False, cache=self.buffer) self.warning(1, "Could not connect to server, working from cache.") except Exception: self.error(1, "Wrong username or password.") self.UpdateProjects() self.UpdateExperimentTypes() # clear lists def Reload(self): self.UpdateExperiments(reload=True) def clear_cache(self): self.buffer.clear() self.Reload() def rtype(self): """Return selected result template type """ if self.result_types: return self.result_types[self.rtypei] else: return None def UpdateExperimentTypes(self): self.expressionTypesCB.clear() items = [self.result_types_labels[desc] for desc in self.result_types] self.expressionTypesCB.addItems(items) #do not update anything if the list is empty if len(self.result_types): self.rtypei = max(0, min(self.rtypei, len(self.result_types) - 1)) def UpdateProjects(self): self.projectCB.clear() items = [desc for pid, desc in self.projects] self.projectCB.addItems(items) #do not update anything if the list if empty if len(self.projects) > 0: self.projecti = max(0, min(self.projecti, len(self.projects) - 1)) def UpdateExperiments(self, reload=False): self.experimentsWidget.clear() if not self.dbc or not self.dbc.projectid: # the connection did not succeed return self.items = [] self.progressBarInit() result_types = [] result_types_labels = [] sucind = False # success indicator for database index try: result_types, result_types_labels = self.dbc.result_types( reload=reload) sucind = True except Exception: try: result_types, result_types_labels = self.dbc.result_types() self.warning(0, "Can not access database - using cached data.") sucind = True except Exception: self.error(0, "Can not access database.") if sucind: self.warning(0) self.error(0) self.result_types = result_types self.result_types_labels = result_types_labels self.UpdateExperimentTypes() self.UpdateResultsList(reload=reload) self.progressBarFinished() if self.currentSelection: self.currentSelection.select( self.experimentsWidget.selectionModel()) self.handle_commit_button() def ProjectChosen(self, reload=False): if self.projects: self.dbc.projectid = self.projects[self.projecti][0] else: self.dbc.projectid = None self.UpdateExperiments(reload=reload) def ServerChosen(self): self.ConnectAndUpdate() def UpdateResultsList(self, reload=False): results_list = self.dbc.results_list(self.rtype(), reload=reload) try: results_list = self.dbc.results_list(self.rtype(), reload=reload) except Exception: try: results_list = self.dbc.results_list(self.rtype()) except Exception: self.error(0, "Can not access database.") self.results_list = results_list #softly change the view so that the selection stays the same items_shown = {} for i, item in enumerate(self.items): c = str(item.text(ID_INDEX)) items_shown[c] = i items_to_show = set(id_ for id_ in self.results_list) add_items = set(items_to_show) - set(items_shown) delete_items = set(items_shown) - set(items_to_show) i = 0 while i < self.experimentsWidget.topLevelItemCount(): it = self.experimentsWidget.topLevelItem(i) if str(it.text(ID_INDEX)) in delete_items: self.experimentsWidget.takeTopLevelItem(i) else: i += 1 delete_ind = set([items_shown[i] for i in delete_items]) self.items = [ it for i, it in enumerate(self.items) if i not in delete_ind ] for r_annot in add_items: d = defaultdict(lambda: "?", self.results_list[r_annot]) row_items = [""] + [ to_text(d.get(key, "?")) for key, _ in HEADER[1:] ] row_items[ID_INDEX] = r_annot ci = MyTreeWidgetItem(self.experimentsWidget, row_items) self.items.append(ci) for i in range(len(self.headerLabels)): self.experimentsWidget.resizeColumnToContents(i) self.wantbufver = lambda x: self.results_list[x]["date_modified"] self.UpdateCached() def UpdateCached(self): if self.wantbufver and self.dbc: for item in self.items: id = str(item.text(ID_INDEX)) version = self.dbc._in_buffer(id + "|||" + self.rtype()) value = " " if version == self.wantbufver(id) else "" item.setData(0, Qt.DisplayRole, value) def SearchUpdate(self, string=""): for item in self.items: item.setHidden(not all(s in item for s in self.searchString.split())) def Commit(self): pb = gui.ProgressBar(self, iterations=100) table = None ids = [] for item in self.experimentsWidget.selectedItems(): unique_id = str(item.text(ID_INDEX)) ids.append(unique_id) transfn = None if self.log2: transfn = lambda x: math.log(x + 1.0, 2) reverse_header_dict = {name: name for key, name in HEADER} reverse_header_dict["ID"] = "id" allowed_labels = None def namefn(a): name = SORTING_MODEL_LIST[self.exnamei] name = reverse_header_dict.get(name, "id") return dict(a)[name] if len(ids): table = self.dbc.get_data( ids=ids, result_type=self.rtype(), callback=pb.advance, exclude_constant_labels=self.excludeconstant, bufver=self.wantbufver, transform=transfn, allowed_labels=allowed_labels, namefn=namefn) if self.joinreplicates: table = dicty.join_replicates( table, ignorenames=self.dbc.IGNORE_REPLICATE, namefn="name", avg=dicty.median, fnshow=lambda x: " | ".join(map(str, x))) # Sort attributes sortOrder = self.columnsSortingWidget.sortingOrder all_values = defaultdict(set) for at in table.domain.attributes: atts = at.attributes for name in sortOrder: all_values[name].add( atts.get(reverse_header_dict[name], "")) isnum = {} for at, vals in all_values.items(): vals = filter(None, vals) try: for a in vals: float(a) isnum[at] = True except ValueError: isnum[at] = False def optfloat(x, at): if x == "": return "" else: return float(x) if isnum[at] else x def sorting_key(attr): atts = attr.attributes return tuple([ optfloat(atts.get(reverse_header_dict[name], ""), name) for name in sortOrder ]) attributes = sorted(table.domain.attributes, key=sorting_key) domain = Orange.data.Domain(attributes, table.domain.class_vars, table.domain.metas) table = Orange.data.Table.from_table(domain, table) table = Orange.data.Table(domain, table) if self.transpose: experiments = [at for at in table.domain.variables] attr = [ compat.ContinuousVariable.make(ex['DDB'].value) for ex in table ] metavars = sorted(table.domain.variables[0].attributes.keys()) metavars = [ compat.StringVariable.make(name) for name in metavars ] domain = compat.create_domain(attr, None, metavars) metavars = compat.get_metas(domain) metas = [[exp.attributes[var.name] for var in metavars] for exp in experiments] table = compat.create_table(domain, table.X.transpose(), None, metas) data_hints.set_hint(table, "taxid", "352472") data_hints.set_hint(table, "genesinrows", False) self.send("Data", table) self.UpdateCached() pb.finish() def onSelectionChanged(self, selected, deselected): self.handle_commit_button() def handle_commit_button(self): self.currentSelection = \ SelectionByKey(self.experimentsWidget.selectionModel().selection(), key=(ID_INDEX,)) self.commit_button.setDisabled(not len(self.currentSelection)) def saveHeaderState(self): hview = self.experimentsWidget.header() for i, label in enumerate(self.headerLabels): self.experimentsHeaderState[label] = hview.isSectionHidden(i) def restoreHeaderState(self): hview = self.experimentsWidget.header() state = self.experimentsHeaderState for i, label in enumerate(self.headerLabels): hview.setSectionHidden(i, state.get(label, True)) self.experimentsWidget.resizeColumnToContents(i)
class OWDatabasesUpdate(OWWidget): name = "Databases Update" description = "Update local systems biology databases." icon = "../widgets/icons/OWDatabasesUpdate.svg" priority = 1 inputs = [] outputs = [] want_main_area = False def __init__(self, parent=None, signalManager=None, name="Databases update"): OWWidget.__init__(self, parent, signalManager, name, wantMainArea=False) self.searchString = "" fbox = gui.widgetBox(self.controlArea, "Filter") self.completer = TokenListCompleter( self, caseSensitivity=Qt.CaseInsensitive) self.lineEditFilter = QLineEdit(textChanged=self.SearchUpdate) self.lineEditFilter.setCompleter(self.completer) fbox.layout().addWidget(self.lineEditFilter) box = gui.widgetBox(self.controlArea, "Files") self.filesView = QTreeWidget(self) self.filesView.setHeaderLabels( ["", "Data Source", "Update", "Last Updated", "Size"]) self.filesView.setRootIsDecorated(False) self.filesView.setUniformRowHeights(True) self.filesView.setSelectionMode(QAbstractItemView.NoSelection) self.filesView.setSortingEnabled(True) self.filesView.sortItems(1, Qt.AscendingOrder) self.filesView.setItemDelegateForColumn( 0, UpdateOptionsItemDelegate(self.filesView)) self.filesView.model().layoutChanged.connect(self.SearchUpdate) box.layout().addWidget(self.filesView) box = gui.widgetBox(self.controlArea, orientation="horizontal") self.updateButton = gui.button( box, self, "Update all", callback=self.UpdateAll, tooltip="Update all updatable files", ) self.downloadButton = gui.button( box, self, "Download all", callback=self.DownloadFiltered, tooltip="Download all filtered files shown" ) self.cancelButton = gui.button( box, self, "Cancel", callback=self.Cancel, tooltip="Cancel scheduled downloads/updates." ) self.retryButton = gui.button( box, self, "Reconnect", callback=self.RetrieveFilesList ) self.retryButton.hide() gui.rubber(box) self.warning(0) box = gui.widgetBox(self.controlArea, orientation="horizontal") gui.rubber(box) self.infoLabel = QLabel() self.infoLabel.setAlignment(Qt.AlignCenter) self.controlArea.layout().addWidget(self.infoLabel) self.infoLabel.setSizePolicy(QSizePolicy.Expanding, QSizePolicy.Fixed) self.updateItems = [] self.resize(800, 600) self.progress = ProgressState(self, maximum=3) self.progress.valueChanged.connect(self._updateProgress) self.progress.rangeChanged.connect(self._updateProgress) self.executor = ThreadExecutor( threadPool=QThreadPool(maxThreadCount=2) ) task = Task(self, function=self.RetrieveFilesList) task.exceptionReady.connect(self.HandleError) task.start() self._tasks = [] self._haveProgress = False def RetrieveFilesList(self): self.retryButton.hide() self.warning(0) self.progress.setRange(0, 3) task = Task(function=partial(retrieveFilesList, methodinvoke(self.progress, "advance"))) task.resultReady.connect(self.SetFilesList) task.exceptionReady.connect(self.HandleError) self.executor.submit(task) self.setEnabled(False) def SetFilesList(self, serverInfo): """ Set the files to show. """ self.setEnabled(True) localInfo = serverfiles.allinfo() all_tags = set() self.filesView.clear() self.updateItems = [] for item in join_info_dict(localInfo, serverInfo): tree_item = UpdateTreeWidgetItem(item) options_widget = UpdateOptionsWidget(item.state) options_widget.item = item options_widget.installClicked.connect( partial(self.SubmitDownloadTask, item.domain, item.filename) ) options_widget.removeClicked.connect( partial(self.SubmitRemoveTask, item.domain, item.filename) ) self.updateItems.append((item, tree_item, options_widget)) all_tags.update(item.tags) self.filesView.addTopLevelItems( [tree_item for _, tree_item, _ in self.updateItems] ) for item, tree_item, options_widget in self.updateItems: self.filesView.setItemWidget(tree_item, 0, options_widget) # Add an update button if the file is updateable if item.state == OUTDATED: button = QToolButton( None, text="Update", maximumWidth=120, minimumHeight=20, maximumHeight=20 ) if sys.platform == "darwin": button.setAttribute(Qt.WA_MacSmallSize) button.clicked.connect( partial(self.SubmitDownloadTask, item.domain, item.filename) ) self.filesView.setItemWidget(tree_item, 2, button) self.progress.advance() self.filesView.setColumnWidth(0, self.filesView.sizeHintForColumn(0)) for column in range(1, 4): contents_hint = self.filesView.sizeHintForColumn(column) header_hint = self.filesView.header().sectionSizeHint(column) width = max(min(contents_hint, 400), header_hint) self.filesView.setColumnWidth(column, width) hints = [hint for hint in sorted(all_tags) if not hint.startswith("#")] self.completer.setTokenList(hints) self.SearchUpdate() self.UpdateInfoLabel() self.toggleButtons() self.cancelButton.setEnabled(False) self.progress.setRange(0, 0) def buttonCheck(self, selected_items, state, button): for item in selected_items: if item.state != state: button.setEnabled(False) else: button.setEnabled(True) break def toggleButtons(self): selected_items = [item for item, tree_item, _ in self.updateItems if not tree_item.isHidden()] self.buttonCheck(selected_items, OUTDATED, self.updateButton) self.buttonCheck(selected_items, AVAILABLE, self.downloadButton) def HandleError(self, exception): if isinstance(exception, ConnectionError): self.warning(0, "Could not connect to server! Check your connection " "and try to reconnect.") self.SetFilesList({}) self.retryButton.show() else: sys.excepthook(type(exception), exception, None) self.progress.setRange(0, 0) self.setEnabled(True) def UpdateInfoLabel(self): local = [item for item, tree_item, _ in self.updateItems if item.state != AVAILABLE and not tree_item.isHidden()] size = sum(float(item.size) for item in local) onServer = [item for item, tree_item, _ in self.updateItems if not tree_item.isHidden()] sizeOnServer = sum(float(item.size) for item in onServer) text = ("%i items, %s (on server: %i items, %s)" % (len(local), serverfiles.sizeformat(size), len(onServer), serverfiles.sizeformat(sizeOnServer))) self.infoLabel.setText(text) def UpdateAll(self): self.warning(0) for item, tree_item, _ in self.updateItems: if item.state == OUTDATED and not tree_item.isHidden(): self.SubmitDownloadTask(item.domain, item.filename) def DownloadFiltered(self): # TODO: submit items in the order shown. for item, tree_item, _ in self.updateItems: if not tree_item.isHidden() and item.state in \ [AVAILABLE, OUTDATED]: self.SubmitDownloadTask(item.domain, item.filename) def SearchUpdate(self, searchString=None): strings = str(self.lineEditFilter.text()).split() for item, tree_item, _ in self.updateItems: hide = not all(UpdateItem_match(item, string) for string in strings) tree_item.setHidden(hide) self.UpdateInfoLabel() self.toggleButtons() def SubmitDownloadTask(self, domain, filename): """ Submit the (domain, filename) to be downloaded/updated. """ self.cancelButton.setEnabled(True) index = self.updateItemIndex(domain, filename) _, tree_item, opt_widget = self.updateItems[index] task = DownloadTask(domain, filename, serverfiles.LOCALFILES) self.progress.adjustRange(0, 100) pb = ItemProgressBar(self.filesView) pb.setRange(0, 100) pb.setTextVisible(False) task.advanced.connect(pb.advance) task.advanced.connect(self.progress.advance) task.finished.connect(pb.hide) task.finished.connect(self.onDownloadFinished, Qt.QueuedConnection) task.exception.connect(self.onDownloadError, Qt.QueuedConnection) self.filesView.setItemWidget(tree_item, 2, pb) # Clear the text so it does not show behind the progress bar. tree_item.setData(2, Qt.DisplayRole, "") pb.show() # Disable the options widget opt_widget.setEnabled(False) self._tasks.append(task) self.executor.submit(task) def EndDownloadTask(self, task): future = task.future() index = self.updateItemIndex(task.domain, task.filename) item, tree_item, opt_widget = self.updateItems[index] self.filesView.removeItemWidget(tree_item, 2) opt_widget.setEnabled(True) if future.cancelled(): # Restore the previous state tree_item.setUpdateItem(item) opt_widget.setState(item.state) elif future.exception(): tree_item.setUpdateItem(item) opt_widget.setState(item.state) # Show the exception string in the size column. self.warning(0, "Error while downloading. Check your connection " "and retry.") # recreate button for download button = QToolButton( None, text="Retry", maximumWidth=120, minimumHeight=20, maximumHeight=20 ) if sys.platform == "darwin": button.setAttribute(Qt.WA_MacSmallSize) button.clicked.connect( partial(self.SubmitDownloadTask, item.domain, item.filename) ) self.filesView.setItemWidget(tree_item, 2, button) else: # get the new updated info dict and replace the the old item self.warning(0) info = serverfiles.info(item.domain, item.filename) new_item = update_item_from_info(item.domain, item.filename, info, info) self.updateItems[index] = (new_item, tree_item, opt_widget) tree_item.setUpdateItem(new_item) opt_widget.setState(new_item.state) self.UpdateInfoLabel() def SubmitRemoveTask(self, domain, filename): serverfiles.LOCALFILES.remove(domain, filename) index = self.updateItemIndex(domain, filename) item, tree_item, opt_widget = self.updateItems[index] if item.info_server: new_item = item._replace(state=AVAILABLE, local=None, info_local=None) else: new_item = item._replace(local=None, info_local=None) # Disable the options widget. No more actions can be performed # for the item. opt_widget.setEnabled(False) tree_item.setUpdateItem(new_item) opt_widget.setState(new_item.state) self.updateItems[index] = (new_item, tree_item, opt_widget) self.UpdateInfoLabel() def Cancel(self): """ Cancel all pending update/download tasks (that have not yet started). """ for task in self._tasks: task.future().cancel() def onDeleteWidget(self): self.Cancel() self.executor.shutdown(wait=False) OWWidget.onDeleteWidget(self) def onDownloadFinished(self): # on download completed/canceled/error assert QThread.currentThread() is self.thread() for task in list(self._tasks): future = task.future() if future.done(): self.EndDownloadTask(task) self._tasks.remove(task) if not self._tasks: # Clear/reset the overall progress self.progress.setRange(0, 0) self.cancelButton.setEnabled(False) def onDownloadError(self, exc_info): sys.excepthook(*exc_info) self.warning(0, "Error while downloading. Check your connection and " "retry.") def updateItemIndex(self, domain, filename): for i, (item, _, _) in enumerate(self.updateItems): if item.domain == domain and item.filename == filename: return i raise ValueError("%r, %r not in update list" % (domain, filename)) def _updateProgress(self, *args): rmin, rmax = self.progress.range() if rmin != rmax: if not self._haveProgress: self._haveProgress = True self.progressBarInit() self.progressBarSet(self.progress.ratioCompleted() * 100, processEvents=None) if rmin == rmax: self._haveProgress = False self.progressBarFinished()
class OWGenExpress(widget.OWWidget): name = "GenExpress" description = "Expression data from GenExpress." icon = "../widgets/icons/GenCloud.svg" priority = 36 inputs = [] outputs = [("Data", Orange.data.Table)] username = settings.Setting("anonymous") password = settings.Setting("") log2 = settings.Setting(False) transpose = settings.Setting(False) rtypei = settings.Setting(0) projecti = settings.Setting(0) serveri = settings.Setting(0) exnamei = settings.Setting(6) excludeconstant = settings.Setting(False) joinreplicates = settings.Setting(False) currentSelection = settings.Setting(None) experimentsHeaderState = settings.Setting({ name: False for _, name in HEADER[:ID_INDEX + 1]} ) storedSortOrder = settings.Setting([]) storedSelections = settings.Setting([]) def __init__(self, parent=None): super().__init__(parent) self.servers = [ ('https://dictyexpress.research.bcm.edu/', 'dictyExpress'), ('https://cloud.genialis.com/', 'Genialis'), ] self.selectedExperiments = [] self.buffer = dicty.CacheSQLite(bufferfile) self.searchString = "" self.items = [] self.result_types = [] self.controlArea.setMaximumWidth(250) self.controlArea.setMinimumWidth(250) box = gui.widgetBox(self.controlArea, 'Project') self.projectCB = gui.comboBox( box, self, "projecti", items=[], callback=self.ProjectChosen) self.projects = [] b = gui.widgetBox(self.controlArea, "Selection bookmarks") self.selectionSetsWidget = SelectionSetsWidget(self) self.selectionSetsWidget.setSizePolicy( QSizePolicy.Preferred, QSizePolicy.Maximum) def store_selections(modified): if not modified: self.storedSelections = self.selectionSetsWidget.selections self.selectionSetsWidget.selectionModified.connect(store_selections) b.layout().addWidget(self.selectionSetsWidget) gui.separator(self.controlArea) b = gui.widgetBox(self.controlArea, "Sort output columns") self.columnsSortingWidget = SortedListWidget(self) self.columnsSortingWidget.setSizePolicy( QSizePolicy.Preferred, QSizePolicy.Maximum) box = gui.widgetBox(self.controlArea, 'Experiment name') self.experimentNameCB = gui.comboBox( box, self, "exnamei", items=SORTING_MODEL_LIST) b.layout().addWidget(self.columnsSortingWidget) sorting_model = QStringListModel(SORTING_MODEL_LIST) self.columnsSortingWidget.setModel(sorting_model) self.columnsSortingWidget.sortingOrder = self.storedSortOrder def store_sort_order(): self.storedSortOrder = self.columnsSortingWidget.sortingOrder self.columnsSortingWidget.sortingOrderChanged.connect(store_sort_order) gui.separator(self.controlArea) box = gui.widgetBox(self.controlArea, 'Expression Type') self.expressionTypesCB = gui.comboBox( box, self, "rtypei", items=[], callback=self.UpdateResultsList) gui.checkBox(self.controlArea, self, "excludeconstant", "Exclude labels with constant values") gui.checkBox(self.controlArea, self, "joinreplicates", "Average replicates (use median)") gui.checkBox(self.controlArea, self, "log2", "Logarithmic (base 2) transformation") gui.checkBox(self.controlArea, self, "transpose", "Genes as columns") self.commit_button = gui.button(self.controlArea, self, "&Commit", callback=self.Commit) self.commit_button.setDisabled(True) gui.rubber(self.controlArea) box = gui.widgetBox(self.controlArea, 'Server') gui.comboBox(box, self, "serveri", items=[title for url, title in self.servers], callback=self.ServerChosen) gui.lineEdit(box, self, "username", "Username:"******"password", "Password:"******"Clear cache", callback=self.clear_cache) gui.lineEdit(self.mainArea, self, "searchString", "Search", callbackOnType=True, callback=self.SearchUpdate) self.headerLabels = [t[1] for t in HEADER] self.experimentsWidget = QTreeWidget() self.experimentsWidget.setHeaderLabels(self.headerLabels) self.experimentsWidget.setSelectionMode(QTreeWidget.ExtendedSelection) self.experimentsWidget.setRootIsDecorated(False) self.experimentsWidget.setSortingEnabled(True) contextEventFilter = gui.VisibleHeaderSectionContextEventFilter( self.experimentsWidget, self.experimentsWidget) self.experimentsWidget.header().installEventFilter(contextEventFilter) self.experimentsWidget.setItemDelegateForColumn( 0, gui.IndicatorItemDelegate(self, role=Qt.DisplayRole)) self.experimentsWidget.setAlternatingRowColors(True) self.experimentsWidget.selectionModel().selectionChanged.connect( self.onSelectionChanged) self.selectionSetsWidget.setSelectionModel( self.experimentsWidget.selectionModel()) self.selectionSetsWidget.setSelections(self.storedSelections) self.mainArea.layout().addWidget(self.experimentsWidget) self.restoreHeaderState() self.experimentsWidget.header().geometriesChanged.connect( self.saveHeaderState) self.dbc = None self.AuthSet() QTimer.singleShot(100, self.ConnectAndUpdate) def sizeHint(self): return QSize(800, 600) def AuthSet(self): if len(self.username): self.passf.setDisabled(False) else: self.passf.setDisabled(True) def AuthChanged(self): self.AuthSet() self.ConnectAndUpdate() def ConnectAndUpdate(self): self.Connect() if self.dbc: def get_data_count(project_id): # XXX: is there a better way? # Note: limit 0 would return all objects return self.dbc.gen.api.data.get(case_ids__contains=project_id, type__startswith='data:expression:', limit=1)['meta']['total_count'] self.projects = sorted([p for p in self.dbc.projects().items() if get_data_count(p[0]) > 0], key=lambda x: x[1]) self.UpdateProjects() self.ProjectChosen() self.UpdateExperimentTypes() def Connect(self): self.error(1) self.warning(1) username = '******' password = '******' url = self.servers[self.serveri][0] if self.username: username = self.username password = self.password if username.lower() in ['*****@*****.**', 'anonymous']: username = '******' password = '******' self.dbc = None self.projects = [] self.result_types = [] try: self.dbc = Genesis( address=url, username=username, password=password, cache=self.buffer) except requests.exceptions.ConnectionError: self.dbc = Genesis( address=url, username=username, password=password, connect=False, cache=self.buffer) self.warning(1, "Could not connect to server, working from cache.") except Exception: self.error(1, "Wrong username or password.") self.UpdateProjects() self.UpdateExperimentTypes() # clear lists def Reload(self): self.UpdateExperiments(reload=True) def clear_cache(self): self.buffer.clear() self.Reload() def rtype(self): """Return selected result template type """ if self.result_types: return self.result_types[self.rtypei] else: return None def UpdateExperimentTypes(self): self.expressionTypesCB.clear() items = [self.result_types_labels[desc] for desc in self.result_types] self.expressionTypesCB.addItems(items) #do not update anything if the list is empty if len(self.result_types): self.rtypei = max(0, min(self.rtypei, len(self.result_types) - 1)) def UpdateProjects(self): self.projectCB.clear() items = [desc for pid, desc in self.projects] self.projectCB.addItems(items) #do not update anything if the list if empty if len(self.projects) > 0: self.projecti = max(0, min(self.projecti, len(self.projects) - 1)) def UpdateExperiments(self, reload=False): self.experimentsWidget.clear() if not self.dbc or not self.dbc.projectid: # the connection did not succeed return self.items = [] self.progressBarInit() result_types = [] result_types_labels = [] sucind = False # success indicator for database index try: result_types, result_types_labels = self.dbc.result_types(reload=reload) sucind = True except Exception: try: result_types, result_types_labels = self.dbc.result_types() self.warning(0, "Can not access database - using cached data.") sucind = True except Exception: self.error(0, "Can not access database.") if sucind: self.warning(0) self.error(0) self.result_types = result_types self.result_types_labels = result_types_labels self.UpdateExperimentTypes() self.UpdateResultsList(reload=reload) self.progressBarFinished() if self.currentSelection: self.currentSelection.select(self.experimentsWidget.selectionModel()) self.handle_commit_button() def ProjectChosen(self, reload=False): if self.projects: self.dbc.projectid = self.projects[self.projecti][0] else: self.dbc.projectid = None self.UpdateExperiments(reload=reload) def ServerChosen(self): self.ConnectAndUpdate() def UpdateResultsList(self, reload=False): results_list = self.dbc.results_list(self.rtype(), reload=reload) try: results_list = self.dbc.results_list(self.rtype(), reload=reload) except Exception: try: results_list = self.dbc.results_list(self.rtype()) except Exception: self.error(0, "Can not access database.") self.results_list = results_list #softly change the view so that the selection stays the same items_shown = {} for i, item in enumerate(self.items): c = str(item.text(ID_INDEX)) items_shown[c] = i items_to_show = set(id_ for id_ in self.results_list) add_items = set(items_to_show) - set(items_shown) delete_items = set(items_shown) - set(items_to_show) i = 0 while i < self.experimentsWidget.topLevelItemCount(): it = self.experimentsWidget.topLevelItem(i) if str(it.text(ID_INDEX)) in delete_items: self.experimentsWidget.takeTopLevelItem(i) else: i += 1 delete_ind = set([items_shown[i] for i in delete_items]) self.items = [it for i, it in enumerate(self.items) if i not in delete_ind] for r_annot in add_items: d = defaultdict(lambda: "?", self.results_list[r_annot]) row_items = [""] + [to_text(d.get(key, "?")) for key, _ in HEADER[1:]] row_items[ID_INDEX] = r_annot ci = MyTreeWidgetItem(self.experimentsWidget, row_items) self.items.append(ci) for i in range(len(self.headerLabels)): self.experimentsWidget.resizeColumnToContents(i) self.wantbufver = lambda x: self.results_list[x]["date_modified"] self.UpdateCached() def UpdateCached(self): if self.wantbufver and self.dbc: for item in self.items: id = str(item.text(ID_INDEX)) version = self.dbc._in_buffer(id + "|||" + self.rtype()) value = " " if version == self.wantbufver(id) else "" item.setData(0, Qt.DisplayRole, value) def SearchUpdate(self, string=""): for item in self.items: item.setHidden( not all(s in item for s in self.searchString.split())) def Commit(self): pb = gui.ProgressBar(self, iterations=100) table = None ids = [] for item in self.experimentsWidget.selectedItems(): unique_id = str(item.text(ID_INDEX)) ids.append(unique_id) transfn = None if self.log2: transfn = lambda x: math.log(x + 1.0, 2) reverse_header_dict = {name: name for key, name in HEADER} reverse_header_dict["ID"] = "id" allowed_labels = None def namefn(a): name = SORTING_MODEL_LIST[self.exnamei] name = reverse_header_dict.get(name, "id") return dict(a)[name] if len(ids): table = self.dbc.get_data( ids=ids, result_type=self.rtype(), callback=pb.advance, exclude_constant_labels=self.excludeconstant, bufver=self.wantbufver, transform=transfn, allowed_labels=allowed_labels, namefn=namefn) if self.joinreplicates: table = dicty.join_replicates(table, ignorenames=self.dbc.IGNORE_REPLICATE, namefn="name", avg=dicty.median, fnshow=lambda x: " | ".join(map(str, x))) # Sort attributes sortOrder = self.columnsSortingWidget.sortingOrder all_values = defaultdict(set) for at in table.domain.attributes: atts = at.attributes for name in sortOrder: all_values[name].add(atts.get(reverse_header_dict[name], "")) isnum = {} for at, vals in all_values.items(): vals = filter(None, vals) try: for a in vals: float(a) isnum[at] = True except ValueError: isnum[at] = False def optfloat(x, at): if x == "": return "" else: return float(x) if isnum[at] else x def sorting_key(attr): atts = attr.attributes return tuple([optfloat(atts.get(reverse_header_dict[name], ""), name) for name in sortOrder]) attributes = sorted(table.domain.attributes, key=sorting_key) domain = Orange.data.Domain( attributes, table.domain.class_vars, table.domain.metas) table = Orange.data.Table.from_table(domain, table) table = Orange.data.Table(domain, table) if self.transpose: experiments = [at for at in table.domain.variables] attr = [compat.ContinuousVariable.make(ex['DDB'].value) for ex in table] metavars = sorted(table.domain.variables[0].attributes.keys()) metavars = [compat.StringVariable.make(name) for name in metavars] domain = compat.create_domain(attr, None, metavars) metas = [[exp.attributes[var.name] for var in metavars] for exp in experiments] table = compat.create_table(domain, table.X.transpose(), None, metas) data_hints.set_hint(table, "taxid", "352472") data_hints.set_hint(table, "genesinrows", False) self.send("Data", table) self.UpdateCached() pb.finish() def onSelectionChanged(self, selected, deselected): self.handle_commit_button() def handle_commit_button(self): self.currentSelection = \ SelectionByKey(self.experimentsWidget.selectionModel().selection(), key=(ID_INDEX,)) self.commit_button.setDisabled(not len(self.currentSelection)) def saveHeaderState(self): hview = self.experimentsWidget.header() for i, label in enumerate(self.headerLabels): self.experimentsHeaderState[label] = hview.isSectionHidden(i) def restoreHeaderState(self): hview = self.experimentsWidget.header() state = self.experimentsHeaderState for i, label in enumerate(self.headerLabels): hview.setSectionHidden(i, state.get(label, True)) self.experimentsWidget.resizeColumnToContents(i)
class OWPIPAx(widget.OWWidget): name = "PIPAx" description = "Access data from PIPA RNA-Seq database." icon = "../widgets/icons/PIPA.svg" priority = 35 inputs = [] outputs = [("Data", Orange.data.Table)] username = settings.Setting("") password = settings.Setting("") log2 = settings.Setting(False) rtypei = settings.Setting(5) # hardcoded rpkm mapability polya excludeconstant = settings.Setting(False) joinreplicates = settings.Setting(False) #: The stored current selection (in experiments view) #: SelectionByKey | None currentSelection = settings.Setting(None) #: Stored selections (presets) #: list of SelectionByKey storedSelections = settings.Setting([]) #: Stored column sort keys (from Sort view) #: list of strings storedSortingOrder = settings.Setting( ["Strain", "Experiment", "Genotype", "Timepoint"]) experimentsHeaderState = settings.Setting( {name: False for _, name in HEADER[:ID_INDEX + 1]}) def __init__(self, parent=None, signalManager=None, name="PIPAx"): super().__init__(parent) self.selectedExperiments = [] self.buffer = dicty.CacheSQLite(bufferfile) self.searchString = "" self.result_types = [] self.mappings = {} self.controlArea.setMaximumWidth(250) self.controlArea.setMinimumWidth(250) gui.button(self.controlArea, self, "Reload", callback=self.Reload) gui.button(self.controlArea, self, "Clear cache", callback=self.clear_cache) b = gui.widgetBox(self.controlArea, "Experiment Sets") self.selectionSetsWidget = SelectionSetsWidget(self) self.selectionSetsWidget.setSizePolicy(QSizePolicy.Preferred, QSizePolicy.Maximum) def store_selections(modified): if not modified: self.storedSelections = self.selectionSetsWidget.selections self.selectionSetsWidget.selectionModified.connect(store_selections) b.layout().addWidget(self.selectionSetsWidget) gui.separator(self.controlArea) b = gui.widgetBox(self.controlArea, "Sort output columns") self.columnsSortingWidget = SortedListWidget(self) self.columnsSortingWidget.setSizePolicy(QSizePolicy.Preferred, QSizePolicy.Maximum) def store_sort_order(): self.storedSortingOrder = self.columnsSortingWidget.sortingOrder self.columnsSortingWidget.sortingOrderChanged.connect(store_sort_order) b.layout().addWidget(self.columnsSortingWidget) sorting_model = QStringListModel(SORTING_MODEL_LIST) self.columnsSortingWidget.setModel(sorting_model) gui.separator(self.controlArea) box = gui.widgetBox(self.controlArea, 'Expression Type') self.expressionTypesCB = gui.comboBox(box, self, "rtypei", items=[], callback=self.UpdateResultsList) gui.checkBox(self.controlArea, self, "excludeconstant", "Exclude labels with constant values") gui.checkBox(self.controlArea, self, "joinreplicates", "Average replicates (use median)") gui.checkBox(self.controlArea, self, "log2", "Logarithmic (base 2) transformation") self.commit_button = gui.button(self.controlArea, self, "&Commit", callback=self.Commit) self.commit_button.setDisabled(True) gui.rubber(self.controlArea) box = gui.widgetBox(self.controlArea, "Authentication") gui.lineEdit(box, self, "username", "Username:"******"password", "Password:"******"searchString", "Search", callbackOnType=True, callback=self.SearchUpdate) self.headerLabels = [t[1] for t in HEADER] self.experimentsWidget = QTreeWidget() self.experimentsWidget.setHeaderLabels(self.headerLabels) self.experimentsWidget.setSelectionMode(QTreeWidget.ExtendedSelection) self.experimentsWidget.setRootIsDecorated(False) self.experimentsWidget.setSortingEnabled(True) contextEventFilter = gui.VisibleHeaderSectionContextEventFilter( self.experimentsWidget, self.experimentsWidget) self.experimentsWidget.header().installEventFilter(contextEventFilter) self.experimentsWidget.setItemDelegateForColumn( 0, gui.IndicatorItemDelegate(self, role=Qt.DisplayRole)) self.experimentsWidget.setAlternatingRowColors(True) self.experimentsWidget.selectionModel().selectionChanged.connect( self.onSelectionChanged) self.selectionSetsWidget.setSelectionModel( self.experimentsWidget.selectionModel()) self.mainArea.layout().addWidget(self.experimentsWidget) # Restore the selection states from the stored settings self.selectionSetsWidget.selections = self.storedSelections self.columnsSortingWidget.sortingOrder = self.storedSortingOrder self.restoreHeaderState() self.experimentsWidget.header().geometriesChanged.connect( self.saveHeaderState) self.dbc = None self.AuthSet() QTimer.singleShot(100, self.UpdateExperiments) def sizeHint(self): return QSize(800, 600) def AuthSet(self): if len(self.username): self.passf.setDisabled(False) else: self.passf.setDisabled(True) def AuthChanged(self): self.AuthSet() self.ConnectAndUpdate() def ConnectAndUpdate(self): self.Connect() self.UpdateExperiments(reload=True) def Connect(self): self.error(1) self.warning(1) def en(x): return x if len(x) else None self.dbc = dicty.PIPAx(cache=self.buffer, username=en(self.username), password=self.password) # check password if en(self.username) != None: try: self.dbc.mappings(reload=True) except dicty.AuthenticationError: self.error(1, "Wrong username or password") self.dbc = None except Exception as ex: print("Error when contacting the PIPA database", ex) sys.excepthook(*sys.exc_info()) try: # maybe cached? self.dbc.mappings() self.warning( 1, "Can not access database - using cached data.") except Exception as ex: self.dbc = None self.error(1, "Can not access database.") def Reload(self): self.UpdateExperiments(reload=True) def clear_cache(self): self.buffer.clear() self.Reload() def rtype(self): """Return selected result template type """ if self.result_types: return self.result_types[self.rtypei][0] else: return "-1" def UpdateExperimentTypes(self): self.expressionTypesCB.clear() items = [desc for _, desc in self.result_types] self.expressionTypesCB.addItems(items) self.rtypei = max(0, min(self.rtypei, len(self.result_types) - 1)) def UpdateExperiments(self, reload=False): self.experimentsWidget.clear() self.items = [] self.progressBarInit() if not self.dbc: self.Connect() mappings = {} result_types = [] sucind = False # success indicator for database index try: mappings = self.dbc.mappings(reload=reload) result_types = self.dbc.result_types(reload=reload) sucind = True except Exception as ex: try: mappings = self.dbc.mappings() result_types = self.dbc.result_types() self.warning(0, "Can not access database - using cached data.") sucind = True except Exception as ex: self.error(0, "Can not access database.") if sucind: self.warning(0) self.error(0) self.mappings = mappings self.result_types = result_types self.UpdateExperimentTypes() self.UpdateResultsList(reload=reload) self.progressBarFinished() if self.currentSelection: self.currentSelection.select( self.experimentsWidget.selectionModel()) self.handle_commit_button() def UpdateResultsList(self, reload=False): results_list = {} try: results_list = self.dbc.results_list(self.rtype(), reload=reload) except Exception as ex: try: results_list = self.dbc.results_list(self.rtype()) except Exception as ex: self.error(0, "Can not access database.") self.results_list = results_list mappings_key_dict = dict(((m["data_id"], m["id"]), key) \ for key, m in self.mappings.items()) def mapping_unique_id(annot): """Map annotations dict from results_list to unique `mappings` ids. """ data_id, mappings_id = annot["data_id"], annot["mappings_id"] return mappings_key_dict[data_id, mappings_id] elements = [] # softly change the view so that the selection stays the same items_shown = {} for i, item in enumerate(self.items): c = str(item.text(10)) items_shown[c] = i items_to_show = dict((mapping_unique_id(annot), annot) for annot in self.results_list.values()) add_items = set(items_to_show) - set(items_shown) delete_items = set(items_shown) - set(items_to_show) i = 0 while i < self.experimentsWidget.topLevelItemCount(): it = self.experimentsWidget.topLevelItem(i) if str(it.text(10)) in delete_items: self.experimentsWidget.takeTopLevelItem(i) else: i += 1 delete_ind = set([items_shown[i] for i in delete_items]) self.items = [ it for i, it in enumerate(self.items) if i not in delete_ind ] for r_annot in [items_to_show[i] for i in add_items]: d = defaultdict(lambda: "?", r_annot) row_items = [""] + [d.get(key, "?") for key, _ in HEADER[1:]] try: time_dict = literal_eval(row_items[DATE_INDEX]) date_rna = date( time_dict["fullYearUTC"], time_dict["monthUTC"] + 1, # Why is month 0 based? time_dict["dateUTC"]) row_items[DATE_INDEX] = date_rna.strftime("%x") except Exception: row_items[DATE_INDEX] = '' row_items[ID_INDEX] = mapping_unique_id(r_annot) elements.append(row_items) ci = MyTreeWidgetItem(self.experimentsWidget, row_items) self.items.append(ci) for i in range(len(self.headerLabels)): self.experimentsWidget.resizeColumnToContents(i) # which is the ok buffer version # FIXME: what attribute to use for version? self.wantbufver = \ lambda x, ad=self.results_list: \ defaultdict(lambda: "?", ad[x])["date"] self.wantbufver = lambda x: "0" self.UpdateCached() def UpdateCached(self): if self.wantbufver and self.dbc: fn = self.dbc.download_key_function() result_id_key = dict(((m["data_id"], m["mappings_id"]), key) \ for key, m in self.results_list.items()) for item in self.items: c = str(item.text(10)) mapping = self.mappings[c] data_id, mappings_id = mapping["data_id"], mapping["id"] r_id = result_id_key[data_id, mappings_id] # Get the buffered version buffered = self.dbc.inBuffer(fn(r_id)) value = " " if buffered == self.wantbufver(r_id) else "" item.setData(0, Qt.DisplayRole, value) def SearchUpdate(self, string=""): for item in self.items: item.setHidden(not all(s in item \ for s in self.searchString.split()) ) def Commit(self): if not self.dbc: self.Connect() pb = gui.ProgressBar(self, iterations=100) table = None ids = [] for item in self.experimentsWidget.selectedItems(): unique_id = str(item.text(10)) annots = self.mappings[unique_id] ids.append((annots["data_id"], annots["id"])) transfn = None if self.log2: transfn = lambda x: math.log(x + 1.0, 2) reverse_header_dict = dict((name, key) for key, name in HEADER) hview = self.experimentsWidget.header() shownHeaders = [label for i, label in \ list(enumerate(self.headerLabels))[1:] \ if not hview.isSectionHidden(i) ] allowed_labels = [reverse_header_dict.get(label, label) \ for label in shownHeaders] if self.joinreplicates and "id" not in allowed_labels: # need 'id' labels in join_replicates for attribute names allowed_labels.append("id") if len(ids): table = self.dbc.get_data( ids=ids, result_type=self.rtype(), callback=pb.advance, exclude_constant_labels=self.excludeconstant, # bufver=self.wantbufver, transform=transfn, allowed_labels=allowed_labels) if self.joinreplicates: table = dicty.join_replicates(table, ignorenames=[ "replicate", "data_id", "mappings_id", "data_name", "id", "unique_id" ], namefn=None, avg=dicty.median) # Sort attributes sortOrder = self.columnsSortingWidget.sortingOrder all_values = defaultdict(set) for at in table.domain.attributes: atts = at.attributes for name in sortOrder: all_values[name].add( atts.get(reverse_header_dict[name], "")) isnum = {} for at, vals in all_values.items(): vals = filter(None, vals) try: for a in vals: float(a) isnum[at] = True except: isnum[at] = False def optfloat(x, at): if x == "": return "" else: return float(x) if isnum[at] else x def sorting_key(attr): atts = attr.attributes return tuple([optfloat(atts.get(reverse_header_dict[name], ""), name) \ for name in sortOrder]) attributes = sorted(table.domain.attributes, key=sorting_key) domain = Orange.data.Domain(attributes, table.domain.class_var, table.domain.metas) table = table.from_table(domain, table) data_hints.set_hint(table, "taxid", "352472") data_hints.set_hint(table, "genesinrows", False) self.send("Data", table) self.UpdateCached() pb.finish() def onSelectionChanged(self, selected, deselected): self.handle_commit_button() def handle_commit_button(self): self.currentSelection = \ SelectionByKey(self.experimentsWidget.selectionModel().selection(), key=(1, 2, 3, 10)) self.commit_button.setDisabled(not len(self.currentSelection)) def saveHeaderState(self): hview = self.experimentsWidget.header() for i, label in enumerate(self.headerLabels): self.experimentsHeaderState[label] = hview.isSectionHidden(i) def restoreHeaderState(self): hview = self.experimentsWidget.header() state = self.experimentsHeaderState for i, label in enumerate(self.headerLabels): hview.setSectionHidden(i, state.get(label, True)) self.experimentsWidget.resizeColumnToContents(i)
class OWWordEnrichment(OWWidget, ConcurrentWidgetMixin): # Basic widget info name = "Word Enrichment" description = "Word enrichment analysis for selected documents." icon = "icons/SetEnrichment.svg" priority = 600 # Input/output class Inputs: selected_data = Input("Selected Data", Table) data = Input("Data", Table) want_main_area = True class Error(OWWidget.Error): no_bow_features = Msg('No bag-of-words features!') no_words_overlap = Msg('No words overlap!') empty_selection = Msg('Selected data is empty!') all_selected = Msg('All examples can not be selected!') # Settings filter_by_p: bool = Setting(False) filter_p_value: float = Setting(0.01) filter_by_fdr: bool = Setting(True) filter_fdr_value: float = Setting(0.2) def __init__(self): OWWidget.__init__(self) ConcurrentWidgetMixin.__init__(self) # Init data self.data = None self.selected_data = None # used for transforming the 'selected data' into the 'data' domain self.selected_data_transformed = None self.results = Result() # info box fbox = gui.widgetBox(self.controlArea, "Info") self.info_fil = gui.label(fbox, self, 'Words displayed: 0') # Filtering settings fbox = gui.widgetBox(self.controlArea, "Filter") hbox = gui.widgetBox(fbox, orientation=0) self.chb_p = gui.checkBox(hbox, self, "filter_by_p", "p-value", callback=self.filter_and_display, tooltip="Filter by word p-value") self.spin_p = gui.doubleSpin(hbox, self, 'filter_p_value', 1e-4, 1, step=1e-4, labelWidth=15, callback=self.filter_and_display, tooltip="Max p-value for word") self.spin_p.setEnabled(self.filter_by_p) hbox = gui.widgetBox(fbox, orientation=0) self.chb_fdr = gui.checkBox(hbox, self, "filter_by_fdr", "FDR", callback=self.filter_and_display, tooltip="Filter by word FDR") self.spin_fdr = gui.doubleSpin(hbox, self, 'filter_fdr_value', 1e-4, 1, step=1e-4, labelWidth=15, callback=self.filter_and_display, tooltip="Max p-value for word") self.spin_fdr.setEnabled(self.filter_by_fdr) gui.rubber(self.controlArea) # Word's list view self.cols = ['Word', 'p-value', 'FDR'] self.sig_words = QTreeWidget() self.sig_words.setColumnCount(len(self.cols)) self.sig_words.setHeaderLabels(self.cols) self.sig_words.setSortingEnabled(True) self.sig_words.setSelectionMode(QTreeView.NoSelection) self.sig_words.sortByColumn(2, 0) # 0 is ascending order for i in range(len(self.cols)): self.sig_words.resizeColumnToContents(i) self.mainArea.layout().addWidget(self.sig_words) def sizeHint(self): return QSize(450, 240) @Inputs.data def set_data(self, data=None): self.data = data # selected data transformed depends on data domain self.selected_data_transformed = None @Inputs.selected_data def set_data_selected(self, data=None): self.selected_data = data def handleNewSignals(self): self.check_data() def get_bow_domain(self): domain = self.data.domain return Domain(attributes=[ a for a in domain.attributes if a.attributes.get('bow-feature', False) ], class_vars=domain.class_vars, metas=domain.metas, source=domain) def check_data(self): self.Error.clear() if isinstance(self.data, Table) and \ isinstance(self.selected_data, Table): if len(self.selected_data) == 0: self.Error.empty_selection() self.clear() return # keep only BoW features bow_domain = self.get_bow_domain() if len(bow_domain.attributes) == 0: self.Error.no_bow_features() self.clear() return self.data = Corpus.from_table(bow_domain, self.data) self.selected_data_transformed = Corpus.from_table( bow_domain, self.selected_data) if np_sp_sum(self.selected_data_transformed.X) == 0: self.Error.no_words_overlap() self.clear() elif len(self.data) == len(self.selected_data): self.Error.all_selected() self.clear() else: self.set_input_info() self.apply() else: self.clear() def clear(self): self.sig_words.clear() self.info.set_input_summary(self.info.NoInput) self.set_displayed_info(0) def filter_enabled(self, b): self.chb_p.setEnabled(b) self.chb_fdr.setEnabled(b) self.spin_p.setEnabled(b) self.spin_fdr.setEnabled(b) def filter_and_display(self): self.spin_p.setEnabled(self.filter_by_p) self.spin_fdr.setEnabled(self.filter_by_fdr) self.sig_words.clear() if self.selected_data_transformed is None: # do nothing when no Data return if self.results.words: count = self.build_tree() else: count = 0 for i in range(len(self.cols)): self.sig_words.resizeColumnToContents(i) self.set_displayed_info(count) def build_tree(self) -> int: count = 0 for word, pval, fval in zip(self.results.words, self.results.p_values, self.results.fdr_values): if ((not self.filter_by_p or pval <= self.filter_p_value) and (not self.filter_by_fdr or fval <= self.filter_fdr_value)): it = EATreeWidgetItem(word, pval, fval, self.sig_words) self.sig_words.addTopLevelItem(it) count += 1 return count def set_input_info(self) -> None: cluster_words = len(self.selected_data_transformed.domain.attributes) selected_words = np.count_nonzero( np_sp_sum(self.selected_data_transformed.X, axis=0)) self.info.set_input_summary( f"{cluster_words}|{selected_words}", f"Total words: {cluster_words}\n" f"Words in subset: {selected_words}") def set_displayed_info(self, count: int) -> None: self.info_fil.setText(f"Words displayed: {count}") def apply(self): self.sig_words.clear() self.filter_enabled(False) self.start(Runner.run, self.selected_data_transformed, self.data, self.results) def on_done(self, result: Result) -> None: self.filter_and_display() self.filter_enabled(True) def on_exception(self, ex: Exception) -> None: self.filter_enabled(True) def tree_to_table(self): view = [self.cols] items = self.sig_words.topLevelItemCount() for i in range(items): line = [] for j in range(3): line.append(self.sig_words.topLevelItem(i).text(j)) view.append(line) return view def send_report(self): if self.results.words: self.report_table("Enriched words", self.tree_to_table())