class OWGEODatasets(OWWidget): name = "GEO Data Sets" description = DESCRIPTION icon = "../widgets/icons/GEODataSets.svg" priority = PRIORITY inputs = [] outputs = [("Expression Data", Orange.data.Table)] settingsList = ["outputRows", "mergeSpots", "gdsSelectionStates", "splitterSettings", "currentGds", "autoCommit", "datasetNames"] outputRows = Setting(True) mergeSpots = Setting(True) gdsSelectionStates = Setting({}) currentGds = Setting(None) datasetNames = Setting({}) splitterSettings = Setting( (b'\x00\x00\x00\xff\x00\x00\x00\x00\x00\x00\x00\x02\x00\x00\x01\xea\x00\x00\x00\xd7\x01\x00\x00\x00\x07\x01\x00\x00\x00\x02', b'\x00\x00\x00\xff\x00\x00\x00\x00\x00\x00\x00\x02\x00\x00\x01\xb5\x00\x00\x02\x10\x01\x00\x00\x00\x07\x01\x00\x00\x00\x01') ) autoCommit = Setting(False) def __init__(self, parent=None, signalManager=None, name=" GEO Data Sets"): OWWidget.__init__(self, parent, signalManager, name) self.selectionChanged = False self.filterString = "" self.datasetName = "" ## GUI box = gui.widgetBox(self.controlArea, "Info", addSpace=True) self.infoBox = gui.widgetLabel(box, "Initializing\n\n") box = gui.widgetBox(self.controlArea, "Output", addSpace=True) gui.radioButtonsInBox(box, self, "outputRows", ["Genes in rows", "Samples in rows"], "Rows", callback=self.commitIf) gui.checkBox(box, self, "mergeSpots", "Merge spots of same gene", callback=self.commitIf) gui.separator(box) self.nameEdit = gui.lineEdit( box, self, "datasetName", "Data set name", tooltip="Override the default output data set name", callback=self.onNameEdited ) self.nameEdit.setPlaceholderText("") if sys.version_info < (3, ): box = gui.widgetBox(self.controlArea, "Commit", addSpace=True) self.commitButton = gui.button( box, self, "Commit", callback=self.commit) cb = gui.checkBox(box, self, "autoCommit", "Commit on any change") gui.setStopper(self, self.commitButton, cb, "selectionChanged", self.commit) else: gui.auto_commit(self.controlArea, self, "autoCommit", "Commit", box="Commit") self.commitIf = self.commit gui.rubber(self.controlArea) gui.widgetLabel(self.mainArea, "Filter") self.filterLineEdit = QLineEdit( textChanged=self.filter ) self.completer = TokenListCompleter( self, caseSensitivity=Qt.CaseInsensitive ) self.filterLineEdit.setCompleter(self.completer) self.mainArea.layout().addWidget(self.filterLineEdit) splitter = QSplitter(Qt.Vertical, self.mainArea) self.mainArea.layout().addWidget(splitter) self.treeWidget = QTreeView(splitter) self.treeWidget.setSelectionMode(QTreeView.SingleSelection) self.treeWidget.setRootIsDecorated(False) self.treeWidget.setSortingEnabled(True) self.treeWidget.setAlternatingRowColors(True) self.treeWidget.setUniformRowHeights(True) self.treeWidget.setEditTriggers(QTreeView.NoEditTriggers) linkdelegate = LinkStyledItemDelegate(self.treeWidget) self.treeWidget.setItemDelegateForColumn(1, linkdelegate) self.treeWidget.setItemDelegateForColumn(8, linkdelegate) self.treeWidget.setItemDelegateForColumn( 0, gui.IndicatorItemDelegate(self.treeWidget, role=Qt.DisplayRole)) proxyModel = MySortFilterProxyModel(self.treeWidget) self.treeWidget.setModel(proxyModel) self.treeWidget.selectionModel().selectionChanged.connect( self.updateSelection ) self.treeWidget.viewport().setMouseTracking(True) splitterH = QSplitter(Qt.Horizontal, splitter) box = gui.widgetBox(splitterH, "Description") self.infoGDS = gui.widgetLabel(box, "") self.infoGDS.setWordWrap(True) gui.rubber(box) box = gui.widgetBox(splitterH, "Sample Annotations") self.annotationsTree = QTreeWidget(box) self.annotationsTree.setHeaderLabels( ["Type (Sample annotations)", "Sample count"] ) self.annotationsTree.setRootIsDecorated(True) box.layout().addWidget(self.annotationsTree) self.annotationsTree.itemChanged.connect( self.annotationSelectionChanged ) self._annotationsUpdating = False self.splitters = splitter, splitterH for sp, setting in zip(self.splitters, self.splitterSettings): sp.splitterMoved.connect(self.splitterMoved) sp.restoreState(setting) self.searchKeys = ["dataset_id", "title", "platform_organism", "description"] self.gds = [] self.gds_info = None self.resize(1000, 600) self.setBlocking(True) self.setEnabled(False) self.progressBarInit() self._executor = ThreadExecutor() func = partial(get_gds_model, methodinvoke(self, "_setProgress", (float,))) self._inittask = Task(function=func) self._inittask.finished.connect(self._initializemodel) self._executor.submit(self._inittask) self._datatask = None @Slot(float) def _setProgress(self, value): self.progressBarValue = value def _initializemodel(self): assert self.thread() is QThread.currentThread() model, self.gds_info, self.gds = self._inittask.result() model.setParent(self) proxy = self.treeWidget.model() proxy.setFilterKeyColumn(0) proxy.setFilterRole(TextFilterRole) proxy.setFilterCaseSensitivity(False) proxy.setFilterFixedString(self.filterString) proxy.setSourceModel(model) proxy.sort(0, Qt.DescendingOrder) self.progressBarFinished() self.setBlocking(False) self.setEnabled(True) filter_items = " ".join( gds[key] for gds in self.gds for key in self.searchKeys ) tr_chars = ",.:;!?(){}[]_-+\\|/%#@$^&*<>~`" tr_table = str.maketrans(tr_chars, " " * len(tr_chars)) filter_items = filter_items.translate(tr_table) filter_items = sorted(set(filter_items.split(" "))) filter_items = [item for item in filter_items if len(item) > 3] self.completer.setTokenList(filter_items) if self.currentGds: current_id = self.currentGds["dataset_id"] gdss = [(i, qunpack(proxy.data(proxy.index(i, 1), Qt.DisplayRole))) for i in range(proxy.rowCount())] current = [i for i, data in gdss if data and data == current_id] if current: current_index = proxy.index(current[0], 0) self.treeWidget.selectionModel().select( current_index, QItemSelectionModel.Select | QItemSelectionModel.Rows ) self.treeWidget.scrollTo( current_index, QTreeView.PositionAtCenter) for i in range(8): self.treeWidget.resizeColumnToContents(i) self.treeWidget.setColumnWidth( 1, min(self.treeWidget.columnWidth(1), 300)) self.treeWidget.setColumnWidth( 2, min(self.treeWidget.columnWidth(2), 200)) self.updateInfo() def updateInfo(self): gds_info = self.gds_info text = ("%i datasets\n%i datasets cached\n" % (len(gds_info), len(glob.glob(serverfiles.localpath("GEO") + "/GDS*")))) filtered = self.treeWidget.model().rowCount() if len(self.gds) != filtered: text += ("%i after filtering") % filtered self.infoBox.setText(text) def updateSelection(self, *args): current = self.treeWidget.selectedIndexes() mapToSource = self.treeWidget.model().mapToSource current = [mapToSource(index).row() for index in current] if current: self.currentGds = self.gds[current[0]] self.setAnnotations(self.currentGds) self.infoGDS.setText(self.currentGds.get("description", "")) self.nameEdit.setPlaceholderText(self.currentGds["title"]) self.datasetName = \ self.datasetNames.get(self.currentGds["dataset_id"], "") else: self.currentGds = None self.nameEdit.setPlaceholderText("") self.datasetName = "" self.commitIf() def setAnnotations(self, gds): self._annotationsUpdating = True self.annotationsTree.clear() annotations = defaultdict(set) subsetscount = {} for desc in gds["subsets"]: annotations[desc["type"]].add(desc["description"]) subsetscount[desc["description"]] = str(len(desc["sample_id"])) for type, subsets in annotations.items(): key = (gds["dataset_id"], type) subsetItem = QTreeWidgetItem(self.annotationsTree, [type]) subsetItem.setFlags(subsetItem.flags() | Qt.ItemIsUserCheckable | Qt.ItemIsTristate) subsetItem.setCheckState( 0, self.gdsSelectionStates.get(key, Qt.Checked) ) subsetItem.key = key for subset in subsets: key = (gds["dataset_id"], type, subset) item = QTreeWidgetItem( subsetItem, [subset, subsetscount.get(subset, "")] ) item.setFlags(item.flags() | Qt.ItemIsUserCheckable) item.setCheckState( 0, self.gdsSelectionStates.get(key, Qt.Checked) ) item.key = key self._annotationsUpdating = False self.annotationsTree.expandAll() for i in range(self.annotationsTree.columnCount()): self.annotationsTree.resizeColumnToContents(i) def annotationSelectionChanged(self, item, column): if self._annotationsUpdating: return for i in range(self.annotationsTree.topLevelItemCount()): item = self.annotationsTree.topLevelItem(i) self.gdsSelectionStates[item.key] = item.checkState(0) for j in range(item.childCount()): child = item.child(j) self.gdsSelectionStates[child.key] = child.checkState(0) def filter(self): filter_string = unicode(self.filterLineEdit.text()) proxyModel = self.treeWidget.model() if proxyModel: strings = filter_string.lower().strip().split() proxyModel.setFilterFixedStrings(strings) self.updateInfo() def selectedSamples(self): """ Return the currently selected sample annotations. The return value is a list of selected (sample type, sample value) tuples. .. note:: if some Sample annotation type has no selected values. this method will return all values for it. """ samples = [] unused_types = [] used_types = [] for stype in childiter(self.annotationsTree.invisibleRootItem()): selected_values = [] all_values = [] for sval in childiter(stype): value = (str(stype.text(0)), str(sval.text(0))) if self.gdsSelectionStates.get(sval.key, True): selected_values.append(value) all_values.append(value) if selected_values: samples.extend(selected_values) used_types.append(str(stype.text(0))) else: # If no sample of sample type is selected we don't filter # on it. samples.extend(all_values) unused_types.append(str(stype.text(0))) return samples, used_types def commitIf(self): if self.autoCommit: self.commit() else: self.selectionChanged = True @Slot(int, int) def progressCompleted(self, value, total): if total > 0: self.progressBarSet(100. * value / total, processEvents=False) else: pass # TODO: report 'indeterminate progress' def commit(self): if self.currentGds: self.error(0) sample_type = None self.progressBarInit(processEvents=None) _, groups = self.selectedSamples() if len(groups) == 1 and self.outputRows: sample_type = groups[0] self.setEnabled(False) self.setBlocking(True) progress = methodinvoke(self, "progressCompleted", (int, int)) def get_data(gds_id, report_genes, transpose, sample_type, title): gds_ensure_downloaded(gds_id, progress) gds = geo.GDS(gds_id) data = gds.getdata( report_genes=report_genes, transpose=transpose, sample_type=sample_type ) data.name = title return data get_data = partial( get_data, self.currentGds["dataset_id"], report_genes=self.mergeSpots, transpose=self.outputRows, sample_type=sample_type, title=self.datasetName or self.currentGds["title"] ) self._datatask = Task(function=get_data) self._datatask.finished.connect(self._on_dataready) self._executor.submit(self._datatask) def _on_dataready(self): self.setEnabled(True) self.setBlocking(False) self.progressBarFinished(processEvents=False) try: data = self._datatask.result() except urlrequest.URLError as error: self.error(0, ("Error while connecting to the NCBI ftp server! " "'%s'" % error)) sys.excepthook(type(error), error, getattr(error, "__traceback__")) return finally: self._datatask = None data_name = data.name samples, _ = self.selectedSamples() self.warning(0) message = None if self.outputRows: def samplesinst(ex): out = [] for meta in data.domain.metas: out.append((meta.name, ex[meta].value)) if data.domain.class_var.name != 'class': out.append((data.domain.class_var.name, ex[data.domain.class_var].value)) return out samples = set(samples) mask = [samples.issuperset(samplesinst(ex)) for ex in data] data = data[numpy.array(mask, dtype=bool)] if len(data) == 0: message = "No samples with selected sample annotations." else: samples = set(samples) domain = Orange.data.Domain( [attr for attr in data.domain.attributes if samples.issuperset(attr.attributes.items())], data.domain.class_var, data.domain.metas ) # domain.addmetas(data.domain.getmetas()) if len(domain.attributes) == 0: message = "No samples with selected sample annotations." stypes = set(s[0] for s in samples) for attr in domain.attributes: attr.attributes = dict( (key, value) for key, value in attr.attributes.items() if key in stypes ) data = Orange.data.Table(domain, data) if message is not None: self.warning(0, message) data_hints.set_hint(data, "taxid", self.currentGds.get("taxid", ""), 10.0) data_hints.set_hint(data, "genesinrows", self.outputRows, 10.0) data.name = data_name self.send("Expression Data", data) model = self.treeWidget.model().sourceModel() row = self.gds.index(self.currentGds) model.setData(model.index(row, 0), " ", Qt.DisplayRole) self.updateInfo() self.selectionChanged = False def splitterMoved(self, *args): self.splitterSettings = [bytes(sp.saveState()) for sp in self.splitters] def send_report(self): self.report_items("GEO Dataset", [("ID", self.currentGds['dataset_id']), ("Title", self.currentGds['title']), ("Organism", self.currentGds['sample_organism'])]) self.report_items("Data", [("Samples", self.currentGds['sample_count']), ("Features", self.currentGds['feature_count']), ("Genes", self.currentGds['gene_count'])]) self.report_name("Sample annotations") subsets = defaultdict(list) for subset in self.currentGds['subsets']: subsets[subset['type']].append((subset['description'], len(subset['sample_id']))) self.report_html += "<ul>" for type in subsets: self.report_html += "<b>" + type + ":</b></br>" for desc, count in subsets[type]: self.report_html += 9 * " " + "<b>{}:</b> {}</br>".format(desc, count) self.report_html += "</ul>" def onDeleteWidget(self): if self._inittask: self._inittask.future().cancel() self._inittask.finished.disconnect(self._initializemodel) if self._datatask: self._datatask.future().cancel() self._datatask.finished.disconnect(self._on_dataready) self._executor.shutdown(wait=False) super(OWGEODatasets, self).onDeleteWidget() def onNameEdited(self): if self.currentGds: gds_id = self.currentGds["dataset_id"] self.datasetNames[gds_id] = unicode(self.nameEdit.text()) self.commitIf()
class OWGEODatasets(OWWidget): name = "GEO Data Sets" description = DESCRIPTION icon = "../widgets/icons/GEODataSets.svg" priority = PRIORITY inputs = [] outputs = [("Expression Data", Orange.data.Table)] settingsList = [ "outputRows", "mergeSpots", "gdsSelectionStates", "splitterSettings", "currentGds", "autoCommit", "datasetNames" ] outputRows = Setting(True) mergeSpots = Setting(True) gdsSelectionStates = Setting({}) currentGds = Setting(None) datasetNames = Setting({}) splitterSettings = Setting(( b'\x00\x00\x00\xff\x00\x00\x00\x00\x00\x00\x00\x02\x00\x00\x01\xea\x00\x00\x00\xd7\x01\x00\x00\x00\x07\x01\x00\x00\x00\x02', b'\x00\x00\x00\xff\x00\x00\x00\x00\x00\x00\x00\x02\x00\x00\x01\xb5\x00\x00\x02\x10\x01\x00\x00\x00\x07\x01\x00\x00\x00\x01' )) autoCommit = Setting(False) def __init__(self, parent=None, signalManager=None, name=" GEO Data Sets"): OWWidget.__init__(self, parent, signalManager, name) self.selectionChanged = False self.filterString = "" self.datasetName = "" ## GUI box = gui.widgetBox(self.controlArea, "Info", addSpace=True) self.infoBox = gui.widgetLabel(box, "Initializing\n\n") box = gui.widgetBox(self.controlArea, "Output", addSpace=True) gui.radioButtonsInBox(box, self, "outputRows", ["Genes in rows", "Samples in rows"], "Rows", callback=self.commitIf) gui.checkBox(box, self, "mergeSpots", "Merge spots of same gene", callback=self.commitIf) gui.separator(box) self.nameEdit = gui.lineEdit( box, self, "datasetName", "Data set name", tooltip="Override the default output data set name", callback=self.onNameEdited) self.nameEdit.setPlaceholderText("") if sys.version_info < (3, ): box = gui.widgetBox(self.controlArea, "Commit", addSpace=True) self.commitButton = gui.button(box, self, "Commit", callback=self.commit) cb = gui.checkBox(box, self, "autoCommit", "Commit on any change") gui.setStopper(self, self.commitButton, cb, "selectionChanged", self.commit) else: gui.auto_commit(self.controlArea, self, "autoCommit", "Commit", box="Commit") self.commitIf = self.commit gui.rubber(self.controlArea) gui.widgetLabel(self.mainArea, "Filter") self.filterLineEdit = QLineEdit(textChanged=self.filter) self.completer = TokenListCompleter(self, caseSensitivity=Qt.CaseInsensitive) self.filterLineEdit.setCompleter(self.completer) self.mainArea.layout().addWidget(self.filterLineEdit) splitter = QSplitter(Qt.Vertical, self.mainArea) self.mainArea.layout().addWidget(splitter) self.treeWidget = QTreeView(splitter) self.treeWidget.setSelectionMode(QTreeView.SingleSelection) self.treeWidget.setRootIsDecorated(False) self.treeWidget.setSortingEnabled(True) self.treeWidget.setAlternatingRowColors(True) self.treeWidget.setUniformRowHeights(True) self.treeWidget.setEditTriggers(QTreeView.NoEditTriggers) linkdelegate = LinkStyledItemDelegate(self.treeWidget) self.treeWidget.setItemDelegateForColumn(1, linkdelegate) self.treeWidget.setItemDelegateForColumn(8, linkdelegate) self.treeWidget.setItemDelegateForColumn( 0, gui.IndicatorItemDelegate(self.treeWidget, role=Qt.DisplayRole)) proxyModel = MySortFilterProxyModel(self.treeWidget) self.treeWidget.setModel(proxyModel) self.treeWidget.selectionModel().selectionChanged.connect( self.updateSelection) self.treeWidget.viewport().setMouseTracking(True) splitterH = QSplitter(Qt.Horizontal, splitter) box = gui.widgetBox(splitterH, "Description") self.infoGDS = gui.widgetLabel(box, "") self.infoGDS.setWordWrap(True) gui.rubber(box) box = gui.widgetBox(splitterH, "Sample Annotations") self.annotationsTree = QTreeWidget(box) self.annotationsTree.setHeaderLabels( ["Type (Sample annotations)", "Sample count"]) self.annotationsTree.setRootIsDecorated(True) box.layout().addWidget(self.annotationsTree) self.annotationsTree.itemChanged.connect( self.annotationSelectionChanged) self._annotationsUpdating = False self.splitters = splitter, splitterH for sp, setting in zip(self.splitters, self.splitterSettings): sp.splitterMoved.connect(self.splitterMoved) sp.restoreState(setting) self.searchKeys = [ "dataset_id", "title", "platform_organism", "description" ] self.gds = [] self.gds_info = None self.resize(1000, 600) self.setBlocking(True) self.setEnabled(False) self.progressBarInit() self._executor = ThreadExecutor() func = partial(get_gds_model, methodinvoke(self, "_setProgress", (float, ))) self._inittask = Task(function=func) self._inittask.finished.connect(self._initializemodel) self._executor.submit(self._inittask) self._datatask = None @Slot(float) def _setProgress(self, value): self.progressBarValue = value def _initializemodel(self): assert self.thread() is QThread.currentThread() model, self.gds_info, self.gds = self._inittask.result() model.setParent(self) proxy = self.treeWidget.model() proxy.setFilterKeyColumn(0) proxy.setFilterRole(TextFilterRole) proxy.setFilterCaseSensitivity(False) proxy.setFilterFixedString(self.filterString) proxy.setSourceModel(model) proxy.sort(0, Qt.DescendingOrder) self.progressBarFinished() self.setBlocking(False) self.setEnabled(True) filter_items = " ".join(gds[key] for gds in self.gds for key in self.searchKeys) tr_chars = ",.:;!?(){}[]_-+\\|/%#@$^&*<>~`" tr_table = str.maketrans(tr_chars, " " * len(tr_chars)) filter_items = filter_items.translate(tr_table) filter_items = sorted(set(filter_items.split(" "))) filter_items = [item for item in filter_items if len(item) > 3] self.completer.setTokenList(filter_items) if self.currentGds: current_id = self.currentGds["dataset_id"] gdss = [(i, qunpack(proxy.data(proxy.index(i, 1), Qt.DisplayRole))) for i in range(proxy.rowCount())] current = [i for i, data in gdss if data and data == current_id] if current: current_index = proxy.index(current[0], 0) self.treeWidget.selectionModel().select( current_index, QItemSelectionModel.Select | QItemSelectionModel.Rows) self.treeWidget.scrollTo(current_index, QTreeView.PositionAtCenter) for i in range(8): self.treeWidget.resizeColumnToContents(i) self.treeWidget.setColumnWidth( 1, min(self.treeWidget.columnWidth(1), 300)) self.treeWidget.setColumnWidth( 2, min(self.treeWidget.columnWidth(2), 200)) self.updateInfo() def updateInfo(self): gds_info = self.gds_info text = ("%i datasets\n%i datasets cached\n" % (len(gds_info), len(glob.glob(serverfiles.localpath("GEO") + "/GDS*")))) filtered = self.treeWidget.model().rowCount() if len(self.gds) != filtered: text += ("%i after filtering") % filtered self.infoBox.setText(text) def updateSelection(self, *args): current = self.treeWidget.selectedIndexes() mapToSource = self.treeWidget.model().mapToSource current = [mapToSource(index).row() for index in current] if current: self.currentGds = self.gds[current[0]] self.setAnnotations(self.currentGds) self.infoGDS.setText(self.currentGds.get("description", "")) self.nameEdit.setPlaceholderText(self.currentGds["title"]) self.datasetName = \ self.datasetNames.get(self.currentGds["dataset_id"], "") else: self.currentGds = None self.nameEdit.setPlaceholderText("") self.datasetName = "" self.commitIf() def setAnnotations(self, gds): self._annotationsUpdating = True self.annotationsTree.clear() annotations = defaultdict(set) subsetscount = {} for desc in gds["subsets"]: annotations[desc["type"]].add(desc["description"]) subsetscount[desc["description"]] = str(len(desc["sample_id"])) for type, subsets in annotations.items(): key = (gds["dataset_id"], type) subsetItem = QTreeWidgetItem(self.annotationsTree, [type]) subsetItem.setFlags(subsetItem.flags() | Qt.ItemIsUserCheckable | Qt.ItemIsTristate) subsetItem.setCheckState( 0, self.gdsSelectionStates.get(key, Qt.Checked)) subsetItem.key = key for subset in subsets: key = (gds["dataset_id"], type, subset) item = QTreeWidgetItem( subsetItem, [subset, subsetscount.get(subset, "")]) item.setFlags(item.flags() | Qt.ItemIsUserCheckable) item.setCheckState( 0, self.gdsSelectionStates.get(key, Qt.Checked)) item.key = key self._annotationsUpdating = False self.annotationsTree.expandAll() for i in range(self.annotationsTree.columnCount()): self.annotationsTree.resizeColumnToContents(i) def annotationSelectionChanged(self, item, column): if self._annotationsUpdating: return for i in range(self.annotationsTree.topLevelItemCount()): item = self.annotationsTree.topLevelItem(i) self.gdsSelectionStates[item.key] = item.checkState(0) for j in range(item.childCount()): child = item.child(j) self.gdsSelectionStates[child.key] = child.checkState(0) def filter(self): filter_string = unicode(self.filterLineEdit.text()) proxyModel = self.treeWidget.model() if proxyModel: strings = filter_string.lower().strip().split() proxyModel.setFilterFixedStrings(strings) self.updateInfo() def selectedSamples(self): """ Return the currently selected sample annotations. The return value is a list of selected (sample type, sample value) tuples. .. note:: if some Sample annotation type has no selected values. this method will return all values for it. """ samples = [] unused_types = [] used_types = [] for stype in childiter(self.annotationsTree.invisibleRootItem()): selected_values = [] all_values = [] for sval in childiter(stype): value = (str(stype.text(0)), str(sval.text(0))) if self.gdsSelectionStates.get(sval.key, True): selected_values.append(value) all_values.append(value) if selected_values: samples.extend(selected_values) used_types.append(str(stype.text(0))) else: # If no sample of sample type is selected we don't filter # on it. samples.extend(all_values) unused_types.append(str(stype.text(0))) return samples, used_types def commitIf(self): if self.autoCommit: self.commit() else: self.selectionChanged = True @Slot(int, int) def progressCompleted(self, value, total): if total > 0: self.progressBarSet(100. * value / total, processEvents=False) else: pass # TODO: report 'indeterminate progress' def commit(self): if self.currentGds: self.error(0) sample_type = None self.progressBarInit(processEvents=None) _, groups = self.selectedSamples() if len(groups) == 1 and self.outputRows: sample_type = groups[0] self.setEnabled(False) self.setBlocking(True) progress = methodinvoke(self, "progressCompleted", (int, int)) def get_data(gds_id, report_genes, transpose, sample_type, title): gds_ensure_downloaded(gds_id, progress) gds = geo.GDS(gds_id) data = gds.getdata(report_genes=report_genes, transpose=transpose, sample_type=sample_type) data.name = title return data get_data = partial(get_data, self.currentGds["dataset_id"], report_genes=self.mergeSpots, transpose=self.outputRows, sample_type=sample_type, title=self.datasetName or self.currentGds["title"]) self._datatask = Task(function=get_data) self._datatask.finished.connect(self._on_dataready) self._executor.submit(self._datatask) def _on_dataready(self): self.setEnabled(True) self.setBlocking(False) self.progressBarFinished(processEvents=False) try: data = self._datatask.result() except urlrequest.URLError as error: self.error(0, ("Error while connecting to the NCBI ftp server! " "'%s'" % error)) sys.excepthook(type(error), error, getattr(error, "__traceback__")) return finally: self._datatask = None data_name = data.name samples, _ = self.selectedSamples() self.warning(0) message = None if self.outputRows: def samplesinst(ex): out = [] for meta in data.domain.metas: out.append((meta.name, ex[meta].value)) if data.domain.class_var.name != 'class': out.append((data.domain.class_var.name, ex[data.domain.class_var].value)) return out samples = set(samples) mask = [samples.issuperset(samplesinst(ex)) for ex in data] data = data[numpy.array(mask, dtype=bool)] if len(data) == 0: message = "No samples with selected sample annotations." else: samples = set(samples) domain = Orange.data.Domain([ attr for attr in data.domain.attributes if samples.issuperset(attr.attributes.items()) ], data.domain.class_var, data.domain.metas) # domain.addmetas(data.domain.getmetas()) if len(domain.attributes) == 0: message = "No samples with selected sample annotations." stypes = set(s[0] for s in samples) for attr in domain.attributes: attr.attributes = dict( (key, value) for key, value in attr.attributes.items() if key in stypes) data = Orange.data.Table(domain, data) if message is not None: self.warning(0, message) data_hints.set_hint(data, "taxid", self.currentGds.get("taxid", ""), 10.0) data_hints.set_hint(data, "genesinrows", self.outputRows, 10.0) data.name = data_name self.send("Expression Data", data) model = self.treeWidget.model().sourceModel() row = self.gds.index(self.currentGds) model.setData(model.index(row, 0), " ", Qt.DisplayRole) self.updateInfo() self.selectionChanged = False def splitterMoved(self, *args): self.splitterSettings = [ bytes(sp.saveState()) for sp in self.splitters ] def send_report(self): self.report_items("GEO Dataset", [("ID", self.currentGds['dataset_id']), ("Title", self.currentGds['title']), ("Organism", self.currentGds['sample_organism'])]) self.report_items("Data", [("Samples", self.currentGds['sample_count']), ("Features", self.currentGds['feature_count']), ("Genes", self.currentGds['gene_count'])]) self.report_name("Sample annotations") subsets = defaultdict(list) for subset in self.currentGds['subsets']: subsets[subset['type']].append( (subset['description'], len(subset['sample_id']))) self.report_html += "<ul>" for type in subsets: self.report_html += "<b>" + type + ":</b></br>" for desc, count in subsets[type]: self.report_html += 9 * " " + "<b>{}:</b> {}</br>".format( desc, count) self.report_html += "</ul>" def onDeleteWidget(self): if self._inittask: self._inittask.future().cancel() self._inittask.finished.disconnect(self._initializemodel) if self._datatask: self._datatask.future().cancel() self._datatask.finished.disconnect(self._on_dataready) self._executor.shutdown(wait=False) super(OWGEODatasets, self).onDeleteWidget() def onNameEdited(self): if self.currentGds: gds_id = self.currentGds["dataset_id"] self.datasetNames[gds_id] = unicode(self.nameEdit.text()) self.commitIf()
class OWItemsets(widget.OWWidget): name = 'Frequent Itemsets' description = 'Explore sets of items that frequently appear together.' icon = 'icons/FrequentItemsets.svg' priority = 10 class Inputs: data = Input("Data", Table) class Outputs: matching_data = Output("Matching Data", Table) class Error(widget.OWWidget.Error): need_discrete_data = widget.Msg( "Need some discrete data to work with.") no_disc_features = widget.Msg( "Discrete features required but data has none.") class Warning(widget.OWWidget.Warning): cont_attrs = widget.Msg( "Data has continuous attributes which will be skipped.") err_reg_expression = widget.Msg("Error in regular expression: {}") minSupport = settings.Setting(30) maxItemsets = settings.Setting(10000) filterSearch = settings.Setting(True) autoFind = settings.Setting(False) autoSend = settings.Setting(True) filterKeywords = settings.Setting('') filterMinItems = settings.Setting(1) filterMaxItems = settings.Setting(10000) UserAdviceMessages = [ widget.Message( 'Itemset are listed in item-sorted order, i.e. ' 'an itemset containing A and B is only listed once, as ' 'A > B (and not also B > A).', 'itemsets-order', widget.Message.Warning), widget.Message( 'To select all the itemsets that are descendants of ' '(include) some item X (i.e. the whole subtree), you ' 'can fold the subtree at that item and then select it.', 'itemsets-order', widget.Message.Information) ] def __init__(self): self.data = None self.output = None self._is_running = False self.isRegexMatch = lambda x: True self.tree = QTreeWidget(self.mainArea, columnCount=2, allColumnsShowFocus=True, alternatingRowColors=True, selectionMode=QTreeWidget.ExtendedSelection, uniformRowHeights=True) self.tree.setHeaderLabels(["Itemsets", "Support", "%"]) self.tree.header().setStretchLastSection(True) self.tree.itemSelectionChanged.connect(self.selectionChanged) self.mainArea.layout().addWidget(self.tree) box = gui.widgetBox(self.controlArea, "Info") self.nItemsets = self.nSelectedExamples = self.nSelectedItemsets = '' gui.label(box, self, "Number of itemsets: %(nItemsets)s") gui.label(box, self, "Selected itemsets: %(nSelectedItemsets)s") gui.label(box, self, "Selected examples: %(nSelectedExamples)s") hbox = gui.widgetBox(box, orientation='horizontal') gui.button(hbox, self, "Expand all", callback=self.tree.expandAll) gui.button(hbox, self, "Collapse all", callback=self.tree.collapseAll) box = gui.widgetBox(self.controlArea, 'Find itemsets') gui.valueSlider(box, self, 'minSupport', values=[.0001, .0005, .001, .005, .01, .05, .1, .5] + list(range(1, 101)), label='Minimal support:', labelFormat="%g%%", callback=lambda: self.find_itemsets()) gui.hSlider(box, self, 'maxItemsets', minValue=10000, maxValue=100000, step=10000, label='Max. number of itemsets:', labelFormat="%d", callback=lambda: self.find_itemsets()) self.button = gui.auto_commit( box, self, 'autoFind', 'Find Itemsets', commit=self.find_itemsets, callback=lambda: self.autoFind and self.find_itemsets()) box = gui.widgetBox(self.controlArea, 'Filter itemsets') gui.lineEdit(box, self, 'filterKeywords', 'Contains:', callback=self.filter_change, orientation='horizontal', tooltip='A comma or space-separated list of regular ' 'expressions.') hbox = gui.widgetBox(box, orientation='horizontal') gui.spin(hbox, self, 'filterMinItems', 1, 998, label='Min. items:', callback=self.filter_change) gui.spin(hbox, self, 'filterMaxItems', 2, 999, label='Max. items:', callback=self.filter_change) gui.checkBox(box, self, 'filterSearch', label='Apply these filters in search', tooltip='If checked, the itemsets are filtered according ' 'to these filter conditions already in the search ' 'phase. \nIf unchecked, the only filters applied ' 'during search are the ones above, ' 'and the itemsets are \nfiltered afterwards only for ' 'display, i.e. only the matching itemsets are shown.') gui.rubber(hbox) gui.rubber(self.controlArea) gui.auto_commit(self.controlArea, self, 'autoSend', 'Send selection') self.filter_change() ITEM_DATA_ROLE = Qt.UserRole + 1 def selectionChanged(self): X = self.X mapping = self.onehot_mapping instances = set() where = np.where def whole_subtree(node): yield node for i in range(node.childCount()): yield from whole_subtree(node.child(i)) def itemset(node): while node: yield node.data(0, self.ITEM_DATA_ROLE) node = node.parent() def selection_ranges(node): n_children = node.childCount() if n_children: yield (self.tree.indexFromItem(node.child(0)), self.tree.indexFromItem(node.child(n_children - 1))) for i in range(n_children): yield from selection_ranges(node.child(i)) nSelectedItemsets = 0 item_selection = QItemSelection() for node in self.tree.selectedItems(): nodes = (node, ) if node.isExpanded() else whole_subtree(node) if not node.isExpanded(): for srange in selection_ranges(node): item_selection.select(*srange) for node in nodes: nSelectedItemsets += 1 cols, vals = zip(*(mapping[i] for i in itemset(node))) if issparse(X): rows = (len(cols) == np.bincount( (X[:, cols] != 0).indices, minlength=X.shape[0])).nonzero()[0] else: rows = where((X[:, cols] == vals).all(axis=1))[0] instances.update(rows) self.tree.itemSelectionChanged.disconnect(self.selectionChanged) self.tree.selectionModel().select( item_selection, QItemSelectionModel.Select | QItemSelectionModel.Rows) self.tree.itemSelectionChanged.connect(self.selectionChanged) self.nSelectedExamples = len(instances) self.nSelectedItemsets = nSelectedItemsets self.output = self.data[sorted(instances)] or None self.commit() def commit(self): self.Outputs.matching_data.send(self.output) def filter_change(self): self.Warning.err_reg_expression.clear() try: isRegexMatch = self.isRegexMatch = re.compile( '|'.join( i.strip() for i in re.split('(,|\s)+', self.filterKeywords.strip()) if i.strip()), re.IGNORECASE).search except Exception as e: self.Warning.err_reg_expression(e.args[0]) isRegexMatch = self.isRegexMatch = lambda x: True def hide(node, depth, has_kw): if not has_kw: has_kw = isRegexMatch(node.text(0)) hidden = ( sum( hide(node.child(i), depth + 1, has_kw) for i in range(node.childCount())) == node.childCount() if node.childCount() else (not has_kw or not self.filterMinItems <= depth <= self.filterMaxItems)) node.setHidden(hidden) return hidden hide(self.tree.invisibleRootItem(), 0, False) class TreeWidgetItem(QTreeWidgetItem): def data(self, column, role): """Construct lazy tooltips""" if role != Qt.ToolTipRole: return super().data(column, role) tooltip = [] while self: tooltip.append(self.text(0)) self = self.parent() return '\n'.join(reversed(tooltip)) def find_itemsets(self): if self.data is None or not len(self.data): return if self._is_running: self._is_running = False return self._is_running = True self.button.button.setText('Cancel') data = self.data self.tree.clear() self.tree.setUpdatesEnabled(False) self.tree.blockSignals(True) class ItemDict(dict): def __init__(self, item): self.item = item top = ItemDict(self.tree.invisibleRootItem()) X, mapping = OneHot.encode(data) self.Error.need_discrete_data.clear() if X is None: self.Error.need_discrete_data() self.onehot_mapping = mapping ITEM_FMT = '{}' if issparse(data.X) else '{}={}' names = { item: ITEM_FMT.format(var.name, val) for item, var, val in OneHot.decode(mapping.keys(), data, mapping) } nItemsets = 0 filterSearch = self.filterSearch filterMinItems, filterMaxItems = self.filterMinItems, self.filterMaxItems isRegexMatch = self.isRegexMatch # Find itemsets and populate the TreeView with self.progressBar(self.maxItemsets + 1) as progress: for itemset, support in frequent_itemsets(X, self.minSupport / 100): if filterSearch and not filterMinItems <= len( itemset) <= filterMaxItems: continue parent = top first_new_item = None itemset_matches_filter = False for item in sorted(itemset): name = names[item] if filterSearch and not itemset_matches_filter: itemset_matches_filter = isRegexMatch(name) child = parent.get(name) if child is None: try: wi = self.TreeWidgetItem(parent.item, [ name, str(support), '{:.4g}'.format( 100 * support / len(data)) ]) except RuntimeError: # FIXME: When autoFind was in effect and the support # slider was moved, this line excepted with: # RuntimeError: wrapped C/C++ object of type # TreeWidgetItem has been deleted return wi.setData(0, self.ITEM_DATA_ROLE, item) child = parent[name] = ItemDict(wi) if first_new_item is None: first_new_item = (parent, name) parent = child if filterSearch and not itemset_matches_filter: parent, name = first_new_item parent.item.removeChild(parent[name].item) del parent[name].item del parent[name] else: nItemsets += 1 progress.advance() if not self._is_running or nItemsets >= self.maxItemsets: break qApp.processEvents() if not filterSearch: self.filter_change() self.nItemsets = nItemsets self.nSelectedItemsets = 0 self.nSelectedExamples = 0 self.tree.expandAll() for i in range(self.tree.columnCount()): self.tree.resizeColumnToContents(i) self.tree.setUpdatesEnabled(True) self.tree.blockSignals(False) self._is_running = False self.button.button.setText('Find Itemsets') @Inputs.data def set_data(self, data): self.data = data is_error = False if data is not None: self.Warning.cont_attrs.clear() self.Error.no_disc_features.clear() self.button.setDisabled(False) self.X = data.X if issparse(data.X): self.X = data.X.tocsc() else: if not data.domain.has_discrete_attributes(): self.Error.no_disc_features() is_error = True self.button.setDisabled(True) elif data.domain.has_continuous_attributes(): self.Warning.cont_attrs() else: self.output = None self.commit() if self.autoFind and not is_error: self.find_itemsets()
class OWKEGGPathwayBrowser(widget.OWWidget): name = "KEGG Pathways" description = "Browse KEGG pathways that include an input set of genes." icon = "../widgets/icons/OWKEGGPathwayBrowser.svg" priority = 70 inputs = [("Data", Orange.data.Table, "SetData", widget.Default), ("Reference", Orange.data.Table, "SetRefData")] outputs = [("Selected Data", Orange.data.Table, widget.Default), ("Unselected Data", Orange.data.Table)] autoCommit = settings.Setting(False) autoResize = settings.Setting(True) useReference = settings.Setting(False) showOrthology = settings.Setting(True) Ready, Initializing, Running = 0, 1, 2 class Error(widget.OWWidget.Error): missing_annotation = widget.Msg(ERROR_ON_MISSING_ANNOTATION) missing_gene_id = widget.Msg(ERROR_ON_MISSING_GENE_ID) missing_tax_id = widget.Msg(ERROR_ON_MISSING_TAX_ID) def __init__(self, parent=None): super().__init__(parent) self._changedFlag = False self.__invalidated = False self.__runstate = OWKEGGPathwayBrowser.Initializing self.__in_setProgress = False self.controlArea.setMaximumWidth(250) box = gui.widgetBox(self.controlArea, "Info") self.infoLabel = gui.widgetLabel(box, "No data on input\n") gui.separator(self.controlArea) gui.checkBox(self.controlArea, self, "useReference", "From signal", box="Reference", callback=self.Update) gui.separator(self.controlArea) gui.checkBox( self.controlArea, self, "showOrthology", "Show pathways in full orthology", box="Orthology", callback=self.UpdateListView, ) gui.checkBox( self.controlArea, self, "autoResize", "Resize to fit", box="Image", callback=self.UpdatePathwayViewTransform, ) box = gui.widgetBox(self.controlArea, "Cache Control") gui.button( box, self, "Clear cache", callback=self.ClearCache, tooltip="Clear all locally cached KEGG data.", default=False, autoDefault=False, ) gui.separator(self.controlArea) gui.auto_commit(self.controlArea, self, "autoCommit", "Commit") gui.rubber(self.controlArea) spliter = QSplitter(Qt.Vertical, self.mainArea) self.pathwayView = PathwayView(self, spliter) self.pathwayView.scene().selectionChanged.connect( self._onSelectionChanged) self.mainArea.layout().addWidget(spliter) self.listView = QTreeWidget(allColumnsShowFocus=True, selectionMode=QTreeWidget.SingleSelection, sortingEnabled=True, maximumHeight=200) spliter.addWidget(self.listView) self.listView.setColumnCount(4) self.listView.setHeaderLabels( ["Pathway", "P value", "Genes", "Reference"]) self.listView.itemSelectionChanged.connect(self.UpdatePathwayView) select = QAction("Select All", self, shortcut=QKeySequence.SelectAll) select.triggered.connect(self.selectAll) self.addAction(select) self.data = None self.input_genes = [] self.tax_id = None self.use_attr_names = None self.gene_id_attribute = None self.gene_id_column = None self.ref_data = None self.ref_genes = [] self.ref_tax_id = None self.ref_use_attr_names = None self.ref_gene_id_attribute = None self.ref_gene_id_column = None self.pathways = {} self.ncbi_gene_map = [] self.org = None self._executor = concurrent.ThreadExecutor() self.setEnabled(False) self.setBlocking(True) progress = concurrent.methodinvoke(self, "setProgress", (float, )) def get_genome(): """Return a KEGGGenome with the common org entries precached.""" genome = kegg.KEGGGenome() essential = genome.essential_organisms() common = genome.common_organisms() # Remove duplicates of essential from common. # (essential + common list as defined here will be used in the # GUI.) common = [c for c in common if c not in essential] # TODO: Add option to specify additional organisms not # in the common list. keys = list(map(genome.org_code_to_entry_key, essential + common)) genome.pre_cache(keys, progress_callback=progress) return (keys, genome) self._genomeTask = task = concurrent.Task(function=get_genome) task.finished.connect(self.__initialize_finish) self.progressBarInit() self.infoLabel.setText("Fetching organism definitions\n") self._executor.submit(task) def __initialize_finish(self): if self.__runstate != OWKEGGPathwayBrowser.Initializing: return try: keys, genome = self._genomeTask.result() except Exception as err: self.error(0, str(err)) raise self.progressBarFinished() self.setEnabled(True) self.setBlocking(False) self.infoLabel.setText("No data on input\n") def clear(self): """ Clear the widget state. """ self.pathways = {} self.org = None self.infoLabel.setText("No data on input\n") self.listView.clear() self.pathwayView.SetPathway(None) self.send("Selected Data", None) self.send("Unselected Data", None) def SetData(self, data=None): if self.__runstate == OWKEGGPathwayBrowser.Initializing: self.__initialize_finish() self.Error.clear() if data: self.data = data self.tax_id = str(self.data.attributes.get(TAX_ID, None)) self.use_attr_names = self.data.attributes.get( GENE_AS_ATTRIBUTE_NAME, None) self.gene_id_attribute = self.data.attributes.get( GENE_ID_ATTRIBUTE, None) self.gene_id_column = self.data.attributes.get( GENE_ID_COLUMN, None) if not (self.use_attr_names is not None and ((self.gene_id_attribute is None) ^ (self.gene_id_column is None))): if self.tax_id is None: self.Error.missing_annotation() return self.Error.missing_gene_id() return elif self.tax_id is None: self.Error.missing_tax_id() return self.warning(0) self.error(0) self.information(0) self.__invalidated = True else: self.clear() def SetRefData(self, data=None): self.information(1) if data is not None and self.useReference: self.ref_data = data self.ref_tax_id = str(self.ref_data.attributes.get(TAX_ID, None)) self.ref_use_attr_names = self.ref_data.attributes.get( GENE_AS_ATTRIBUTE_NAME, None) self.ref_gene_id_attribute = self.ref_data.attributes.get( GENE_ID_ATTRIBUTE, None) self.ref_gene_id_column = self.ref_data.attributes.get( GENE_ID_COLUMN, None) if not (self.ref_use_attr_names is not None and ((self.ref_gene_id_attribute is None) ^ (self.ref_gene_id_column is None))): if self.ref_tax_id is None: self.Error.missing_annotation() return self.Error.missing_gene_id() return elif self.ref_tax_id is None: self.Error.missing_tax_id() return self.__invalidated = True def handleNewSignals(self): if self.__invalidated: self.Update() self.__invalidated = False def UpdateListView(self): self.bestPValueItem = None self.listView.clear() if not self.data: return allPathways = self.org.pathways() allRefPathways = kegg.pathways("map") items = [] kegg_pathways = kegg.KEGGPathways() org_code = self.org.org_code if self.showOrthology: self.koOrthology = kegg.KEGGBrite("ko00001") self.listView.setRootIsDecorated(True) path_ids = {s[-5:] for s in self.pathways.keys()} def _walkCollect(koEntry): num = koEntry.title[:5] if koEntry.title else None if num in path_ids: return [koEntry] + reduce( lambda li, c: li + _walkCollect(c), [child for child in koEntry.entries], []) else: c = reduce(lambda li, c: li + _walkCollect(c), [child for child in koEntry.entries], []) return c + (c and [koEntry] or []) allClasses = reduce(lambda li1, li2: li1 + li2, [_walkCollect(c) for c in self.koOrthology], []) def _walkCreate(koEntry, lvItem): item = QTreeWidgetItem(lvItem) id = "path:" + org_code + koEntry.title[:5] if koEntry.title[:5] in path_ids: p = kegg_pathways.get_entry(id) if p is None: # In case the genesets still have obsolete entries name = koEntry.title else: name = p.name genes, p_value, ref = self.pathways[id] item.setText(0, name) item.setText(1, "%.5f" % p_value) item.setText( 2, "%i of %i" % (len(genes), len(self.input_genes))) item.setText(3, "%i of %i" % (ref, len(self.ref_genes))) item.pathway_id = id if p is not None else None else: if id in allPathways: text = kegg_pathways.get_entry(id).name else: text = koEntry.title item.setText(0, text) if id in allPathways: item.pathway_id = id elif "path:map" + koEntry.title[:5] in allRefPathways: item.pathway_id = "path:map" + koEntry.title[:5] else: item.pathway_id = None for child in koEntry.entries: if child in allClasses: _walkCreate(child, item) for koEntry in self.koOrthology: if koEntry in allClasses: _walkCreate(koEntry, self.listView) self.listView.update() else: self.listView.setRootIsDecorated(False) pathways = self.pathways.items() pathways = sorted(pathways, key=lambda item: item[1][1]) for id, (genes, p_value, ref) in pathways: item = QTreeWidgetItem(self.listView) item.setText(0, kegg_pathways.get_entry(id).name) item.setText(1, "%.5f" % p_value) item.setText(2, "%i of %i" % (len(genes), len(self.input_genes))) item.setText(3, "%i of %i" % (ref, len(self.ref_genes))) item.pathway_id = id items.append(item) self.bestPValueItem = items and items[0] or None self.listView.expandAll() for i in range(4): self.listView.resizeColumnToContents(i) if self.bestPValueItem: index = self.listView.indexFromItem(self.bestPValueItem) self.listView.selectionModel().select( index, QItemSelectionModel.ClearAndSelect) def UpdatePathwayView(self): items = self.listView.selectedItems() if len(items) > 0: item = items[0] else: item = None self.commit() item = item or self.bestPValueItem if not item or not item.pathway_id: self.pathwayView.SetPathway(None) return def get_kgml_and_image(pathway_id): """Return an initialized KEGGPathway with pre-cached data""" p = kegg.KEGGPathway(pathway_id) p._get_kgml() # makes sure the kgml file is downloaded p._get_image_filename() # makes sure the image is downloaded return (pathway_id, p) self.setEnabled(False) self._pathwayTask = concurrent.Task( function=lambda: get_kgml_and_image(item.pathway_id)) self._pathwayTask.finished.connect(self._onPathwayTaskFinshed) self._executor.submit(self._pathwayTask) def _onPathwayTaskFinshed(self): self.setEnabled(True) pathway_id, self.pathway = self._pathwayTask.result() objects = self.pathways.get(pathway_id, [[]])[0] # [ncbi_gene_id] # map ncbi_gene_id to keg_id for display objects = flatten(relation_map(self.ncbi_gene_map, objects)) self.pathwayView.SetPathway(self.pathway, objects) def UpdatePathwayViewTransform(self): self.pathwayView.updateTransform() def Update(self): """ Update (recompute enriched pathways) the widget state. """ if not self.data: return self.error(0) self.information(0) # XXX: Check data in setData, do not even allow this to be executed if # data has no genes try: self.__get_input_genes() self.input_genes = set(self.input_genes) except ValueError: self.error(0, "Cannot extract gene names from input.") self.information(1) self.org = kegg.KEGGOrganism(kegg.from_taxid(self.tax_id)) if self.useReference and self.ref_data: self.__get_ref_genes() self.ref_genes = set(self.ref_genes) else: self.ref_genes = self.org.get_ncbi_ids() def run_enrichment(org_code, genes, reference, progress=None): # We use the kegg pathway gene sets provided by 'geneset' for # the enrichment calculation. kegg_api = kegg.api.CachedKeggApi() link_map = kegg_api.link(org_code, "pathway") # [(pathway_id, kegg_gene_id)] ncbi_gene_map = kegg_api.conv( org_code, 'ncbi-geneid') # [(ncbi_gene_id, kegg_gene_id)] ncbi_gene_map = [(_1.split(":", 1)[1], _2) for _1, _2 in ncbi_gene_map] link_map = relation_join( link_map, [(_2, _1) for _1, _2 in ncbi_gene_map]) # [(pathway_id, ncbi_gene_id)] kegg_sets = relation_list_to_multimap( link_map) # {pathway_id -> [ncbi_gene_ids]} # map kegg gene ids to ncbi_gene_ids. kegg_sets = geneset.GeneSets(sets=[ geneset.GeneSet(gs_id=ddi, genes=set(genes)) for ddi, genes in kegg_sets.items() ]) pathways = pathway_enrichment(kegg_sets, genes, reference, callback=progress) # Ensure that pathway entries are pre-cached for later use in the # list/tree view kegg_pathways = kegg.KEGGPathways() kegg_pathways.pre_cache(pathways.keys(), progress_callback=progress) return pathways, ncbi_gene_map self.progressBarInit() self.setEnabled(False) self.infoLabel.setText("Retrieving...\n") progress = concurrent.methodinvoke(self, "setProgress", (float, )) run_func = partial(run_enrichment, self.org.org_code, self.input_genes, self.ref_genes, progress) self._enrichTask = concurrent.Task(function=run_func) self._enrichTask.finished.connect(self._onEnrichTaskFinished) self._executor.submit(self._enrichTask) def _onEnrichTaskFinished(self): self.setEnabled(True) self.setBlocking(False) try: pathways, ncbi_gene_map = self._enrichTask.result() except Exception: raise self.progressBarFinished() self.pathways = pathways self.ncbi_gene_map = ncbi_gene_map if not self.pathways: self.warning(0, "No enriched pathways found.") else: self.warning(0) self.infoLabel.setText("{} unique gene names on input\n".format( len(set(self.input_genes)))) self.UpdateListView() @Slot(float) def setProgress(self, value): if self.__in_setProgress: return self.__in_setProgress = True self.progressBarSet(value) self.__in_setProgress = False def __get_input_genes(self): """ Extract and return gene names from `data`. """ self.input_genes = [] if self.use_attr_names: for variable in self.data.domain.attributes: self.input_genes.append( str(variable.attributes.get(self.gene_id_attribute, '?'))) else: genes, _ = self.data.get_column_view(self.gene_id_column) self.input_genes = [str(g) for g in genes] if len(self.input_genes) <= 0: raise ValueError("No gene names in data.") def __get_ref_genes(self): """ Extract and return gene names from `data`. """ self.ref_genes = [] if self.ref_use_attr_names: for variable in self.ref_data.domain.attributes: self.ref_genes.append( str( variable.attributes.get(self.ref_gene_id_attribute, '?'))) else: genes, _ = self.ref_data.get_column_view(self.ref_gene_id_column) self.ref_genes = [str(g) for g in genes] def selectAll(self): """ Select all items in the pathway view. """ changed = False scene = self.pathwayView.scene() with disconnected(scene.selectionChanged, self._onSelectionChanged): for item in scene.items(): if item.flags( ) & QGraphicsItem.ItemIsSelectable and not item.isSelected(): item.setSelected(True) changed = True if changed: self._onSelectionChanged() def _onSelectionChanged(self): # Item selection in the pathwayView/scene has changed self.commit() def commit(self): if self.data: selectedItems = self.pathwayView.scene().selectedItems() selectedGenes = reduce( set.union, [item.marked_objects for item in selectedItems], set()) # map kegg_ids back to ncbi_gene_id backmap = [(_2, _1) for _1, _2 in self.ncbi_gene_map] selectedGenes = set(flatten(relation_map(backmap, selectedGenes))) if self.use_attr_names: selected = [ column for column in self.data.domain.attributes if self.gene_id_attribute in column.attributes and str(column.attributes[ self.gene_id_attribute]) in selectedGenes ] data = self.data[:, selected] self.send("Selected Data", data) else: selected_indices = [] other_indices = [] for row_index, row in enumerate(self.data): gene_in_row = str(row[self.gene_id_column]) if gene_in_row in self.input_genes and gene_in_row in selectedGenes: selected_indices.append(row_index) else: other_indices.append(row_index) if selected_indices: selected = self.data[selected_indices] else: selected = None if other_indices: other = self.data[other_indices] else: other = None self.send("Selected Data", selected) self.send("Unselected Data", other) else: self.send("Selected Data", None) self.send("Unselected Data", None) def ClearCache(self): kegg.caching.clear_cache() def onDeleteWidget(self): """ Called before the widget is removed from the canvas. """ super().onDeleteWidget() self.org = None self._executor.shutdown(wait=False) gc.collect() # Force collection (WHY?) def sizeHint(self): return QSize(1024, 720)
class OWKEGGPathwayBrowser(widget.OWWidget): name = "KEGG Pathways" description = "Browse KEGG pathways that include an input set of genes." icon = "../widgets/icons/KEGGPathways.svg" priority = 2030 inputs = [("Data", Orange.data.Table, "SetData", widget.Default), ("Reference", Orange.data.Table, "SetRefData")] outputs = [("Selected Data", Orange.data.Table, widget.Default), ("Unselected Data", Orange.data.Table)] settingsHandler = settings.DomainContextHandler() organismIndex = settings.ContextSetting(0) geneAttrIndex = settings.ContextSetting(0) useAttrNames = settings.ContextSetting(False) autoCommit = settings.Setting(False) autoResize = settings.Setting(True) useReference = settings.Setting(False) showOrthology = settings.Setting(True) Ready, Initializing, Running = 0, 1, 2 def __init__(self, parent=None): super().__init__(parent) self.organismCodes = [] self._changedFlag = False self.__invalidated = False self.__runstate = OWKEGGPathwayBrowser.Initializing self.__in_setProgress = False self.controlArea.setMaximumWidth(250) box = gui.widgetBox(self.controlArea, "Info") self.infoLabel = gui.widgetLabel(box, "No data on input\n") # Organism selection. box = gui.widgetBox(self.controlArea, "Organism") self.organismComboBox = gui.comboBox( box, self, "organismIndex", items=[], callback=self.Update, addSpace=True, tooltip="Select the organism of the input genes", ) # Selection of genes attribute box = gui.widgetBox(self.controlArea, "Gene attribute") self.geneAttrCandidates = itemmodels.VariableListModel(parent=self) self.geneAttrCombo = gui.comboBox(box, self, "geneAttrIndex", callback=self.Update) self.geneAttrCombo.setModel(self.geneAttrCandidates) gui.checkBox( box, self, "useAttrNames", "Use variable names", disables=[(-1, self.geneAttrCombo)], callback=self.Update ) self.geneAttrCombo.setDisabled(bool(self.useAttrNames)) gui.separator(self.controlArea) gui.checkBox(self.controlArea, self, "useReference", "From signal", box="Reference", callback=self.Update) gui.separator(self.controlArea) gui.checkBox( self.controlArea, self, "showOrthology", "Show pathways in full orthology", box="Orthology", callback=self.UpdateListView, ) gui.checkBox( self.controlArea, self, "autoResize", "Resize to fit", box="Image", callback=self.UpdatePathwayViewTransform ) box = gui.widgetBox(self.controlArea, "Cache Control") gui.button( box, self, "Clear cache", callback=self.ClearCache, tooltip="Clear all locally cached KEGG data.", default=False, autoDefault=False, ) gui.separator(self.controlArea) gui.auto_commit(self.controlArea, self, "autoCommit", "Commit") gui.rubber(self.controlArea) spliter = QSplitter(Qt.Vertical, self.mainArea) self.pathwayView = PathwayView(self, spliter) self.pathwayView.scene().selectionChanged.connect(self._onSelectionChanged) self.mainArea.layout().addWidget(spliter) self.listView = QTreeWidget( allColumnsShowFocus=True, selectionMode=QTreeWidget.SingleSelection, sortingEnabled=True, maximumHeight=200 ) spliter.addWidget(self.listView) self.listView.setColumnCount(4) self.listView.setHeaderLabels(["Pathway", "P value", "Genes", "Reference"]) self.listView.itemSelectionChanged.connect(self.UpdatePathwayView) select = QAction("Select All", self, shortcut=QKeySequence.SelectAll) select.triggered.connect(self.selectAll) self.addAction(select) self.data = None self.refData = None self._executor = concurrent.ThreadExecutor() self.setEnabled(False) self.setBlocking(True) progress = concurrent.methodinvoke(self, "setProgress", (float,)) def get_genome(): """Return a KEGGGenome with the common org entries precached.""" genome = kegg.KEGGGenome() essential = genome.essential_organisms() common = genome.common_organisms() # Remove duplicates of essential from common. # (essential + common list as defined here will be used in the # GUI.) common = [c for c in common if c not in essential] # TODO: Add option to specify additional organisms not # in the common list. keys = list(map(genome.org_code_to_entry_key, essential + common)) genome.pre_cache(keys, progress_callback=progress) return (keys, genome) self._genomeTask = task = concurrent.Task(function=get_genome) task.finished.connect(self.__initialize_finish) self.progressBarInit() self.infoLabel.setText("Fetching organism definitions\n") self._executor.submit(task) def __initialize_finish(self): if self.__runstate != OWKEGGPathwayBrowser.Initializing: return try: keys, genome = self._genomeTask.result() except Exception as err: self.error(0, str(err)) raise self.progressBarFinished() self.setEnabled(True) self.setBlocking(False) entries = [genome[key] for key in keys] items = [entry.definition for entry in entries] codes = [entry.organism_code for entry in entries] self.organismCodes = codes self.organismComboBox.clear() self.organismComboBox.addItems(items) self.organismComboBox.setCurrentIndex(self.organismIndex) self.infoLabel.setText("No data on input\n") def Clear(self): """ Clear the widget state. """ self.queryGenes = [] self.referenceGenes = [] self.genes = {} self.uniqueGenesDict = {} self.revUniqueGenesDict = {} self.pathways = {} self.org = None self.geneAttrCandidates[:] = [] self.infoLabel.setText("No data on input\n") self.listView.clear() self.pathwayView.SetPathway(None) self.send("Selected Data", None) self.send("Unselected Data", None) def SetData(self, data=None): if self.__runstate == OWKEGGPathwayBrowser.Initializing: self.__initialize_finish() self.closeContext() self.data = data self.warning(0) self.error(0) self.information(0) if data is not None: vars = data.domain.variables + data.domain.metas vars = [var for var in vars if isinstance(var, Orange.data.StringVariable)] self.geneAttrCandidates[:] = vars # Try to guess the gene name variable if vars: names_lower = [v.name.lower() for v in vars] scores = [(name == "gene", "gene" in name) for name in names_lower] imax, _ = max(enumerate(scores), key=itemgetter(1)) else: imax = -1 self.geneAttrIndex = imax taxid = data_hints.get_hint(data, "taxid", None) if taxid: try: code = kegg.from_taxid(taxid) self.organismIndex = self.organismCodes.index(code) except Exception as ex: print(ex, taxid) self.useAttrNames = data_hints.get_hint(data, "genesinrows", self.useAttrNames) self.openContext(data) if len(self.geneAttrCandidates) == 0: self.useAttrNames = True self.geneAttrIndex = -1 else: self.geneAttrIndex = min(self.geneAttrIndex, len(self.geneAttrCandidates) - 1) else: self.Clear() self.__invalidated = True def SetRefData(self, data=None): self.refData = data self.information(1) if data is not None and self.useReference: self.__invalidated = True def handleNewSignals(self): if self.__invalidated: self.Update() self.__invalidated = False def UpdateListView(self): self.bestPValueItem = None self.listView.clear() if not self.data: return allPathways = self.org.pathways() allRefPathways = kegg.pathways("map") items = [] kegg_pathways = kegg.KEGGPathways() org_code = self.organismCodes[min(self.organismIndex, len(self.organismCodes) - 1)] if self.showOrthology: self.koOrthology = kegg.KEGGBrite("ko00001") self.listView.setRootIsDecorated(True) path_ids = set([s[-5:] for s in self.pathways.keys()]) def _walkCollect(koEntry): num = koEntry.title[:5] if koEntry.title else None if num in path_ids: return [koEntry] + reduce( lambda li, c: li + _walkCollect(c), [child for child in koEntry.entries], [] ) else: c = reduce(lambda li, c: li + _walkCollect(c), [child for child in koEntry.entries], []) return c + (c and [koEntry] or []) allClasses = reduce(lambda li1, li2: li1 + li2, [_walkCollect(c) for c in self.koOrthology], []) def _walkCreate(koEntry, lvItem): item = QTreeWidgetItem(lvItem) id = "path:" + org_code + koEntry.title[:5] if koEntry.title[:5] in path_ids: p = kegg_pathways.get_entry(id) if p is None: # In case the genesets still have obsolete entries name = koEntry.title else: name = p.name genes, p_value, ref = self.pathways[id] item.setText(0, name) item.setText(1, "%.5f" % p_value) item.setText(2, "%i of %i" % (len(genes), len(self.genes))) item.setText(3, "%i of %i" % (ref, len(self.referenceGenes))) item.pathway_id = id if p is not None else None else: if id in allPathways: text = kegg_pathways.get_entry(id).name else: text = koEntry.title item.setText(0, text) if id in allPathways: item.pathway_id = id elif "path:map" + koEntry.title[:5] in allRefPathways: item.pathway_id = "path:map" + koEntry.title[:5] else: item.pathway_id = None for child in koEntry.entries: if child in allClasses: _walkCreate(child, item) for koEntry in self.koOrthology: if koEntry in allClasses: _walkCreate(koEntry, self.listView) self.listView.update() else: self.listView.setRootIsDecorated(False) pathways = self.pathways.items() pathways = sorted(pathways, key=lambda item: item[1][1]) for id, (genes, p_value, ref) in pathways: item = QTreeWidgetItem(self.listView) item.setText(0, kegg_pathways.get_entry(id).name) item.setText(1, "%.5f" % p_value) item.setText(2, "%i of %i" % (len(genes), len(self.genes))) item.setText(3, "%i of %i" % (ref, len(self.referenceGenes))) item.pathway_id = id items.append(item) self.bestPValueItem = items and items[0] or None self.listView.expandAll() for i in range(4): self.listView.resizeColumnToContents(i) if self.bestPValueItem: index = self.listView.indexFromItem(self.bestPValueItem) self.listView.selectionModel().select(index, QItemSelectionModel.ClearAndSelect) def UpdatePathwayView(self): items = self.listView.selectedItems() if len(items) > 0: item = items[0] else: item = None self.commit() item = item or self.bestPValueItem if not item or not item.pathway_id: self.pathwayView.SetPathway(None) return def get_kgml_and_image(pathway_id): """Return an initialized KEGGPathway with pre-cached data""" p = kegg.KEGGPathway(pathway_id) p._get_kgml() # makes sure the kgml file is downloaded p._get_image_filename() # makes sure the image is downloaded return (pathway_id, p) self.setEnabled(False) self._pathwayTask = concurrent.Task(function=lambda: get_kgml_and_image(item.pathway_id)) self._pathwayTask.finished.connect(self._onPathwayTaskFinshed) self._executor.submit(self._pathwayTask) def _onPathwayTaskFinshed(self): self.setEnabled(True) pathway_id, self.pathway = self._pathwayTask.result() self.pathwayView.SetPathway(self.pathway, self.pathways.get(pathway_id, [[]])[0]) def UpdatePathwayViewTransform(self): self.pathwayView.updateTransform() def Update(self): """ Update (recompute enriched pathways) the widget state. """ if not self.data: return self.error(0) self.information(0) # XXX: Check data in setData, do not even alow this to be executed if # data has no genes try: genes = self.GeneNamesFromData(self.data) except ValueError: self.error(0, "Cannot extract gene names from input.") genes = [] if not self.useAttrNames and any("," in gene for gene in genes): genes = reduce(add, (split_and_strip(gene, ",") for gene in genes), []) self.information(0, "Separators detected in input gene names. " "Assuming multiple genes per instance.") self.queryGenes = genes self.information(1) reference = None if self.useReference and self.refData: reference = self.GeneNamesFromData(self.refData) if not self.useAttrNames and any("," in gene for gene in reference): reference = reduce(add, (split_and_strip(gene, ",") for gene in reference), []) self.information( 1, "Separators detected in reference gene " "names. Assuming multiple genes per " "instance." ) org_code = self.SelectedOrganismCode() def run_enrichment(org_code, genes, reference=None, progress=None): org = kegg.KEGGOrganism(org_code) if reference is None: reference = org.get_genes() # Map 'genes' and 'reference' sets to unique KEGG identifiers unique_genes, _, _ = org.get_unique_gene_ids(set(genes)) unique_ref_genes, _, _ = org.get_unique_gene_ids(set(reference)) taxid = kegg.to_taxid(org.org_code) # Map the taxid back to standard 'common' taxids # (as used by 'geneset') if applicable r_tax_map = dict((v, k) for k, v in kegg.KEGGGenome.TAXID_MAP.items()) if taxid in r_tax_map: taxid = r_tax_map[taxid] # We use the kegg pathway gene sets provided by 'geneset' for # the enrichment calculation. # Ensure we are using the latest genesets # TODO: ?? Is updating the index enough? serverfiles.update(geneset.sfdomain, "index.pck") kegg_gs_collections = geneset.collections((("KEGG", "pathways"), taxid)) pathways = pathway_enrichment( kegg_gs_collections, unique_genes.keys(), unique_ref_genes.keys(), callback=progress ) # Ensure that pathway entries are pre-cached for later use in the # list/tree view kegg_pathways = kegg.KEGGPathways() kegg_pathways.pre_cache(pathways.keys(), progress_callback=progress) return pathways, org, unique_genes, unique_ref_genes self.progressBarInit() self.setEnabled(False) self.infoLabel.setText("Retrieving...\n") progress = concurrent.methodinvoke(self, "setProgress", (float,)) self._enrichTask = concurrent.Task(function=lambda: run_enrichment(org_code, genes, reference, progress)) self._enrichTask.finished.connect(self._onEnrichTaskFinished) self._executor.submit(self._enrichTask) def _onEnrichTaskFinished(self): self.setEnabled(True) self.setBlocking(False) try: pathways, org, unique_genes, unique_ref_genes = self._enrichTask.result() except Exception: raise self.progressBarFinished() self.org = org self.genes = unique_genes.keys() self.uniqueGenesDict = unique_genes self.revUniqueGenesDict = dict([(val, key) for key, val in self.uniqueGenesDict.items()]) self.referenceGenes = unique_ref_genes.keys() self.pathways = pathways if not self.pathways: self.warning(0, "No enriched pathways found.") else: self.warning(0) count = len(set(self.queryGenes)) self.infoLabel.setText( "%i unique gene names on input\n" "%i (%.1f%%) genes names matched" % (count, len(unique_genes), 100.0 * len(unique_genes) / count if count else 0.0) ) self.UpdateListView() @Slot(float) def setProgress(self, value): if self.__in_setProgress: return self.__in_setProgress = True self.progressBarSet(value) self.__in_setProgress = False def GeneNamesFromData(self, data): """ Extract and return gene names from `data`. """ if self.useAttrNames: genes = [str(v.name).strip() for v in data.domain.attributes] elif self.geneAttrCandidates: assert 0 <= self.geneAttrIndex < len(self.geneAttrCandidates) geneAttr = self.geneAttrCandidates[self.geneAttrIndex] genes = [str(e[geneAttr]) for e in data if not numpy.isnan(e[geneAttr])] else: raise ValueError("No gene names in data.") return genes def SelectedOrganismCode(self): """ Return the selected organism code. """ return self.organismCodes[min(self.organismIndex, len(self.organismCodes) - 1)] def selectAll(self): """ Select all items in the pathway view. """ changed = False scene = self.pathwayView.scene() with disconnected(scene.selectionChanged, self._onSelectionChanged): for item in scene.items(): if item.flags() & QGraphicsItem.ItemIsSelectable and not item.isSelected(): item.setSelected(True) changed = True if changed: self._onSelectionChanged() def _onSelectionChanged(self): # Item selection in the pathwayView/scene has changed self.commit() def commit(self): if self.data: selectedItems = self.pathwayView.scene().selectedItems() selectedGenes = reduce(set.union, [item.marked_objects for item in selectedItems], set()) if self.useAttrNames: selected = [self.data.domain[self.uniqueGenesDict[gene]] for gene in selectedGenes] # newDomain = Orange.data.Domain(selectedVars, 0) data = self.data[:, selected] # data = Orange.data.Table(newDomain, self.data) self.send("Selected Data", data) elif self.geneAttrCandidates: assert 0 <= self.geneAttrIndex < len(self.geneAttrCandidates) geneAttr = self.geneAttrCandidates[self.geneAttrIndex] selectedIndices = [] otherIndices = [] for i, ex in enumerate(self.data): names = [ self.revUniqueGenesDict.get(name, None) for name in split_and_strip(str(ex[geneAttr]), ",") ] if any(name and name in selectedGenes for name in names): selectedIndices.append(i) else: otherIndices.append(i) if selectedIndices: selected = self.data[selectedIndices] else: selected = None if otherIndices: other = self.data[otherIndices] else: other = None self.send("Selected Data", selected) self.send("Unselected Data", other) else: self.send("Selected Data", None) self.send("Unselected Data", None) def ClearCache(self): kegg.caching.clear_cache() def onDeleteWidget(self): """ Called before the widget is removed from the canvas. """ super().onDeleteWidget() self.org = None self._executor.shutdown(wait=False) gc.collect() # Force collection (WHY?) def sizeHint(self): return QSize(1024, 720)
class OWKEGGPathwayBrowser(widget.OWWidget): name = "KEGG Pathways" description = "Browse KEGG pathways that include an input set of genes." icon = "../widgets/icons/OWKEGGPathwayBrowser.svg" priority = 8 inputs = [("Data", Orange.data.Table, "SetData", widget.Default), ("Reference", Orange.data.Table, "SetRefData")] outputs = [("Selected Data", Orange.data.Table, widget.Default), ("Unselected Data", Orange.data.Table)] settingsHandler = settings.DomainContextHandler() organismIndex = settings.ContextSetting(0) geneAttrIndex = settings.ContextSetting(0) useAttrNames = settings.ContextSetting(False) autoCommit = settings.Setting(False) autoResize = settings.Setting(True) useReference = settings.Setting(False) showOrthology = settings.Setting(True) Ready, Initializing, Running = 0, 1, 2 def __init__(self, parent=None): super().__init__(parent) self.organismCodes = [] self._changedFlag = False self.__invalidated = False self.__runstate = OWKEGGPathwayBrowser.Initializing self.__in_setProgress = False self.controlArea.setMaximumWidth(250) box = gui.widgetBox(self.controlArea, "Info") self.infoLabel = gui.widgetLabel(box, "No data on input\n") # Organism selection. box = gui.widgetBox(self.controlArea, "Organism") self.organismComboBox = gui.comboBox( box, self, "organismIndex", items=[], callback=self.Update, addSpace=True, tooltip="Select the organism of the input genes") # Selection of genes attribute box = gui.widgetBox(self.controlArea, "Gene attribute") self.geneAttrCandidates = itemmodels.VariableListModel(parent=self) self.geneAttrCombo = gui.comboBox( box, self, "geneAttrIndex", callback=self.Update) self.geneAttrCombo.setModel(self.geneAttrCandidates) gui.checkBox(box, self, "useAttrNames", "Use variable names", disables=[(-1, self.geneAttrCombo)], callback=self.Update) self.geneAttrCombo.setDisabled(bool(self.useAttrNames)) gui.separator(self.controlArea) gui.checkBox(self.controlArea, self, "useReference", "From signal", box="Reference", callback=self.Update) gui.separator(self.controlArea) gui.checkBox(self.controlArea, self, "showOrthology", "Show pathways in full orthology", box="Orthology", callback=self.UpdateListView) gui.checkBox(self.controlArea, self, "autoResize", "Resize to fit", box="Image", callback=self.UpdatePathwayViewTransform) box = gui.widgetBox(self.controlArea, "Cache Control") gui.button(box, self, "Clear cache", callback=self.ClearCache, tooltip="Clear all locally cached KEGG data.", default=False, autoDefault=False) gui.separator(self.controlArea) gui.auto_commit(self.controlArea, self, "autoCommit", "Commit") gui.rubber(self.controlArea) spliter = QSplitter(Qt.Vertical, self.mainArea) self.pathwayView = PathwayView(self, spliter) self.pathwayView.scene().selectionChanged.connect( self._onSelectionChanged ) self.mainArea.layout().addWidget(spliter) self.listView = QTreeWidget( allColumnsShowFocus=True, selectionMode=QTreeWidget.SingleSelection, sortingEnabled=True, maximumHeight=200) spliter.addWidget(self.listView) self.listView.setColumnCount(4) self.listView.setHeaderLabels( ["Pathway", "P value", "Genes", "Reference"]) self.listView.itemSelectionChanged.connect(self.UpdatePathwayView) select = QAction( "Select All", self, shortcut=QKeySequence.SelectAll ) select.triggered.connect(self.selectAll) self.addAction(select) self.data = None self.refData = None self._executor = concurrent.ThreadExecutor() self.setEnabled(False) self.setBlocking(True) progress = concurrent.methodinvoke(self, "setProgress", (float,)) def get_genome(): """Return a KEGGGenome with the common org entries precached.""" genome = kegg.KEGGGenome() essential = genome.essential_organisms() common = genome.common_organisms() # Remove duplicates of essential from common. # (essential + common list as defined here will be used in the # GUI.) common = [c for c in common if c not in essential] # TODO: Add option to specify additional organisms not # in the common list. keys = list(map(genome.org_code_to_entry_key, essential + common)) genome.pre_cache(keys, progress_callback=progress) return (keys, genome) self._genomeTask = task = concurrent.Task(function=get_genome) task.finished.connect(self.__initialize_finish) self.progressBarInit() self.infoLabel.setText("Fetching organism definitions\n") self._executor.submit(task) def __initialize_finish(self): if self.__runstate != OWKEGGPathwayBrowser.Initializing: return try: keys, genome = self._genomeTask.result() except Exception as err: self.error(0, str(err)) raise self.progressBarFinished() self.setEnabled(True) self.setBlocking(False) entries = [genome[key] for key in keys] items = [entry.definition for entry in entries] codes = [entry.organism_code for entry in entries] self.organismCodes = codes self.organismComboBox.clear() self.organismComboBox.addItems(items) self.organismComboBox.setCurrentIndex(self.organismIndex) self.infoLabel.setText("No data on input\n") def Clear(self): """ Clear the widget state. """ self.queryGenes = [] self.referenceGenes = [] self.genes = {} self.uniqueGenesDict = {} self.revUniqueGenesDict = {} self.pathways = {} self.org = None self.geneAttrCandidates[:] = [] self.infoLabel.setText("No data on input\n") self.listView.clear() self.pathwayView.SetPathway(None) self.send("Selected Data", None) self.send("Unselected Data", None) def SetData(self, data=None): if self.__runstate == OWKEGGPathwayBrowser.Initializing: self.__initialize_finish() self.data = data self.warning(0) self.error(0) self.information(0) if data is not None: vars = data.domain.variables + data.domain.metas vars = [var for var in vars if isinstance(var, Orange.data.StringVariable)] self.geneAttrCandidates[:] = vars # Try to guess the gene name variable if vars: names_lower = [v.name.lower() for v in vars] scores = [(name == "gene", "gene" in name) for name in names_lower] imax, _ = max(enumerate(scores), key=itemgetter(1)) else: imax = -1 self.geneAttrIndex = imax taxid = data_hints.get_hint(data, TAX_ID, None) if taxid: try: code = kegg.from_taxid(taxid) self.organismIndex = self.organismCodes.index(code) except Exception as ex: print(ex, taxid) self.useAttrNames = data_hints.get_hint(data, GENE_NAME, self.useAttrNames) if len(self.geneAttrCandidates) == 0: self.useAttrNames = True self.geneAttrIndex = -1 else: self.geneAttrIndex = min(self.geneAttrIndex, len(self.geneAttrCandidates) - 1) else: self.Clear() self.__invalidated = True def SetRefData(self, data=None): self.refData = data self.information(1) if data is not None and self.useReference: self.__invalidated = True def handleNewSignals(self): if self.__invalidated: self.Update() self.__invalidated = False def UpdateListView(self): self.bestPValueItem = None self.listView.clear() if not self.data: return allPathways = self.org.pathways() allRefPathways = kegg.pathways("map") items = [] kegg_pathways = kegg.KEGGPathways() org_code = self.organismCodes[min(self.organismIndex, len(self.organismCodes) - 1)] if self.showOrthology: self.koOrthology = kegg.KEGGBrite("ko00001") self.listView.setRootIsDecorated(True) path_ids = set([s[-5:] for s in self.pathways.keys()]) def _walkCollect(koEntry): num = koEntry.title[:5] if koEntry.title else None if num in path_ids: return ([koEntry] + reduce(lambda li, c: li + _walkCollect(c), [child for child in koEntry.entries], [])) else: c = reduce(lambda li, c: li + _walkCollect(c), [child for child in koEntry.entries], []) return c + (c and [koEntry] or []) allClasses = reduce(lambda li1, li2: li1 + li2, [_walkCollect(c) for c in self.koOrthology], []) def _walkCreate(koEntry, lvItem): item = QTreeWidgetItem(lvItem) id = "path:" + org_code + koEntry.title[:5] if koEntry.title[:5] in path_ids: p = kegg_pathways.get_entry(id) if p is None: # In case the genesets still have obsolete entries name = koEntry.title else: name = p.name genes, p_value, ref = self.pathways[id] item.setText(0, name) item.setText(1, "%.5f" % p_value) item.setText(2, "%i of %i" % (len(genes), len(self.genes))) item.setText(3, "%i of %i" % (ref, len(self.referenceGenes))) item.pathway_id = id if p is not None else None else: if id in allPathways: text = kegg_pathways.get_entry(id).name else: text = koEntry.title item.setText(0, text) if id in allPathways: item.pathway_id = id elif "path:map" + koEntry.title[:5] in allRefPathways: item.pathway_id = "path:map" + koEntry.title[:5] else: item.pathway_id = None for child in koEntry.entries: if child in allClasses: _walkCreate(child, item) for koEntry in self.koOrthology: if koEntry in allClasses: _walkCreate(koEntry, self.listView) self.listView.update() else: self.listView.setRootIsDecorated(False) pathways = self.pathways.items() pathways = sorted(pathways, key=lambda item: item[1][1]) for id, (genes, p_value, ref) in pathways: item = QTreeWidgetItem(self.listView) item.setText(0, kegg_pathways.get_entry(id).name) item.setText(1, "%.5f" % p_value) item.setText(2, "%i of %i" % (len(genes), len(self.genes))) item.setText(3, "%i of %i" % (ref, len(self.referenceGenes))) item.pathway_id = id items.append(item) self.bestPValueItem = items and items[0] or None self.listView.expandAll() for i in range(4): self.listView.resizeColumnToContents(i) if self.bestPValueItem: index = self.listView.indexFromItem(self.bestPValueItem) self.listView.selectionModel().select( index, QItemSelectionModel.ClearAndSelect ) def UpdatePathwayView(self): items = self.listView.selectedItems() if len(items) > 0: item = items[0] else: item = None self.commit() item = item or self.bestPValueItem if not item or not item.pathway_id: self.pathwayView.SetPathway(None) return def get_kgml_and_image(pathway_id): """Return an initialized KEGGPathway with pre-cached data""" p = kegg.KEGGPathway(pathway_id) p._get_kgml() # makes sure the kgml file is downloaded p._get_image_filename() # makes sure the image is downloaded return (pathway_id, p) self.setEnabled(False) self._pathwayTask = concurrent.Task( function=lambda: get_kgml_and_image(item.pathway_id) ) self._pathwayTask.finished.connect(self._onPathwayTaskFinshed) self._executor.submit(self._pathwayTask) def _onPathwayTaskFinshed(self): self.setEnabled(True) pathway_id, self.pathway = self._pathwayTask.result() self.pathwayView.SetPathway( self.pathway, self.pathways.get(pathway_id, [[]])[0] ) def UpdatePathwayViewTransform(self): self.pathwayView.updateTransform() def Update(self): """ Update (recompute enriched pathways) the widget state. """ if not self.data: return self.error(0) self.information(0) # XXX: Check data in setData, do not even allow this to be executed if # data has no genes try: genes = self.GeneNamesFromData(self.data) except ValueError: self.error(0, "Cannot extract gene names from input.") genes = [] if not self.useAttrNames and any("," in gene for gene in genes): genes = reduce(add, (split_and_strip(gene, ",") for gene in genes), []) self.information(0, "Separators detected in input gene names. " "Assuming multiple genes per instance.") self.queryGenes = genes self.information(1) reference = None if self.useReference and self.refData: reference = self.GeneNamesFromData(self.refData) if not self.useAttrNames \ and any("," in gene for gene in reference): reference = reduce(add, (split_and_strip(gene, ",") for gene in reference), []) self.information(1, "Separators detected in reference gene " "names. Assuming multiple genes per " "instance.") org_code = self.SelectedOrganismCode() from orangecontrib.bioinformatics.ncbi.gene import GeneMatcher gm = GeneMatcher(kegg.to_taxid(org_code)) gm.genes = genes gm.run_matcher() mapped_genes = {gene: str(ncbi_id) for gene, ncbi_id in gm.map_input_to_ncbi().items()} def run_enrichment(org_code, genes, reference=None, progress=None): org = kegg.KEGGOrganism(org_code) if reference is None: reference = org.get_ncbi_ids() # This is here just to keep widget working without any major changes. # map not needed, geneMatcher will not work on widget level. unique_genes = genes unique_ref_genes = dict([(gene, gene) for gene in set(reference)]) taxid = kegg.to_taxid(org.org_code) # Map the taxid back to standard 'common' taxids # (as used by 'geneset') if applicable r_tax_map = dict((v, k) for k, v in kegg.KEGGGenome.TAXID_MAP.items()) if taxid in r_tax_map: taxid = r_tax_map[taxid] # We use the kegg pathway gene sets provided by 'geneset' for # the enrichment calculation. kegg_api = kegg.api.CachedKeggApi() linkmap = kegg_api.link(org.org_code, "pathway") converted_ids = kegg_api.conv(org.org_code, 'ncbi-geneid') kegg_sets = relation_list_to_multimap(linkmap, dict((gene.upper(), ncbi.split(':')[-1]) for ncbi, gene in converted_ids)) kegg_sets = geneset.GeneSets(input=kegg_sets) pathways = pathway_enrichment( kegg_sets, unique_genes.values(), unique_ref_genes.keys(), callback=progress ) # Ensure that pathway entries are pre-cached for later use in the # list/tree view kegg_pathways = kegg.KEGGPathways() kegg_pathways.pre_cache( pathways.keys(), progress_callback=progress ) return pathways, org, unique_genes, unique_ref_genes self.progressBarInit() self.setEnabled(False) self.infoLabel.setText("Retrieving...\n") progress = concurrent.methodinvoke(self, "setProgress", (float,)) self._enrichTask = concurrent.Task( function=lambda: run_enrichment(org_code, mapped_genes, reference, progress) ) self._enrichTask.finished.connect(self._onEnrichTaskFinished) self._executor.submit(self._enrichTask) def _onEnrichTaskFinished(self): self.setEnabled(True) self.setBlocking(False) try: pathways, org, unique_genes, unique_ref_genes = \ self._enrichTask.result() except Exception: raise self.progressBarFinished() self.org = org self.genes = unique_genes.keys() self.uniqueGenesDict = {ncbi_id: input_name for input_name, ncbi_id in unique_genes.items()} self.revUniqueGenesDict = dict([(val, key) for key, val in self.uniqueGenesDict.items()]) self.referenceGenes = unique_ref_genes.keys() self.pathways = pathways if not self.pathways: self.warning(0, "No enriched pathways found.") else: self.warning(0) count = len(set(self.queryGenes)) self.infoLabel.setText( "%i unique gene names on input\n" "%i (%.1f%%) genes names matched" % (count, len(unique_genes), 100.0 * len(unique_genes) / count if count else 0.0) ) self.UpdateListView() @Slot(float) def setProgress(self, value): if self.__in_setProgress: return self.__in_setProgress = True self.progressBarSet(value) self.__in_setProgress = False def GeneNamesFromData(self, data): """ Extract and return gene names from `data`. """ if self.useAttrNames: genes = [str(v.name).strip() for v in data.domain.attributes] elif self.geneAttrCandidates: assert 0 <= self.geneAttrIndex < len(self.geneAttrCandidates) geneAttr = self.geneAttrCandidates[self.geneAttrIndex] genes = [str(e[geneAttr]) for e in data if not numpy.isnan(e[geneAttr])] else: raise ValueError("No gene names in data.") return genes def SelectedOrganismCode(self): """ Return the selected organism code. """ return self.organismCodes[min(self.organismIndex, len(self.organismCodes) - 1)] def selectAll(self): """ Select all items in the pathway view. """ changed = False scene = self.pathwayView.scene() with disconnected(scene.selectionChanged, self._onSelectionChanged): for item in scene.items(): if item.flags() & QGraphicsItem.ItemIsSelectable and \ not item.isSelected(): item.setSelected(True) changed = True if changed: self._onSelectionChanged() def _onSelectionChanged(self): # Item selection in the pathwayView/scene has changed self.commit() def commit(self): if self.data: selectedItems = self.pathwayView.scene().selectedItems() selectedGenes = reduce(set.union, [item.marked_objects for item in selectedItems], set()) if self.useAttrNames: selected = [self.data.domain[self.uniqueGenesDict[gene]] for gene in selectedGenes] # newDomain = Orange.data.Domain(selectedVars, 0) data = self.data[:, selected] # data = Orange.data.Table(newDomain, self.data) self.send("Selected Data", data) elif self.geneAttrCandidates: assert 0 <= self.geneAttrIndex < len(self.geneAttrCandidates) geneAttr = self.geneAttrCandidates[self.geneAttrIndex] selectedIndices = [] otherIndices = [] for i, ex in enumerate(self.data): names = [self.revUniqueGenesDict.get(name, None) for name in split_and_strip(str(ex[geneAttr]), ",")] if any(name and name in selectedGenes for name in names): selectedIndices.append(i) else: otherIndices.append(i) if selectedIndices: selected = self.data[selectedIndices] else: selected = None if otherIndices: other = self.data[otherIndices] else: other = None self.send("Selected Data", selected) self.send("Unselected Data", other) else: self.send("Selected Data", None) self.send("Unselected Data", None) def ClearCache(self): kegg.caching.clear_cache() def onDeleteWidget(self): """ Called before the widget is removed from the canvas. """ super().onDeleteWidget() self.org = None self._executor.shutdown(wait=False) gc.collect() # Force collection (WHY?) def sizeHint(self): return QSize(1024, 720)
class OWGEODatasets(OWWidget, ConcurrentWidgetMixin): name = "GEO Data Sets" description = "Access to Gene Expression Omnibus data sets." icon = "icons/OWGEODatasets.svg" priority = 2 class Outputs: gds_data = Output("Expression Data", Table) search_pattern = Setting('') auto_commit = Setting(True) genes_as_rows = Setting(False) mergeSpots = Setting(True) selected_gds = Setting(None) gdsSelectionStates = Setting({}) splitter_settings = Setting(( b'\x00\x00\x00\xff\x00\x00\x00\x00\x00\x00\x00\x02\x00\x00\x01\xea\x00\x00\x00\xd7\x01\x00\x00\x00\x07\x01\x00\x00\x00\x02', b'\x00\x00\x00\xff\x00\x00\x00\x00\x00\x00\x00\x02\x00\x00\x01\xb5\x00\x00\x02\x10\x01\x00\x00\x00\x07\x01\x00\x00\x00\x01', )) def __init__(self): OWWidget.__init__(self) ConcurrentWidgetMixin.__init__(self) self.gds_info: Optional[GDSInfo] = GDSInfo( ) # TODO: handle possible exceptions self.gds_data: Optional[Table] = None # Control area box = widgetBox(self.controlArea, 'Info', addSpace=True) self.infoBox = widgetLabel(box, 'Initializing\n\n') box = widgetBox(self.controlArea, 'Output', addSpace=True) radioButtonsInBox(box, self, 'genes_as_rows', ['Samples in rows', 'Genes in rows'], callback=self._run) separator(box) rubber(self.controlArea) auto_commit(self.controlArea, self, 'auto_commit', '&Commit', box=False) # Main Area # Filter widget self.filter = lineEdit(self.mainArea, self, 'search_pattern', 'Filter:', callbackOnType=True, callback=self._apply_filter) self.mainArea.layout().addWidget(self.filter) splitter_vertical = QSplitter(Qt.Vertical, self.mainArea) self.mainArea.layout().addWidget(splitter_vertical) # set table view self.table_view = QTableView(splitter_vertical) self.table_view.setShowGrid(False) self.table_view.setSortingEnabled(True) self.table_view.sortByColumn(1, Qt.AscendingOrder) self.table_view.setAlternatingRowColors(True) self.table_view.verticalHeader().setVisible(False) self.table_view.setSelectionBehavior(QAbstractItemView.SelectRows) self.table_view.setSelectionMode(QAbstractItemView.SingleSelection) self.table_view.viewport().setMouseTracking(True) self.table_view.setSizeAdjustPolicy( QAbstractScrollArea.AdjustToContents) self.table_model = GEODatasetsModel() self.table_model.initialize(self.gds_info) self.table_view.setModel(self.table_model) self.table_model.show_table() self.table_view.horizontalHeader().setStretchLastSection(True) self.table_view.resizeColumnsToContents() v_header = self.table_view.verticalHeader() option = self.table_view.viewOptions() size = self.table_view.style().sizeFromContents( QStyle.CT_ItemViewItem, option, QSize(20, 20), self.table_view) v_header.setDefaultSectionSize(size.height() + 2) v_header.setMinimumSectionSize(5) # set item delegates self.table_view.setItemDelegateForColumn( self.table_model.pubmedid_col, LinkStyledItemDelegate(self.table_view)) self.table_view.setItemDelegateForColumn( self.table_model.gds_id_col, LinkStyledItemDelegate(self.table_view)) self.table_view.setItemDelegateForColumn( self.table_model.indicator_col, IndicatorItemDelegate(self.table_view, role=Qt.DisplayRole), ) splitter_horizontal = QSplitter(Qt.Horizontal, splitter_vertical) # Description Widget box = widgetBox(splitter_horizontal, 'Description') self.description_widget = widgetLabel(box, '') self.description_widget.setWordWrap(True) rubber(box) # Sample Annotations Widget box = widgetBox(splitter_horizontal, 'Sample Annotations') self.annotations_widget = QTreeWidget(box) self.annotations_widget.setHeaderLabels( ['Type (Sample annotations)', 'Sample count']) self.annotations_widget.setRootIsDecorated(True) box.layout().addWidget(self.annotations_widget) self._annotations_updating = False self.annotations_widget.itemChanged.connect( self.on_annotation_selection_changed) self.splitters = splitter_vertical, splitter_horizontal for sp, setting in zip(self.splitters, self.splitter_settings): sp.splitterMoved.connect(self._splitter_moved) sp.restoreState(setting) self.table_view.selectionModel().selectionChanged.connect( self.on_gds_selection_changed) self._apply_filter() self._run() def _splitter_moved(self, *args): self.splitter_settings = [ bytes(sp.saveState()) for sp in self.splitters ] def _set_description_widget(self): self.description_widget.setText( self.selected_gds.get('description', 'Description not available.')) def _set_annotations_widget(self, gds): self._annotations_updating = True self.annotations_widget.clear() annotations = defaultdict(set) subsets_count = {} for desc in gds['subsets']: annotations[desc['type']].add(desc['description']) subsets_count[desc['description']] = str(len(desc['sample_id'])) for _type, subsets in annotations.items(): key = (gds["name"], _type) parent = QTreeWidgetItem(self.annotations_widget, [_type]) parent.key = key for subset in subsets: key = (gds['name'], _type, subset) item = QTreeWidgetItem( parent, [subset, subsets_count.get(subset, '')]) item.setFlags(item.flags() | Qt.ItemIsUserCheckable) item.setCheckState( 0, self.gdsSelectionStates.get(key, Qt.Checked)) item.key = key self._annotations_updating = False self.annotations_widget.expandAll() for i in range(self.annotations_widget.columnCount()): self.annotations_widget.resizeColumnToContents(i) def _set_selection(self): if self.selected_gds is not None: index = self.table_model.get_row_index( self.selected_gds.get('name')) if index is not None: self.table_view.selectionModel().blockSignals(True) self.table_view.selectRow(index) self._handle_selection_changed() self.table_view.selectionModel().blockSignals(False) def _handle_selection_changed(self): if self.table_model.table is not None: selection = self.table_view.selectionModel().selectedRows( self.table_model.gds_id_col) selected_gds_name = selection[0].data( ) if len(selection) > 0 else None if selected_gds_name: self.selected_gds = self.table_model.info.get( selected_gds_name) self._set_annotations_widget(self.selected_gds) self._set_description_widget() else: self.annotations_widget.clear() self.description_widget.clear() self.update_info() def _apply_filter(self): if self.table_model.table is not None: self.table_model.show_table( filter_pattern=str(self.search_pattern)) self._set_selection() self.update_info() def _run(self): if self.selected_gds is not None: self.gds_data = None self.start(run_download_task, self.selected_gds.get('name'), self.get_selected_samples(), self.genes_as_rows) def on_gds_selection_changed(self): self._handle_selection_changed() self._run() def on_annotation_selection_changed(self): if self._annotations_updating: return for i in range(self.annotations_widget.topLevelItemCount()): item = self.annotations_widget.topLevelItem(i) if 'key' in item.__dict__: self.gdsSelectionStates[item.key] = item.checkState(0) for j in range(item.childCount()): child = item.child(j) if 'key' in child.__dict__: self.gdsSelectionStates[child.key] = child.checkState(0) self._run() def update_info(self): all_gds = len(self.table_model.info) text = "{} datasets\n{} datasets cached\n".format( all_gds, len(local_files.listfiles())) filtered = self.table_view.model().rowCount() if all_gds != filtered: text += "{} after filtering".format(filtered) self.infoBox.setText(text) def get_selected_samples(self): """ Return the currently selected sample annotations. The return value is a list of selected (sample type, sample value) tuples. .. note:: if some Sample annotation type has no selected values. this method will return all values for it. TODO: this could probably be simplified. """ def childiter(item): """ Iterate over the children of an QTreeWidgetItem instance. """ for i in range(item.childCount()): yield item.child(i) samples = [] unused_types = [] used_types = [] for stype in childiter(self.annotations_widget.invisibleRootItem()): selected_values = [] all_values = [] for sval in childiter(stype): value = (str(stype.text(0)), str(sval.text(0))) if self.gdsSelectionStates.get(sval.key, True): selected_values.append(value) all_values.append(value) if selected_values: samples.extend(selected_values) used_types.append(str(stype.text(0))) else: # If no sample of sample type is selected we don't filter on it. samples.extend(all_values) unused_types.append(str(stype.text(0))) _samples = defaultdict(list) for sample, sample_type in samples: _samples[sample].append(sample_type) return _samples def commit(self): self.Outputs.gds_data.send(self.gds_data) def on_done(self, result: Result): assert isinstance(result.gds_dataset, Table) self.gds_data = result.gds_dataset self.commit() if self.gds_info: self.table_model.initialize(self.gds_info) self._apply_filter() def on_partial_result(self, result: Any) -> None: pass def onDeleteWidget(self): self.shutdown() super().onDeleteWidget() def send_report(self): self.report_items( "GEO Dataset", [ ("ID", self.selected_gds['name']), ("Title", self.selected_gds['title']), ("Organism", self.selected_gds['sample_organism']), ], ) self.report_items( "Data", [ ("Samples", self.selected_gds['sample_count']), ("Features", self.selected_gds['variables']), ("Genes", self.selected_gds['genes']), ], ) self.report_name("Sample annotations") subsets = defaultdict(list) for subset in self.selected_gds['subsets']: subsets[subset['type']].append( (subset['description'], len(subset['sample_id']))) self.report_html += "<ul>" for _type in subsets: self.report_html += "<b>" + _type + ":</b></br>" for desc, count in subsets[_type]: self.report_html += 9 * " " + "<b>{}:</b> {}</br>".format( desc, count) self.report_html += "</ul>"