class OWPIPAx(widget.OWWidget): name = "PIPAx" description = "Access data from PIPA RNA-Seq database." icon = "../widgets/icons/PIPA.svg" priority = 35 inputs = [] outputs = [("Data", Orange.data.Table)] username = settings.Setting("") password = settings.Setting("") log2 = settings.Setting(False) rtypei = settings.Setting(5) # hardcoded rpkm mapability polya excludeconstant = settings.Setting(False) joinreplicates = settings.Setting(False) #: The stored current selection (in experiments view) #: SelectionByKey | None currentSelection = settings.Setting(None) #: Stored selections (presets) #: list of SelectionByKey storedSelections = settings.Setting([]) #: Stored column sort keys (from Sort view) #: list of strings storedSortingOrder = settings.Setting( ["Strain", "Experiment", "Genotype", "Timepoint"]) experimentsHeaderState = settings.Setting( {name: False for _, name in HEADER[:ID_INDEX + 1]} ) def __init__(self, parent=None, signalManager=None, name="PIPAx"): super().__init__(parent) self.selectedExperiments = [] self.buffer = dicty.CacheSQLite(bufferfile) self.searchString = "" self.result_types = [] self.mappings = {} self.controlArea.setMaximumWidth(250) self.controlArea.setMinimumWidth(250) gui.button(self.controlArea, self, "Reload", callback=self.Reload) gui.button(self.controlArea, self, "Clear cache", callback=self.clear_cache) b = gui.widgetBox(self.controlArea, "Experiment Sets") self.selectionSetsWidget = SelectionSetsWidget(self) self.selectionSetsWidget.setSizePolicy( QSizePolicy.Preferred, QSizePolicy.Maximum) def store_selections(modified): if not modified: self.storedSelections = self.selectionSetsWidget.selections self.selectionSetsWidget.selectionModified.connect(store_selections) b.layout().addWidget(self.selectionSetsWidget) gui.separator(self.controlArea) b = gui.widgetBox(self.controlArea, "Sort output columns") self.columnsSortingWidget = SortedListWidget(self) self.columnsSortingWidget.setSizePolicy( QSizePolicy.Preferred, QSizePolicy.Maximum) def store_sort_order(): self.storedSortingOrder = self.columnsSortingWidget.sortingOrder self.columnsSortingWidget.sortingOrderChanged.connect(store_sort_order) b.layout().addWidget(self.columnsSortingWidget) sorting_model = QStringListModel(SORTING_MODEL_LIST) self.columnsSortingWidget.setModel(sorting_model) gui.separator(self.controlArea) box = gui.widgetBox(self.controlArea, 'Expression Type') self.expressionTypesCB = gui.comboBox( box, self, "rtypei", items=[], callback=self.UpdateResultsList) gui.checkBox(self.controlArea, self, "excludeconstant", "Exclude labels with constant values") gui.checkBox(self.controlArea, self, "joinreplicates", "Average replicates (use median)") gui.checkBox(self.controlArea, self, "log2", "Logarithmic (base 2) transformation") self.commit_button = gui.button(self.controlArea, self, "&Commit", callback=self.Commit) self.commit_button.setDisabled(True) gui.rubber(self.controlArea) box = gui.widgetBox(self.controlArea, "Authentication") gui.lineEdit(box, self, "username", "Username:"******"password", "Password:"******"searchString", "Search", callbackOnType=True, callback=self.SearchUpdate) self.headerLabels = [t[1] for t in HEADER] self.experimentsWidget = QTreeWidget() self.experimentsWidget.setHeaderLabels(self.headerLabels) self.experimentsWidget.setSelectionMode(QTreeWidget.ExtendedSelection) self.experimentsWidget.setRootIsDecorated(False) self.experimentsWidget.setSortingEnabled(True) contextEventFilter = gui.VisibleHeaderSectionContextEventFilter( self.experimentsWidget, self.experimentsWidget ) self.experimentsWidget.header().installEventFilter(contextEventFilter) self.experimentsWidget.setItemDelegateForColumn( 0, gui.IndicatorItemDelegate(self, role=Qt.DisplayRole)) self.experimentsWidget.setAlternatingRowColors(True) self.experimentsWidget.selectionModel().selectionChanged.connect( self.onSelectionChanged) self.selectionSetsWidget.setSelectionModel( self.experimentsWidget.selectionModel() ) self.mainArea.layout().addWidget(self.experimentsWidget) # Restore the selection states from the stored settings self.selectionSetsWidget.selections = self.storedSelections self.columnsSortingWidget.sortingOrder = self.storedSortingOrder self.restoreHeaderState() self.experimentsWidget.header().geometriesChanged.connect( self.saveHeaderState) self.dbc = None self.AuthSet() QTimer.singleShot(100, self.UpdateExperiments) def sizeHint(self): return QSize(800, 600) def AuthSet(self): if len(self.username): self.passf.setDisabled(False) else: self.passf.setDisabled(True) def AuthChanged(self): self.AuthSet() self.ConnectAndUpdate() def ConnectAndUpdate(self): self.Connect() self.UpdateExperiments(reload=True) def Connect(self): self.error(1) self.warning(1) def en(x): return x if len(x) else None self.dbc = dicty.PIPAx(cache=self.buffer, username=en(self.username), password=self.password) # check password if en(self.username) != None: try: self.dbc.mappings(reload=True) except dicty.AuthenticationError: self.error(1, "Wrong username or password") self.dbc = None except Exception as ex: print("Error when contacting the PIPA database", ex) sys.excepthook(*sys.exc_info()) try: # maybe cached? self.dbc.mappings() self.warning(1, "Can not access database - using cached data.") except Exception as ex: self.dbc = None self.error(1, "Can not access database.") def Reload(self): self.UpdateExperiments(reload=True) def clear_cache(self): self.buffer.clear() self.Reload() def rtype(self): """Return selected result template type """ if self.result_types: return self.result_types[self.rtypei][0] else: return "-1" def UpdateExperimentTypes(self): self.expressionTypesCB.clear() items = [desc for _, desc in self.result_types] self.expressionTypesCB.addItems(items) self.rtypei = max(0, min(self.rtypei, len(self.result_types) - 1)) def UpdateExperiments(self, reload=False): self.experimentsWidget.clear() self.items = [] self.progressBarInit() if not self.dbc: self.Connect() mappings = {} result_types = [] sucind = False # success indicator for database index try: mappings = self.dbc.mappings(reload=reload) result_types = self.dbc.result_types(reload=reload) sucind = True except Exception as ex: try: mappings = self.dbc.mappings() result_types = self.dbc.result_types() self.warning(0, "Can not access database - using cached data.") sucind = True except Exception as ex: self.error(0, "Can not access database.") if sucind: self.warning(0) self.error(0) self.mappings = mappings self.result_types = result_types self.UpdateExperimentTypes() self.UpdateResultsList(reload=reload) self.progressBarFinished() if self.currentSelection: self.currentSelection.select( self.experimentsWidget.selectionModel()) self.handle_commit_button() def UpdateResultsList(self, reload=False): results_list = {} try: results_list = self.dbc.results_list(self.rtype(), reload=reload) except Exception as ex: try: results_list = self.dbc.results_list(self.rtype()) except Exception as ex: self.error(0, "Can not access database.") self.results_list = results_list mappings_key_dict = dict(((m["data_id"], m["id"]), key) \ for key, m in self.mappings.items()) def mapping_unique_id(annot): """Map annotations dict from results_list to unique `mappings` ids. """ data_id, mappings_id = annot["data_id"], annot["mappings_id"] return mappings_key_dict[data_id, mappings_id] elements = [] # softly change the view so that the selection stays the same items_shown = {} for i, item in enumerate(self.items): c = str(item.text(10)) items_shown[c] = i items_to_show = dict((mapping_unique_id(annot), annot) for annot in self.results_list.values()) add_items = set(items_to_show) - set(items_shown) delete_items = set(items_shown) - set(items_to_show) i = 0 while i < self.experimentsWidget.topLevelItemCount(): it = self.experimentsWidget.topLevelItem(i) if str(it.text(10)) in delete_items: self.experimentsWidget.takeTopLevelItem(i) else: i += 1 delete_ind = set([items_shown[i] for i in delete_items]) self.items = [it for i, it in enumerate(self.items) if i not in delete_ind] for r_annot in [items_to_show[i] for i in add_items]: d = defaultdict(lambda: "?", r_annot) row_items = [""] + [d.get(key, "?") for key, _ in HEADER[1:]] try: time_dict = literal_eval(row_items[DATE_INDEX]) date_rna = date(time_dict["fullYearUTC"], time_dict["monthUTC"] + 1, # Why is month 0 based? time_dict["dateUTC"]) row_items[DATE_INDEX] = date_rna.strftime("%x") except Exception: row_items[DATE_INDEX] = '' row_items[ID_INDEX] = mapping_unique_id(r_annot) elements.append(row_items) ci = MyTreeWidgetItem(self.experimentsWidget, row_items) self.items.append(ci) for i in range(len(self.headerLabels)): self.experimentsWidget.resizeColumnToContents(i) # which is the ok buffer version # FIXME: what attribute to use for version? self.wantbufver = \ lambda x, ad=self.results_list: \ defaultdict(lambda: "?", ad[x])["date"] self.wantbufver = lambda x: "0" self.UpdateCached() def UpdateCached(self): if self.wantbufver and self.dbc: fn = self.dbc.download_key_function() result_id_key = dict(((m["data_id"], m["mappings_id"]), key) \ for key, m in self.results_list.items()) for item in self.items: c = str(item.text(10)) mapping = self.mappings[c] data_id, mappings_id = mapping["data_id"], mapping["id"] r_id = result_id_key[data_id, mappings_id] # Get the buffered version buffered = self.dbc.inBuffer(fn(r_id)) value = " " if buffered == self.wantbufver(r_id) else "" item.setData(0, Qt.DisplayRole, value) def SearchUpdate(self, string=""): for item in self.items: item.setHidden(not all(s in item \ for s in self.searchString.split()) ) def Commit(self): if not self.dbc: self.Connect() pb = gui.ProgressBar(self, iterations=100) table = None ids = [] for item in self.experimentsWidget.selectedItems(): unique_id = str(item.text(10)) annots = self.mappings[unique_id] ids.append((annots["data_id"], annots["id"])) transfn = None if self.log2: transfn = lambda x: math.log(x + 1.0, 2) reverse_header_dict = dict((name, key) for key, name in HEADER) hview = self.experimentsWidget.header() shownHeaders = [label for i, label in \ list(enumerate(self.headerLabels))[1:] \ if not hview.isSectionHidden(i) ] allowed_labels = [reverse_header_dict.get(label, label) \ for label in shownHeaders] if self.joinreplicates and "id" not in allowed_labels: # need 'id' labels in join_replicates for attribute names allowed_labels.append("id") if len(ids): table = self.dbc.get_data(ids=ids, result_type=self.rtype(), callback=pb.advance, exclude_constant_labels=self.excludeconstant, # bufver=self.wantbufver, transform=transfn, allowed_labels=allowed_labels) if self.joinreplicates: table = dicty.join_replicates(table, ignorenames=["replicate", "data_id", "mappings_id", "data_name", "id", "unique_id"], namefn=None, avg=dicty.median ) # Sort attributes sortOrder = self.columnsSortingWidget.sortingOrder all_values = defaultdict(set) for at in table.domain.attributes: atts = at.attributes for name in sortOrder: all_values[name].add(atts.get(reverse_header_dict[name], "")) isnum = {} for at, vals in all_values.items(): vals = filter(None, vals) try: for a in vals: float(a) isnum[at] = True except: isnum[at] = False def optfloat(x, at): if x == "": return "" else: return float(x) if isnum[at] else x def sorting_key(attr): atts = attr.attributes return tuple([optfloat(atts.get(reverse_header_dict[name], ""), name) \ for name in sortOrder]) attributes = sorted(table.domain.attributes, key=sorting_key) domain = Orange.data.Domain( attributes, table.domain.class_var, table.domain.metas) table = table.from_table(domain, table) data_hints.set_hint(table, "taxid", "352472") data_hints.set_hint(table, "genesinrows", False) self.send("Data", table) self.UpdateCached() pb.finish() def onSelectionChanged(self, selected, deselected): self.handle_commit_button() def handle_commit_button(self): self.currentSelection = \ SelectionByKey(self.experimentsWidget.selectionModel().selection(), key=(1, 2, 3, 10)) self.commit_button.setDisabled(not len(self.currentSelection)) def saveHeaderState(self): hview = self.experimentsWidget.header() for i, label in enumerate(self.headerLabels): self.experimentsHeaderState[label] = hview.isSectionHidden(i) def restoreHeaderState(self): hview = self.experimentsWidget.header() state = self.experimentsHeaderState for i, label in enumerate(self.headerLabels): hview.setSectionHidden(i, state.get(label, True)) self.experimentsWidget.resizeColumnToContents(i)
class OWDatabasesUpdate(OWWidget): name = "Databases Update" description = "Update local systems biology databases." icon = "../widgets/icons/Databases.svg" priority = 10 inputs = [] outputs = [] want_main_area = False def __init__(self, parent=None, signalManager=None, name="Databases update"): OWWidget.__init__(self, parent, signalManager, name, wantMainArea=False) self.searchString = "" fbox = gui.widgetBox(self.controlArea, "Filter") self.completer = TokenListCompleter( self, caseSensitivity=Qt.CaseInsensitive) self.lineEditFilter = QLineEdit(textChanged=self.SearchUpdate) self.lineEditFilter.setCompleter(self.completer) fbox.layout().addWidget(self.lineEditFilter) box = gui.widgetBox(self.controlArea, "Files") self.filesView = QTreeWidget(self) self.filesView.setHeaderLabels( ["", "Data Source", "Update", "Last Updated", "Size"]) self.filesView.setRootIsDecorated(False) self.filesView.setUniformRowHeights(True) self.filesView.setSelectionMode(QAbstractItemView.NoSelection) self.filesView.setSortingEnabled(True) self.filesView.sortItems(1, Qt.AscendingOrder) self.filesView.setItemDelegateForColumn( 0, UpdateOptionsItemDelegate(self.filesView)) self.filesView.model().layoutChanged.connect(self.SearchUpdate) box.layout().addWidget(self.filesView) box = gui.widgetBox(self.controlArea, orientation="horizontal") self.updateButton = gui.button( box, self, "Update all", callback=self.UpdateAll, tooltip="Update all updatable files", ) self.downloadButton = gui.button( box, self, "Download all", callback=self.DownloadFiltered, tooltip="Download all filtered files shown" ) self.cancelButton = gui.button( box, self, "Cancel", callback=self.Cancel, tooltip="Cancel scheduled downloads/updates." ) self.retryButton = gui.button( box, self, "Reconnect", callback=self.RetrieveFilesList ) self.retryButton.hide() gui.rubber(box) self.warning(0) box = gui.widgetBox(self.controlArea, orientation="horizontal") gui.rubber(box) self.infoLabel = QLabel() self.infoLabel.setAlignment(Qt.AlignCenter) self.controlArea.layout().addWidget(self.infoLabel) self.infoLabel.setSizePolicy(QSizePolicy.Expanding, QSizePolicy.Fixed) self.updateItems = [] self.resize(800, 600) self.progress = ProgressState(self, maximum=3) self.progress.valueChanged.connect(self._updateProgress) self.progress.rangeChanged.connect(self._updateProgress) self.executor = ThreadExecutor( threadPool=QThreadPool(maxThreadCount=2) ) task = Task(self, function=self.RetrieveFilesList) task.exceptionReady.connect(self.HandleError) task.start() self._tasks = [] self._haveProgress = False def RetrieveFilesList(self): self.retryButton.hide() self.warning(0) self.progress.setRange(0, 3) task = Task(function=partial(retrieveFilesList, methodinvoke(self.progress, "advance"))) task.resultReady.connect(self.SetFilesList) task.exceptionReady.connect(self.HandleError) self.executor.submit(task) self.setEnabled(False) def SetFilesList(self, serverInfo): """ Set the files to show. """ self.setEnabled(True) localInfo = serverfiles.allinfo() all_tags = set() self.filesView.clear() self.updateItems = [] for item in join_info_dict(localInfo, serverInfo): tree_item = UpdateTreeWidgetItem(item) options_widget = UpdateOptionsWidget(item.state) options_widget.item = item options_widget.installClicked.connect( partial(self.SubmitDownloadTask, item.domain, item.filename) ) options_widget.removeClicked.connect( partial(self.SubmitRemoveTask, item.domain, item.filename) ) self.updateItems.append((item, tree_item, options_widget)) all_tags.update(item.tags) self.filesView.addTopLevelItems( [tree_item for _, tree_item, _ in self.updateItems] ) for item, tree_item, options_widget in self.updateItems: self.filesView.setItemWidget(tree_item, 0, options_widget) # Add an update button if the file is updateable if item.state == OUTDATED: button = QToolButton( None, text="Update", maximumWidth=120, minimumHeight=20, maximumHeight=20 ) if sys.platform == "darwin": button.setAttribute(Qt.WA_MacSmallSize) button.clicked.connect( partial(self.SubmitDownloadTask, item.domain, item.filename) ) self.filesView.setItemWidget(tree_item, 2, button) self.progress.advance() self.filesView.setColumnWidth(0, self.filesView.sizeHintForColumn(0)) for column in range(1, 4): contents_hint = self.filesView.sizeHintForColumn(column) header_hint = self.filesView.header().sectionSizeHint(column) width = max(min(contents_hint, 400), header_hint) self.filesView.setColumnWidth(column, width) hints = [hint for hint in sorted(all_tags) if not hint.startswith("#")] self.completer.setTokenList(hints) self.SearchUpdate() self.UpdateInfoLabel() self.toggleButtons() self.cancelButton.setEnabled(False) self.progress.setRange(0, 0) def buttonCheck(self, selected_items, state, button): for item in selected_items: if item.state != state: button.setEnabled(False) else: button.setEnabled(True) break def toggleButtons(self): selected_items = [item for item, tree_item, _ in self.updateItems if not tree_item.isHidden()] self.buttonCheck(selected_items, OUTDATED, self.updateButton) self.buttonCheck(selected_items, AVAILABLE, self.downloadButton) def HandleError(self, exception): if isinstance(exception, ConnectionError): self.warning(0, "Could not connect to server! Check your connection " "and try to reconnect.") self.SetFilesList({}) self.retryButton.show() else: sys.excepthook(type(exception), exception, None) self.progress.setRange(0, 0) self.setEnabled(True) def UpdateInfoLabel(self): local = [item for item, tree_item, _ in self.updateItems if item.state != AVAILABLE and not tree_item.isHidden()] size = sum(float(item.size) for item in local) onServer = [item for item, tree_item, _ in self.updateItems if not tree_item.isHidden()] sizeOnServer = sum(float(item.size) for item in onServer) text = ("%i items, %s (on server: %i items, %s)" % (len(local), sizeof_fmt(size), len(onServer), sizeof_fmt(sizeOnServer))) self.infoLabel.setText(text) def UpdateAll(self): self.warning(0) for item, tree_item, _ in self.updateItems: if item.state == OUTDATED and not tree_item.isHidden(): self.SubmitDownloadTask(item.domain, item.filename) def DownloadFiltered(self): # TODO: submit items in the order shown. for item, tree_item, _ in self.updateItems: if not tree_item.isHidden() and item.state in \ [AVAILABLE, OUTDATED]: self.SubmitDownloadTask(item.domain, item.filename) def SearchUpdate(self, searchString=None): strings = str(self.lineEditFilter.text()).split() for item, tree_item, _ in self.updateItems: hide = not all(UpdateItem_match(item, string) for string in strings) tree_item.setHidden(hide) self.UpdateInfoLabel() self.toggleButtons() def SubmitDownloadTask(self, domain, filename): """ Submit the (domain, filename) to be downloaded/updated. """ self.cancelButton.setEnabled(True) index = self.updateItemIndex(domain, filename) _, tree_item, opt_widget = self.updateItems[index] sf = LocalFiles(serverfiles.PATH, serverfiles.ServerFiles()) task = DownloadTask(domain, filename, sf) self.progress.adjustRange(0, 100) pb = ItemProgressBar(self.filesView) pb.setRange(0, 100) pb.setTextVisible(False) task.advanced.connect(pb.advance) task.advanced.connect(self.progress.advance) task.finished.connect(pb.hide) task.finished.connect(self.onDownloadFinished, Qt.QueuedConnection) task.exception.connect(self.onDownloadError, Qt.QueuedConnection) self.filesView.setItemWidget(tree_item, 2, pb) # Clear the text so it does not show behind the progress bar. tree_item.setData(2, Qt.DisplayRole, "") pb.show() # Disable the options widget opt_widget.setEnabled(False) self._tasks.append(task) self.executor.submit(task) def EndDownloadTask(self, task): future = task.future() index = self.updateItemIndex(task.domain, task.filename) item, tree_item, opt_widget = self.updateItems[index] self.filesView.removeItemWidget(tree_item, 2) opt_widget.setEnabled(True) if future.cancelled(): # Restore the previous state tree_item.setUpdateItem(item) opt_widget.setState(item.state) elif future.exception(): tree_item.setUpdateItem(item) opt_widget.setState(item.state) # Show the exception string in the size column. self.warning(0, "Error while downloading. Check your connection " "and retry.") # recreate button for download button = QToolButton( None, text="Retry", maximumWidth=120, minimumHeight=20, maximumHeight=20 ) if sys.platform == "darwin": button.setAttribute(Qt.WA_MacSmallSize) button.clicked.connect( partial(self.SubmitDownloadTask, item.domain, item.filename) ) self.filesView.setItemWidget(tree_item, 2, button) else: # get the new updated info dict and replace the the old item self.warning(0) info = serverfiles.info(item.domain, item.filename) new_item = update_item_from_info(item.domain, item.filename, info, info) self.updateItems[index] = (new_item, tree_item, opt_widget) tree_item.setUpdateItem(new_item) opt_widget.setState(new_item.state) self.UpdateInfoLabel() def SubmitRemoveTask(self, domain, filename): serverfiles.LOCALFILES.remove(domain, filename) index = self.updateItemIndex(domain, filename) item, tree_item, opt_widget = self.updateItems[index] if item.info_server: new_item = item._replace(state=AVAILABLE, local=None, info_local=None) else: new_item = item._replace(local=None, info_local=None) # Disable the options widget. No more actions can be performed # for the item. opt_widget.setEnabled(False) tree_item.setUpdateItem(new_item) opt_widget.setState(new_item.state) self.updateItems[index] = (new_item, tree_item, opt_widget) self.UpdateInfoLabel() def Cancel(self): """ Cancel all pending update/download tasks (that have not yet started). """ for task in self._tasks: task.future().cancel() def onDeleteWidget(self): self.Cancel() self.executor.shutdown(wait=False) OWWidget.onDeleteWidget(self) def onDownloadFinished(self): # on download completed/canceled/error assert QThread.currentThread() is self.thread() for task in list(self._tasks): future = task.future() if future.done(): self.EndDownloadTask(task) self._tasks.remove(task) if not self._tasks: # Clear/reset the overall progress self.progress.setRange(0, 0) self.cancelButton.setEnabled(False) def onDownloadError(self, exc_info): sys.excepthook(*exc_info) self.warning(0, "Error while downloading. Check your connection and " "retry.") def updateItemIndex(self, domain, filename): for i, (item, _, _) in enumerate(self.updateItems): if item.domain == domain and item.filename == filename: return i raise ValueError("%r, %r not in update list" % (domain, filename)) def _updateProgress(self, *args): rmin, rmax = self.progress.range() if rmin != rmax: if not self._haveProgress: self._haveProgress = True self.progressBarInit() self.progressBarSet(self.progress.ratioCompleted() * 100, processEvents=None) if rmin == rmax: self._haveProgress = False self.progressBarFinished()
class OWGEODatasets(OWWidget): name = "GEO Data Sets" description = DESCRIPTION icon = "../widgets/icons/GEODataSets.svg" priority = PRIORITY inputs = [] outputs = [("Expression Data", Orange.data.Table)] settingsList = [ "outputRows", "mergeSpots", "gdsSelectionStates", "splitterSettings", "currentGds", "autoCommit", "datasetNames" ] outputRows = Setting(True) mergeSpots = Setting(True) gdsSelectionStates = Setting({}) currentGds = Setting(None) datasetNames = Setting({}) splitterSettings = Setting(( b'\x00\x00\x00\xff\x00\x00\x00\x00\x00\x00\x00\x02\x00\x00\x01\xea\x00\x00\x00\xd7\x01\x00\x00\x00\x07\x01\x00\x00\x00\x02', b'\x00\x00\x00\xff\x00\x00\x00\x00\x00\x00\x00\x02\x00\x00\x01\xb5\x00\x00\x02\x10\x01\x00\x00\x00\x07\x01\x00\x00\x00\x01' )) autoCommit = Setting(False) def __init__(self, parent=None, signalManager=None, name=" GEO Data Sets"): OWWidget.__init__(self, parent, signalManager, name) self.selectionChanged = False self.filterString = "" self.datasetName = "" ## GUI box = gui.widgetBox(self.controlArea, "Info", addSpace=True) self.infoBox = gui.widgetLabel(box, "Initializing\n\n") box = gui.widgetBox(self.controlArea, "Output", addSpace=True) gui.radioButtonsInBox(box, self, "outputRows", ["Genes in rows", "Samples in rows"], "Rows", callback=self.commitIf) gui.checkBox(box, self, "mergeSpots", "Merge spots of same gene", callback=self.commitIf) gui.separator(box) self.nameEdit = gui.lineEdit( box, self, "datasetName", "Data set name", tooltip="Override the default output data set name", callback=self.onNameEdited) self.nameEdit.setPlaceholderText("") if sys.version_info < (3, ): box = gui.widgetBox(self.controlArea, "Commit", addSpace=True) self.commitButton = gui.button(box, self, "Commit", callback=self.commit) cb = gui.checkBox(box, self, "autoCommit", "Commit on any change") gui.setStopper(self, self.commitButton, cb, "selectionChanged", self.commit) else: gui.auto_commit(self.controlArea, self, "autoCommit", "Commit", box="Commit") self.commitIf = self.commit gui.rubber(self.controlArea) gui.widgetLabel(self.mainArea, "Filter") self.filterLineEdit = QLineEdit(textChanged=self.filter) self.completer = TokenListCompleter(self, caseSensitivity=Qt.CaseInsensitive) self.filterLineEdit.setCompleter(self.completer) self.mainArea.layout().addWidget(self.filterLineEdit) splitter = QSplitter(Qt.Vertical, self.mainArea) self.mainArea.layout().addWidget(splitter) self.treeWidget = QTreeView(splitter) self.treeWidget.setSelectionMode(QTreeView.SingleSelection) self.treeWidget.setRootIsDecorated(False) self.treeWidget.setSortingEnabled(True) self.treeWidget.setAlternatingRowColors(True) self.treeWidget.setUniformRowHeights(True) self.treeWidget.setEditTriggers(QTreeView.NoEditTriggers) linkdelegate = LinkStyledItemDelegate(self.treeWidget) self.treeWidget.setItemDelegateForColumn(1, linkdelegate) self.treeWidget.setItemDelegateForColumn(8, linkdelegate) self.treeWidget.setItemDelegateForColumn( 0, gui.IndicatorItemDelegate(self.treeWidget, role=Qt.DisplayRole)) proxyModel = MySortFilterProxyModel(self.treeWidget) self.treeWidget.setModel(proxyModel) self.treeWidget.selectionModel().selectionChanged.connect( self.updateSelection) self.treeWidget.viewport().setMouseTracking(True) splitterH = QSplitter(Qt.Horizontal, splitter) box = gui.widgetBox(splitterH, "Description") self.infoGDS = gui.widgetLabel(box, "") self.infoGDS.setWordWrap(True) gui.rubber(box) box = gui.widgetBox(splitterH, "Sample Annotations") self.annotationsTree = QTreeWidget(box) self.annotationsTree.setHeaderLabels( ["Type (Sample annotations)", "Sample count"]) self.annotationsTree.setRootIsDecorated(True) box.layout().addWidget(self.annotationsTree) self.annotationsTree.itemChanged.connect( self.annotationSelectionChanged) self._annotationsUpdating = False self.splitters = splitter, splitterH for sp, setting in zip(self.splitters, self.splitterSettings): sp.splitterMoved.connect(self.splitterMoved) sp.restoreState(setting) self.searchKeys = [ "dataset_id", "title", "platform_organism", "description" ] self.gds = [] self.gds_info = None self.resize(1000, 600) self.setBlocking(True) self.setEnabled(False) self.progressBarInit() self._executor = ThreadExecutor() func = partial(get_gds_model, methodinvoke(self, "_setProgress", (float, ))) self._inittask = Task(function=func) self._inittask.finished.connect(self._initializemodel) self._executor.submit(self._inittask) self._datatask = None @Slot(float) def _setProgress(self, value): self.progressBarValue = value def _initializemodel(self): assert self.thread() is QThread.currentThread() model, self.gds_info, self.gds = self._inittask.result() model.setParent(self) proxy = self.treeWidget.model() proxy.setFilterKeyColumn(0) proxy.setFilterRole(TextFilterRole) proxy.setFilterCaseSensitivity(False) proxy.setFilterFixedString(self.filterString) proxy.setSourceModel(model) proxy.sort(0, Qt.DescendingOrder) self.progressBarFinished() self.setBlocking(False) self.setEnabled(True) filter_items = " ".join(gds[key] for gds in self.gds for key in self.searchKeys) tr_chars = ",.:;!?(){}[]_-+\\|/%#@$^&*<>~`" tr_table = str.maketrans(tr_chars, " " * len(tr_chars)) filter_items = filter_items.translate(tr_table) filter_items = sorted(set(filter_items.split(" "))) filter_items = [item for item in filter_items if len(item) > 3] self.completer.setTokenList(filter_items) if self.currentGds: current_id = self.currentGds["dataset_id"] gdss = [(i, qunpack(proxy.data(proxy.index(i, 1), Qt.DisplayRole))) for i in range(proxy.rowCount())] current = [i for i, data in gdss if data and data == current_id] if current: current_index = proxy.index(current[0], 0) self.treeWidget.selectionModel().select( current_index, QItemSelectionModel.Select | QItemSelectionModel.Rows) self.treeWidget.scrollTo(current_index, QTreeView.PositionAtCenter) for i in range(8): self.treeWidget.resizeColumnToContents(i) self.treeWidget.setColumnWidth( 1, min(self.treeWidget.columnWidth(1), 300)) self.treeWidget.setColumnWidth( 2, min(self.treeWidget.columnWidth(2), 200)) self.updateInfo() def updateInfo(self): gds_info = self.gds_info text = ("%i datasets\n%i datasets cached\n" % (len(gds_info), len(glob.glob(serverfiles.localpath("GEO") + "/GDS*")))) filtered = self.treeWidget.model().rowCount() if len(self.gds) != filtered: text += ("%i after filtering") % filtered self.infoBox.setText(text) def updateSelection(self, *args): current = self.treeWidget.selectedIndexes() mapToSource = self.treeWidget.model().mapToSource current = [mapToSource(index).row() for index in current] if current: self.currentGds = self.gds[current[0]] self.setAnnotations(self.currentGds) self.infoGDS.setText(self.currentGds.get("description", "")) self.nameEdit.setPlaceholderText(self.currentGds["title"]) self.datasetName = \ self.datasetNames.get(self.currentGds["dataset_id"], "") else: self.currentGds = None self.nameEdit.setPlaceholderText("") self.datasetName = "" self.commitIf() def setAnnotations(self, gds): self._annotationsUpdating = True self.annotationsTree.clear() annotations = defaultdict(set) subsetscount = {} for desc in gds["subsets"]: annotations[desc["type"]].add(desc["description"]) subsetscount[desc["description"]] = str(len(desc["sample_id"])) for type, subsets in annotations.items(): key = (gds["dataset_id"], type) subsetItem = QTreeWidgetItem(self.annotationsTree, [type]) subsetItem.setFlags(subsetItem.flags() | Qt.ItemIsUserCheckable | Qt.ItemIsTristate) subsetItem.setCheckState( 0, self.gdsSelectionStates.get(key, Qt.Checked)) subsetItem.key = key for subset in subsets: key = (gds["dataset_id"], type, subset) item = QTreeWidgetItem( subsetItem, [subset, subsetscount.get(subset, "")]) item.setFlags(item.flags() | Qt.ItemIsUserCheckable) item.setCheckState( 0, self.gdsSelectionStates.get(key, Qt.Checked)) item.key = key self._annotationsUpdating = False self.annotationsTree.expandAll() for i in range(self.annotationsTree.columnCount()): self.annotationsTree.resizeColumnToContents(i) def annotationSelectionChanged(self, item, column): if self._annotationsUpdating: return for i in range(self.annotationsTree.topLevelItemCount()): item = self.annotationsTree.topLevelItem(i) self.gdsSelectionStates[item.key] = item.checkState(0) for j in range(item.childCount()): child = item.child(j) self.gdsSelectionStates[child.key] = child.checkState(0) def filter(self): filter_string = unicode(self.filterLineEdit.text()) proxyModel = self.treeWidget.model() if proxyModel: strings = filter_string.lower().strip().split() proxyModel.setFilterFixedStrings(strings) self.updateInfo() def selectedSamples(self): """ Return the currently selected sample annotations. The return value is a list of selected (sample type, sample value) tuples. .. note:: if some Sample annotation type has no selected values. this method will return all values for it. """ samples = [] unused_types = [] used_types = [] for stype in childiter(self.annotationsTree.invisibleRootItem()): selected_values = [] all_values = [] for sval in childiter(stype): value = (str(stype.text(0)), str(sval.text(0))) if self.gdsSelectionStates.get(sval.key, True): selected_values.append(value) all_values.append(value) if selected_values: samples.extend(selected_values) used_types.append(str(stype.text(0))) else: # If no sample of sample type is selected we don't filter # on it. samples.extend(all_values) unused_types.append(str(stype.text(0))) return samples, used_types def commitIf(self): if self.autoCommit: self.commit() else: self.selectionChanged = True @Slot(int, int) def progressCompleted(self, value, total): if total > 0: self.progressBarSet(100. * value / total, processEvents=False) else: pass # TODO: report 'indeterminate progress' def commit(self): if self.currentGds: self.error(0) sample_type = None self.progressBarInit(processEvents=None) _, groups = self.selectedSamples() if len(groups) == 1 and self.outputRows: sample_type = groups[0] self.setEnabled(False) self.setBlocking(True) progress = methodinvoke(self, "progressCompleted", (int, int)) def get_data(gds_id, report_genes, transpose, sample_type, title): gds_ensure_downloaded(gds_id, progress) gds = geo.GDS(gds_id) data = gds.getdata(report_genes=report_genes, transpose=transpose, sample_type=sample_type) data.name = title return data get_data = partial(get_data, self.currentGds["dataset_id"], report_genes=self.mergeSpots, transpose=self.outputRows, sample_type=sample_type, title=self.datasetName or self.currentGds["title"]) self._datatask = Task(function=get_data) self._datatask.finished.connect(self._on_dataready) self._executor.submit(self._datatask) def _on_dataready(self): self.setEnabled(True) self.setBlocking(False) self.progressBarFinished(processEvents=False) try: data = self._datatask.result() except urlrequest.URLError as error: self.error(0, ("Error while connecting to the NCBI ftp server! " "'%s'" % error)) sys.excepthook(type(error), error, getattr(error, "__traceback__")) return finally: self._datatask = None data_name = data.name samples, _ = self.selectedSamples() self.warning(0) message = None if self.outputRows: def samplesinst(ex): out = [] for meta in data.domain.metas: out.append((meta.name, ex[meta].value)) if data.domain.class_var.name != 'class': out.append((data.domain.class_var.name, ex[data.domain.class_var].value)) return out samples = set(samples) mask = [samples.issuperset(samplesinst(ex)) for ex in data] data = data[numpy.array(mask, dtype=bool)] if len(data) == 0: message = "No samples with selected sample annotations." else: samples = set(samples) domain = Orange.data.Domain([ attr for attr in data.domain.attributes if samples.issuperset(attr.attributes.items()) ], data.domain.class_var, data.domain.metas) # domain.addmetas(data.domain.getmetas()) if len(domain.attributes) == 0: message = "No samples with selected sample annotations." stypes = set(s[0] for s in samples) for attr in domain.attributes: attr.attributes = dict( (key, value) for key, value in attr.attributes.items() if key in stypes) data = Orange.data.Table(domain, data) if message is not None: self.warning(0, message) data_hints.set_hint(data, "taxid", self.currentGds.get("taxid", ""), 10.0) data_hints.set_hint(data, "genesinrows", self.outputRows, 10.0) data.name = data_name self.send("Expression Data", data) model = self.treeWidget.model().sourceModel() row = self.gds.index(self.currentGds) model.setData(model.index(row, 0), " ", Qt.DisplayRole) self.updateInfo() self.selectionChanged = False def splitterMoved(self, *args): self.splitterSettings = [ bytes(sp.saveState()) for sp in self.splitters ] def send_report(self): self.report_items("GEO Dataset", [("ID", self.currentGds['dataset_id']), ("Title", self.currentGds['title']), ("Organism", self.currentGds['sample_organism'])]) self.report_items("Data", [("Samples", self.currentGds['sample_count']), ("Features", self.currentGds['feature_count']), ("Genes", self.currentGds['gene_count'])]) self.report_name("Sample annotations") subsets = defaultdict(list) for subset in self.currentGds['subsets']: subsets[subset['type']].append( (subset['description'], len(subset['sample_id']))) self.report_html += "<ul>" for type in subsets: self.report_html += "<b>" + type + ":</b></br>" for desc, count in subsets[type]: self.report_html += 9 * " " + "<b>{}:</b> {}</br>".format( desc, count) self.report_html += "</ul>" def onDeleteWidget(self): if self._inittask: self._inittask.future().cancel() self._inittask.finished.disconnect(self._initializemodel) if self._datatask: self._datatask.future().cancel() self._datatask.finished.disconnect(self._on_dataready) self._executor.shutdown(wait=False) super(OWGEODatasets, self).onDeleteWidget() def onNameEdited(self): if self.currentGds: gds_id = self.currentGds["dataset_id"] self.datasetNames[gds_id] = unicode(self.nameEdit.text()) self.commitIf()
class OWDatabasesUpdate(OWWidget): name = "Databases Update" description = "Update local systems biology databases." icon = "../widgets/icons/OWDatabasesUpdate.svg" priority = 1 inputs = [] outputs = [] want_main_area = False def __init__(self, parent=None, signalManager=None, name="Databases update"): OWWidget.__init__(self, parent, signalManager, name, wantMainArea=False) self.searchString = "" fbox = gui.widgetBox(self.controlArea, "Filter") self.completer = TokenListCompleter(self, caseSensitivity=Qt.CaseInsensitive) self.lineEditFilter = QLineEdit(textChanged=self.search_update) self.lineEditFilter.setCompleter(self.completer) fbox.layout().addWidget(self.lineEditFilter) box = gui.widgetBox(self.controlArea, "Files") self.filesView = QTreeWidget(self) self.filesView.setHeaderLabels(header_labels) self.filesView.setRootIsDecorated(False) self.filesView.setUniformRowHeights(True) self.filesView.setSelectionMode(QAbstractItemView.NoSelection) self.filesView.setSortingEnabled(True) self.filesView.sortItems(header.Title, Qt.AscendingOrder) self.filesView.setItemDelegateForColumn( 0, UpdateOptionsItemDelegate(self.filesView)) self.filesView.model().layoutChanged.connect(self.search_update) box.layout().addWidget(self.filesView) layout = QHBoxLayout() gui.widgetBox(self.controlArea, margin=0, orientation=layout) self.updateButton = gui.button( box, self, "Update all", callback=self.update_all, tooltip="Update all updatable files", ) self.downloadButton = gui.button( box, self, "Download all", callback=self.download_filtered, tooltip="Download all filtered files shown") self.cancelButton = gui.button( box, self, "Cancel", callback=self.cancel_active_threads, tooltip="Cancel scheduled downloads/updates.") self.addButton = gui.button(box, self, "Add ...", callback=self.__handle_dialog, tooltip="Add files for personal use.") layout.addWidget(self.updateButton) layout.addWidget(self.downloadButton) layout.addWidget(self.cancelButton) layout.addStretch() layout.addWidget(self.addButton) # Enable retryButton once connection is established # self.retryButton = gui.button( # box, self, "Reconnect", callback=self.initialize_files_view # ) # self.retryButton.hide() self.resize(800, 600) self.update_items = [] self._dialog = None self.progress_bar = None # threads self.threadpool = QThreadPool(self) #self.threadpool.setMaxThreadCount(1) self.workers = list() self.initialize_files_view() def __handle_dialog(self): if not self._dialog: self._dialog = FileUploadHelper(self) self._dialog.show() def __progress_advance(self): # GUI should be updated in main thread. That's why we are calling advance method here if self.progress_bar: self.progress_bar.advance() def handle_worker_exception(self, ex): self.progress_bar.finish() self.setStatusMessage('') if isinstance(ex, ConnectionError): # TODO: set warning messages pass print(ex) def initialize_files_view(self): # self.retryButton.hide() # clear view self.filesView.clear() # init progress bar self.progress_bar = gui.ProgressBar(self, iterations=3) # status message self.setStatusMessage('initializing') worker = Worker(evaluate_files_state, progress_callback=True) worker.signals.progress.connect(self.__progress_advance) worker.signals.result.connect(self.set_files_list) worker.signals.error.connect(self.handle_worker_exception) # move download process to worker thread self.threadpool.start(worker) self.setEnabled(False) def __create_action_button(self, fs, retry=None): if not fs.state not in [OUTDATED, USER_FILE] or not retry: self.filesView.setItemWidget(fs.tree_item, header.Update, None) button = QToolButton(None) if not retry: if fs.state == OUTDATED: button.setText('Update') button.clicked.connect( partial(self.submit_download_task, fs.domain, fs.filename, True)) elif fs.state == USER_FILE: if not fs.info_server: button.setText('Remove') button.clicked.connect( partial(self.submit_remove_task, fs.domain, fs.filename)) else: button.setText('Use server version') button.clicked.connect( partial(self.submit_download_task, fs.domain, fs.filename, True)) else: button.setText('Retry') button.clicked.connect( partial(self.submit_download_task, fs.domain, fs.filename, True)) button.setMaximumWidth(120) button.setMaximumHeight(20) button.setMinimumHeight(20) if sys.platform == "darwin": button.setAttribute(Qt.WA_MacSmallSize) self.filesView.setItemWidget(fs.tree_item, header.Update, button) def set_files_list(self, result): """ Set the files to show. """ assert threading.current_thread() == threading.main_thread() self.progress_bar.finish() self.setStatusMessage('') self.setEnabled(True) self.update_items = result all_tags = set() for fs in self.update_items: fs.tree_item = FileStateItem(fs) fs.download_option = DownloadOption(state=fs.state) fs.download_option.download_clicked.connect( partial(self.submit_download_task, fs.domain, fs.filename)) fs.download_option.remove_clicked.connect( partial(self.submit_remove_task, fs.domain, fs.filename)) # add widget items to the QTreeWidget self.filesView.addTopLevelItems( [fs.tree_item for fs in self.update_items]) # add action widgets to tree items for fs in self.update_items: self.filesView.setItemWidget(fs.tree_item, header.Download, fs.download_option) if fs.state in [USER_FILE, OUTDATED]: self.__create_action_button(fs) all_tags.update(fs.tags) self.filesView.setColumnWidth( header.Download, self.filesView.sizeHintForColumn(header.Download)) for column in range(1, len(header_labels)): self.filesView.resizeColumnToContents(column) hints = [hint for hint in sorted(all_tags) if not hint.startswith("#")] self.completer.setTokenList(hints) self.search_update() self.toggle_action_buttons() self.cancelButton.setEnabled(False) def toggle_action_buttons(self): selected_items = [ fs for fs in self.update_items if not fs.tree_item.isHidden() ] def button_check(sel_items, state, button): for item in sel_items: if item.state != state: button.setEnabled(False) else: button.setEnabled(True) break button_check(selected_items, OUTDATED, self.updateButton) button_check(selected_items, AVAILABLE, self.downloadButton) def search_update(self, searchString=None): strings = str(self.lineEditFilter.text()).split() for fs in self.update_items: hide = not all(UpdateItem_match(fs, string) for string in strings) fs.tree_item.setHidden(hide) self.toggle_action_buttons() def update_all(self): for fs in self.update_items: if fs.state == OUTDATED and not fs.tree_item.isHidden(): self.submit_download_task(fs.domain, fs.filename) def download_filtered(self): for fs in self.update_items: if not fs.tree_item.isHidden() and fs.state in [ AVAILABLE, OUTDATED ]: self.submit_download_task(fs.domain, fs.filename, start=False) self.run_download_tasks() def submit_download_task(self, domain, filename, start=True): """ Submit the (domain, filename) to be downloaded/updated. """ # get selected tree item index = self.tree_item_index(domain, filename) fs = self.update_items[index] worker = Worker(download_server_file, fs, index, progress_callback=True) worker.signals.progress.connect(self.__progress_advance) worker.signals.result.connect(self.on_download_finished) worker.signals.error.connect(self.on_download_exception) self.workers.append(worker) if start: self.run_download_tasks() def run_download_tasks(self): self.cancelButton.setEnabled(True) # init progress bar self.progress_bar = gui.ProgressBar(self, iterations=len(self.workers) * 100) # status message self.setStatusMessage('downloading') # move workers to threadpool [self.threadpool.start(worker) for worker in self.workers] self.filesView.setDisabled(True) # reset list of workers self.workers = list() def on_download_exception(self, ex): assert threading.current_thread() == threading.main_thread() self.progress_bar.finish() self.setStatusMessage('') print(ex) if isinstance(ex, ValueError): fs, index = ex.args # restore state and retry fs.refresh_state() fs.tree_item.update_data(fs) fs.download_option.state = fs.state self.__create_action_button(fs, retry=True) def on_download_finished(self, result): assert threading.current_thread() == threading.main_thread() # We check if all workers have completed. If not, continue if self.progress_bar.count == 100 or self.threadpool.activeThreadCount( ) == 0: self.filesView.setDisabled(False) self.progress_bar.finish() self.setStatusMessage('') fs, index = result # re-evaluate File State info = serverfiles.info(fs.domain, fs.filename) fs.refresh_state(info_local=info, info_server=info) # reinitialize treeWidgetItem fs.tree_item.update_data(fs) # reinitialize OptionWidget fs.download_option.state = fs.state self.filesView.setItemWidget(fs.tree_item, header.Update, None) self.toggle_action_buttons() for column in range(1, len(header_labels)): self.filesView.resizeColumnToContents(column) def submit_remove_task(self, domain, filename): serverfiles.LOCALFILES.remove(domain, filename) index = self.tree_item_index(domain, filename) fs = self.update_items[index] if fs.state == USER_FILE: self.filesView.takeTopLevelItem( self.filesView.indexOfTopLevelItem(fs.tree_item)) self.update_items.remove(fs) # self.filesView.removeItemWidget(index) else: # refresh item state fs.info_local = None fs.refresh_state() # reinitialize treeWidgetItem fs.tree_item.update_data(fs) # reinitialize OptionWidget fs.download_option.state = fs.state self.toggle_action_buttons() def cancel_active_threads(self): """ Cancel all pending update/download tasks (that have not yet started). """ if self.threadpool: self.threadpool.clear() def tree_item_index(self, domain, filename): for i, fs in enumerate(self.update_items): if fs.domain == domain and fs.filename == filename: return i raise ValueError("%r, %r not in update list" % (domain, filename)) def onDeleteWidget(self): self.cancel_active_threads() OWWidget.onDeleteWidget(self)
class OWDatabasesUpdate(OWWidget): name = "Databases Update" description = "Update local systems biology databases." icon = "../widgets/icons/OWDatabasesUpdate.svg" priority = 1 inputs = [] outputs = [] want_main_area = False def __init__(self, parent=None, signalManager=None, name="Databases update"): OWWidget.__init__(self, parent, signalManager, name, wantMainArea=False) self.searchString = "" fbox = gui.widgetBox(self.controlArea, "Filter") self.completer = TokenListCompleter( self, caseSensitivity=Qt.CaseInsensitive) self.lineEditFilter = QLineEdit(textChanged=self.SearchUpdate) self.lineEditFilter.setCompleter(self.completer) fbox.layout().addWidget(self.lineEditFilter) box = gui.widgetBox(self.controlArea, "Files") self.filesView = QTreeWidget(self) self.filesView.setHeaderLabels( ["", "Data Source", "Update", "Last Updated", "Size"]) self.filesView.setRootIsDecorated(False) self.filesView.setUniformRowHeights(True) self.filesView.setSelectionMode(QAbstractItemView.NoSelection) self.filesView.setSortingEnabled(True) self.filesView.sortItems(1, Qt.AscendingOrder) self.filesView.setItemDelegateForColumn( 0, UpdateOptionsItemDelegate(self.filesView)) self.filesView.model().layoutChanged.connect(self.SearchUpdate) box.layout().addWidget(self.filesView) box = gui.widgetBox(self.controlArea, orientation="horizontal") self.updateButton = gui.button( box, self, "Update all", callback=self.UpdateAll, tooltip="Update all updatable files", ) self.downloadButton = gui.button( box, self, "Download all", callback=self.DownloadFiltered, tooltip="Download all filtered files shown" ) self.cancelButton = gui.button( box, self, "Cancel", callback=self.Cancel, tooltip="Cancel scheduled downloads/updates." ) self.retryButton = gui.button( box, self, "Reconnect", callback=self.RetrieveFilesList ) self.retryButton.hide() gui.rubber(box) self.warning(0) box = gui.widgetBox(self.controlArea, orientation="horizontal") gui.rubber(box) self.infoLabel = QLabel() self.infoLabel.setAlignment(Qt.AlignCenter) self.controlArea.layout().addWidget(self.infoLabel) self.infoLabel.setSizePolicy(QSizePolicy.Expanding, QSizePolicy.Fixed) self.updateItems = [] self.resize(800, 600) self.progress = ProgressState(self, maximum=3) self.progress.valueChanged.connect(self._updateProgress) self.progress.rangeChanged.connect(self._updateProgress) self.executor = ThreadExecutor( threadPool=QThreadPool(maxThreadCount=2) ) task = Task(self, function=self.RetrieveFilesList) task.exceptionReady.connect(self.HandleError) task.start() self._tasks = [] self._haveProgress = False def RetrieveFilesList(self): self.retryButton.hide() self.warning(0) self.progress.setRange(0, 3) task = Task(function=partial(retrieveFilesList, methodinvoke(self.progress, "advance"))) task.resultReady.connect(self.SetFilesList) task.exceptionReady.connect(self.HandleError) self.executor.submit(task) self.setEnabled(False) def SetFilesList(self, serverInfo): """ Set the files to show. """ self.setEnabled(True) localInfo = serverfiles.allinfo() all_tags = set() self.filesView.clear() self.updateItems = [] for item in join_info_dict(localInfo, serverInfo): tree_item = UpdateTreeWidgetItem(item) options_widget = UpdateOptionsWidget(item.state) options_widget.item = item options_widget.installClicked.connect( partial(self.SubmitDownloadTask, item.domain, item.filename) ) options_widget.removeClicked.connect( partial(self.SubmitRemoveTask, item.domain, item.filename) ) self.updateItems.append((item, tree_item, options_widget)) all_tags.update(item.tags) self.filesView.addTopLevelItems( [tree_item for _, tree_item, _ in self.updateItems] ) for item, tree_item, options_widget in self.updateItems: self.filesView.setItemWidget(tree_item, 0, options_widget) # Add an update button if the file is updateable if item.state == OUTDATED: button = QToolButton( None, text="Update", maximumWidth=120, minimumHeight=20, maximumHeight=20 ) if sys.platform == "darwin": button.setAttribute(Qt.WA_MacSmallSize) button.clicked.connect( partial(self.SubmitDownloadTask, item.domain, item.filename) ) self.filesView.setItemWidget(tree_item, 2, button) self.progress.advance() self.filesView.setColumnWidth(0, self.filesView.sizeHintForColumn(0)) for column in range(1, 4): contents_hint = self.filesView.sizeHintForColumn(column) header_hint = self.filesView.header().sectionSizeHint(column) width = max(min(contents_hint, 400), header_hint) self.filesView.setColumnWidth(column, width) hints = [hint for hint in sorted(all_tags) if not hint.startswith("#")] self.completer.setTokenList(hints) self.SearchUpdate() self.UpdateInfoLabel() self.toggleButtons() self.cancelButton.setEnabled(False) self.progress.setRange(0, 0) def buttonCheck(self, selected_items, state, button): for item in selected_items: if item.state != state: button.setEnabled(False) else: button.setEnabled(True) break def toggleButtons(self): selected_items = [item for item, tree_item, _ in self.updateItems if not tree_item.isHidden()] self.buttonCheck(selected_items, OUTDATED, self.updateButton) self.buttonCheck(selected_items, AVAILABLE, self.downloadButton) def HandleError(self, exception): if isinstance(exception, ConnectionError): self.warning(0, "Could not connect to server! Check your connection " "and try to reconnect.") self.SetFilesList({}) self.retryButton.show() else: sys.excepthook(type(exception), exception, None) self.progress.setRange(0, 0) self.setEnabled(True) def UpdateInfoLabel(self): local = [item for item, tree_item, _ in self.updateItems if item.state != AVAILABLE and not tree_item.isHidden()] size = sum(float(item.size) for item in local) onServer = [item for item, tree_item, _ in self.updateItems if not tree_item.isHidden()] sizeOnServer = sum(float(item.size) for item in onServer) text = ("%i items, %s (on server: %i items, %s)" % (len(local), serverfiles.sizeformat(size), len(onServer), serverfiles.sizeformat(sizeOnServer))) self.infoLabel.setText(text) def UpdateAll(self): self.warning(0) for item, tree_item, _ in self.updateItems: if item.state == OUTDATED and not tree_item.isHidden(): self.SubmitDownloadTask(item.domain, item.filename) def DownloadFiltered(self): # TODO: submit items in the order shown. for item, tree_item, _ in self.updateItems: if not tree_item.isHidden() and item.state in \ [AVAILABLE, OUTDATED]: self.SubmitDownloadTask(item.domain, item.filename) def SearchUpdate(self, searchString=None): strings = str(self.lineEditFilter.text()).split() for item, tree_item, _ in self.updateItems: hide = not all(UpdateItem_match(item, string) for string in strings) tree_item.setHidden(hide) self.UpdateInfoLabel() self.toggleButtons() def SubmitDownloadTask(self, domain, filename): """ Submit the (domain, filename) to be downloaded/updated. """ self.cancelButton.setEnabled(True) index = self.updateItemIndex(domain, filename) _, tree_item, opt_widget = self.updateItems[index] task = DownloadTask(domain, filename, serverfiles.LOCALFILES) self.progress.adjustRange(0, 100) pb = ItemProgressBar(self.filesView) pb.setRange(0, 100) pb.setTextVisible(False) task.advanced.connect(pb.advance) task.advanced.connect(self.progress.advance) task.finished.connect(pb.hide) task.finished.connect(self.onDownloadFinished, Qt.QueuedConnection) task.exception.connect(self.onDownloadError, Qt.QueuedConnection) self.filesView.setItemWidget(tree_item, 2, pb) # Clear the text so it does not show behind the progress bar. tree_item.setData(2, Qt.DisplayRole, "") pb.show() # Disable the options widget opt_widget.setEnabled(False) self._tasks.append(task) self.executor.submit(task) def EndDownloadTask(self, task): future = task.future() index = self.updateItemIndex(task.domain, task.filename) item, tree_item, opt_widget = self.updateItems[index] self.filesView.removeItemWidget(tree_item, 2) opt_widget.setEnabled(True) if future.cancelled(): # Restore the previous state tree_item.setUpdateItem(item) opt_widget.setState(item.state) elif future.exception(): tree_item.setUpdateItem(item) opt_widget.setState(item.state) # Show the exception string in the size column. self.warning(0, "Error while downloading. Check your connection " "and retry.") # recreate button for download button = QToolButton( None, text="Retry", maximumWidth=120, minimumHeight=20, maximumHeight=20 ) if sys.platform == "darwin": button.setAttribute(Qt.WA_MacSmallSize) button.clicked.connect( partial(self.SubmitDownloadTask, item.domain, item.filename) ) self.filesView.setItemWidget(tree_item, 2, button) else: # get the new updated info dict and replace the the old item self.warning(0) info = serverfiles.info(item.domain, item.filename) new_item = update_item_from_info(item.domain, item.filename, info, info) self.updateItems[index] = (new_item, tree_item, opt_widget) tree_item.setUpdateItem(new_item) opt_widget.setState(new_item.state) self.UpdateInfoLabel() def SubmitRemoveTask(self, domain, filename): serverfiles.LOCALFILES.remove(domain, filename) index = self.updateItemIndex(domain, filename) item, tree_item, opt_widget = self.updateItems[index] if item.info_server: new_item = item._replace(state=AVAILABLE, local=None, info_local=None) else: new_item = item._replace(local=None, info_local=None) # Disable the options widget. No more actions can be performed # for the item. opt_widget.setEnabled(False) tree_item.setUpdateItem(new_item) opt_widget.setState(new_item.state) self.updateItems[index] = (new_item, tree_item, opt_widget) self.UpdateInfoLabel() def Cancel(self): """ Cancel all pending update/download tasks (that have not yet started). """ for task in self._tasks: task.future().cancel() def onDeleteWidget(self): self.Cancel() self.executor.shutdown(wait=False) OWWidget.onDeleteWidget(self) def onDownloadFinished(self): # on download completed/canceled/error assert QThread.currentThread() is self.thread() for task in list(self._tasks): future = task.future() if future.done(): self.EndDownloadTask(task) self._tasks.remove(task) if not self._tasks: # Clear/reset the overall progress self.progress.setRange(0, 0) self.cancelButton.setEnabled(False) def onDownloadError(self, exc_info): sys.excepthook(*exc_info) self.warning(0, "Error while downloading. Check your connection and " "retry.") def updateItemIndex(self, domain, filename): for i, (item, _, _) in enumerate(self.updateItems): if item.domain == domain and item.filename == filename: return i raise ValueError("%r, %r not in update list" % (domain, filename)) def _updateProgress(self, *args): rmin, rmax = self.progress.range() if rmin != rmax: if not self._haveProgress: self._haveProgress = True self.progressBarInit() self.progressBarSet(self.progress.ratioCompleted() * 100, processEvents=None) if rmin == rmax: self._haveProgress = False self.progressBarFinished()
class OWGenExpress(widget.OWWidget): name = "GenExpress" description = "Expression data from GenExpress." icon = "../widgets/icons/GenCloud.svg" priority = 36 inputs = [] outputs = [("Data", Orange.data.Table)] username = settings.Setting("anonymous") password = settings.Setting("") log2 = settings.Setting(False) transpose = settings.Setting(False) rtypei = settings.Setting(0) projecti = settings.Setting(0) serveri = settings.Setting(0) exnamei = settings.Setting(6) excludeconstant = settings.Setting(False) joinreplicates = settings.Setting(False) currentSelection = settings.Setting(None) experimentsHeaderState = settings.Setting( {name: False for _, name in HEADER[:ID_INDEX + 1]}) storedSortOrder = settings.Setting([]) storedSelections = settings.Setting([]) def __init__(self, parent=None): super().__init__(parent) self.servers = [ ('https://dictyexpress.research.bcm.edu/', 'dictyExpress'), ('https://cloud.genialis.com/', 'Genialis'), ] self.selectedExperiments = [] self.buffer = dicty.CacheSQLite(bufferfile) self.searchString = "" self.items = [] self.result_types = [] self.controlArea.setMaximumWidth(250) self.controlArea.setMinimumWidth(250) box = gui.widgetBox(self.controlArea, 'Project') self.projectCB = gui.comboBox(box, self, "projecti", items=[], callback=self.ProjectChosen) self.projects = [] b = gui.widgetBox(self.controlArea, "Selection bookmarks") self.selectionSetsWidget = SelectionSetsWidget(self) self.selectionSetsWidget.setSizePolicy(QSizePolicy.Preferred, QSizePolicy.Maximum) def store_selections(modified): if not modified: self.storedSelections = self.selectionSetsWidget.selections self.selectionSetsWidget.selectionModified.connect(store_selections) b.layout().addWidget(self.selectionSetsWidget) gui.separator(self.controlArea) b = gui.widgetBox(self.controlArea, "Sort output columns") self.columnsSortingWidget = SortedListWidget(self) self.columnsSortingWidget.setSizePolicy(QSizePolicy.Preferred, QSizePolicy.Maximum) box = gui.widgetBox(self.controlArea, 'Experiment name') self.experimentNameCB = gui.comboBox(box, self, "exnamei", items=SORTING_MODEL_LIST) b.layout().addWidget(self.columnsSortingWidget) sorting_model = QStringListModel(SORTING_MODEL_LIST) self.columnsSortingWidget.setModel(sorting_model) self.columnsSortingWidget.sortingOrder = self.storedSortOrder def store_sort_order(): self.storedSortOrder = self.columnsSortingWidget.sortingOrder self.columnsSortingWidget.sortingOrderChanged.connect(store_sort_order) gui.separator(self.controlArea) box = gui.widgetBox(self.controlArea, 'Expression Type') self.expressionTypesCB = gui.comboBox(box, self, "rtypei", items=[], callback=self.UpdateResultsList) gui.checkBox(self.controlArea, self, "excludeconstant", "Exclude labels with constant values") gui.checkBox(self.controlArea, self, "joinreplicates", "Average replicates (use median)") gui.checkBox(self.controlArea, self, "log2", "Logarithmic (base 2) transformation") gui.checkBox(self.controlArea, self, "transpose", "Genes as columns") self.commit_button = gui.button(self.controlArea, self, "&Commit", callback=self.Commit) self.commit_button.setDisabled(True) gui.rubber(self.controlArea) box = gui.widgetBox(self.controlArea, 'Server') gui.comboBox(box, self, "serveri", items=[title for url, title in self.servers], callback=self.ServerChosen) gui.lineEdit(box, self, "username", "Username:"******"password", "Password:"******"Clear cache", callback=self.clear_cache) gui.lineEdit(self.mainArea, self, "searchString", "Search", callbackOnType=True, callback=self.SearchUpdate) self.headerLabels = [t[1] for t in HEADER] self.experimentsWidget = QTreeWidget() self.experimentsWidget.setHeaderLabels(self.headerLabels) self.experimentsWidget.setSelectionMode(QTreeWidget.ExtendedSelection) self.experimentsWidget.setRootIsDecorated(False) self.experimentsWidget.setSortingEnabled(True) contextEventFilter = gui.VisibleHeaderSectionContextEventFilter( self.experimentsWidget, self.experimentsWidget) self.experimentsWidget.header().installEventFilter(contextEventFilter) self.experimentsWidget.setItemDelegateForColumn( 0, gui.IndicatorItemDelegate(self, role=Qt.DisplayRole)) self.experimentsWidget.setAlternatingRowColors(True) self.experimentsWidget.selectionModel().selectionChanged.connect( self.onSelectionChanged) self.selectionSetsWidget.setSelectionModel( self.experimentsWidget.selectionModel()) self.selectionSetsWidget.setSelections(self.storedSelections) self.mainArea.layout().addWidget(self.experimentsWidget) self.restoreHeaderState() self.experimentsWidget.header().geometriesChanged.connect( self.saveHeaderState) self.dbc = None self.AuthSet() QTimer.singleShot(100, self.ConnectAndUpdate) def sizeHint(self): return QSize(800, 600) def AuthSet(self): if len(self.username): self.passf.setDisabled(False) else: self.passf.setDisabled(True) def AuthChanged(self): self.AuthSet() self.ConnectAndUpdate() def ConnectAndUpdate(self): self.Connect() if self.dbc: def get_data_count(project_id): # XXX: is there a better way? # Note: limit 0 would return all objects return self.dbc.gen.api.data.get( case_ids__contains=project_id, type__startswith='data:expression:', limit=1)['meta']['total_count'] self.projects = sorted([ p for p in self.dbc.projects().items() if get_data_count(p[0]) > 0 ], key=lambda x: x[1]) self.UpdateProjects() self.ProjectChosen() self.UpdateExperimentTypes() def Connect(self): self.error(1) self.warning(1) username = '******' password = '******' url = self.servers[self.serveri][0] if self.username: username = self.username password = self.password if username.lower() in ['*****@*****.**', 'anonymous']: username = '******' password = '******' self.dbc = None self.projects = [] self.result_types = [] try: self.dbc = Genesis(address=url, username=username, password=password, cache=self.buffer) except requests.exceptions.ConnectionError: self.dbc = Genesis(address=url, username=username, password=password, connect=False, cache=self.buffer) self.warning(1, "Could not connect to server, working from cache.") except Exception: self.error(1, "Wrong username or password.") self.UpdateProjects() self.UpdateExperimentTypes() # clear lists def Reload(self): self.UpdateExperiments(reload=True) def clear_cache(self): self.buffer.clear() self.Reload() def rtype(self): """Return selected result template type """ if self.result_types: return self.result_types[self.rtypei] else: return None def UpdateExperimentTypes(self): self.expressionTypesCB.clear() items = [self.result_types_labels[desc] for desc in self.result_types] self.expressionTypesCB.addItems(items) #do not update anything if the list is empty if len(self.result_types): self.rtypei = max(0, min(self.rtypei, len(self.result_types) - 1)) def UpdateProjects(self): self.projectCB.clear() items = [desc for pid, desc in self.projects] self.projectCB.addItems(items) #do not update anything if the list if empty if len(self.projects) > 0: self.projecti = max(0, min(self.projecti, len(self.projects) - 1)) def UpdateExperiments(self, reload=False): self.experimentsWidget.clear() if not self.dbc or not self.dbc.projectid: # the connection did not succeed return self.items = [] self.progressBarInit() result_types = [] result_types_labels = [] sucind = False # success indicator for database index try: result_types, result_types_labels = self.dbc.result_types( reload=reload) sucind = True except Exception: try: result_types, result_types_labels = self.dbc.result_types() self.warning(0, "Can not access database - using cached data.") sucind = True except Exception: self.error(0, "Can not access database.") if sucind: self.warning(0) self.error(0) self.result_types = result_types self.result_types_labels = result_types_labels self.UpdateExperimentTypes() self.UpdateResultsList(reload=reload) self.progressBarFinished() if self.currentSelection: self.currentSelection.select( self.experimentsWidget.selectionModel()) self.handle_commit_button() def ProjectChosen(self, reload=False): if self.projects: self.dbc.projectid = self.projects[self.projecti][0] else: self.dbc.projectid = None self.UpdateExperiments(reload=reload) def ServerChosen(self): self.ConnectAndUpdate() def UpdateResultsList(self, reload=False): results_list = self.dbc.results_list(self.rtype(), reload=reload) try: results_list = self.dbc.results_list(self.rtype(), reload=reload) except Exception: try: results_list = self.dbc.results_list(self.rtype()) except Exception: self.error(0, "Can not access database.") self.results_list = results_list #softly change the view so that the selection stays the same items_shown = {} for i, item in enumerate(self.items): c = str(item.text(ID_INDEX)) items_shown[c] = i items_to_show = set(id_ for id_ in self.results_list) add_items = set(items_to_show) - set(items_shown) delete_items = set(items_shown) - set(items_to_show) i = 0 while i < self.experimentsWidget.topLevelItemCount(): it = self.experimentsWidget.topLevelItem(i) if str(it.text(ID_INDEX)) in delete_items: self.experimentsWidget.takeTopLevelItem(i) else: i += 1 delete_ind = set([items_shown[i] for i in delete_items]) self.items = [ it for i, it in enumerate(self.items) if i not in delete_ind ] for r_annot in add_items: d = defaultdict(lambda: "?", self.results_list[r_annot]) row_items = [""] + [ to_text(d.get(key, "?")) for key, _ in HEADER[1:] ] row_items[ID_INDEX] = r_annot ci = MyTreeWidgetItem(self.experimentsWidget, row_items) self.items.append(ci) for i in range(len(self.headerLabels)): self.experimentsWidget.resizeColumnToContents(i) self.wantbufver = lambda x: self.results_list[x]["date_modified"] self.UpdateCached() def UpdateCached(self): if self.wantbufver and self.dbc: for item in self.items: id = str(item.text(ID_INDEX)) version = self.dbc._in_buffer(id + "|||" + self.rtype()) value = " " if version == self.wantbufver(id) else "" item.setData(0, Qt.DisplayRole, value) def SearchUpdate(self, string=""): for item in self.items: item.setHidden(not all(s in item for s in self.searchString.split())) def Commit(self): pb = gui.ProgressBar(self, iterations=100) table = None ids = [] for item in self.experimentsWidget.selectedItems(): unique_id = str(item.text(ID_INDEX)) ids.append(unique_id) transfn = None if self.log2: transfn = lambda x: math.log(x + 1.0, 2) reverse_header_dict = {name: name for key, name in HEADER} reverse_header_dict["ID"] = "id" allowed_labels = None def namefn(a): name = SORTING_MODEL_LIST[self.exnamei] name = reverse_header_dict.get(name, "id") return dict(a)[name] if len(ids): table = self.dbc.get_data( ids=ids, result_type=self.rtype(), callback=pb.advance, exclude_constant_labels=self.excludeconstant, bufver=self.wantbufver, transform=transfn, allowed_labels=allowed_labels, namefn=namefn) if self.joinreplicates: table = dicty.join_replicates( table, ignorenames=self.dbc.IGNORE_REPLICATE, namefn="name", avg=dicty.median, fnshow=lambda x: " | ".join(map(str, x))) # Sort attributes sortOrder = self.columnsSortingWidget.sortingOrder all_values = defaultdict(set) for at in table.domain.attributes: atts = at.attributes for name in sortOrder: all_values[name].add( atts.get(reverse_header_dict[name], "")) isnum = {} for at, vals in all_values.items(): vals = filter(None, vals) try: for a in vals: float(a) isnum[at] = True except ValueError: isnum[at] = False def optfloat(x, at): if x == "": return "" else: return float(x) if isnum[at] else x def sorting_key(attr): atts = attr.attributes return tuple([ optfloat(atts.get(reverse_header_dict[name], ""), name) for name in sortOrder ]) attributes = sorted(table.domain.attributes, key=sorting_key) domain = Orange.data.Domain(attributes, table.domain.class_vars, table.domain.metas) table = Orange.data.Table.from_table(domain, table) table = Orange.data.Table(domain, table) if self.transpose: experiments = [at for at in table.domain.variables] attr = [ compat.ContinuousVariable.make(ex['DDB'].value) for ex in table ] metavars = sorted(table.domain.variables[0].attributes.keys()) metavars = [ compat.StringVariable.make(name) for name in metavars ] domain = compat.create_domain(attr, None, metavars) metavars = compat.get_metas(domain) metas = [[exp.attributes[var.name] for var in metavars] for exp in experiments] table = compat.create_table(domain, table.X.transpose(), None, metas) data_hints.set_hint(table, "taxid", "352472") data_hints.set_hint(table, "genesinrows", False) self.send("Data", table) self.UpdateCached() pb.finish() def onSelectionChanged(self, selected, deselected): self.handle_commit_button() def handle_commit_button(self): self.currentSelection = \ SelectionByKey(self.experimentsWidget.selectionModel().selection(), key=(ID_INDEX,)) self.commit_button.setDisabled(not len(self.currentSelection)) def saveHeaderState(self): hview = self.experimentsWidget.header() for i, label in enumerate(self.headerLabels): self.experimentsHeaderState[label] = hview.isSectionHidden(i) def restoreHeaderState(self): hview = self.experimentsWidget.header() state = self.experimentsHeaderState for i, label in enumerate(self.headerLabels): hview.setSectionHidden(i, state.get(label, True)) self.experimentsWidget.resizeColumnToContents(i)
class OWGenExpress(widget.OWWidget): name = "GenExpress" description = "Expression data from GenExpress." icon = "../widgets/icons/GenCloud.svg" priority = 36 inputs = [] outputs = [("Data", Orange.data.Table)] username = settings.Setting("anonymous") password = settings.Setting("") log2 = settings.Setting(False) transpose = settings.Setting(False) rtypei = settings.Setting(0) projecti = settings.Setting(0) serveri = settings.Setting(0) exnamei = settings.Setting(6) excludeconstant = settings.Setting(False) joinreplicates = settings.Setting(False) currentSelection = settings.Setting(None) experimentsHeaderState = settings.Setting({ name: False for _, name in HEADER[:ID_INDEX + 1]} ) storedSortOrder = settings.Setting([]) storedSelections = settings.Setting([]) def __init__(self, parent=None): super().__init__(parent) self.servers = [ ('https://dictyexpress.research.bcm.edu/', 'dictyExpress'), ('https://cloud.genialis.com/', 'Genialis'), ] self.selectedExperiments = [] self.buffer = dicty.CacheSQLite(bufferfile) self.searchString = "" self.items = [] self.result_types = [] self.controlArea.setMaximumWidth(250) self.controlArea.setMinimumWidth(250) box = gui.widgetBox(self.controlArea, 'Project') self.projectCB = gui.comboBox( box, self, "projecti", items=[], callback=self.ProjectChosen) self.projects = [] b = gui.widgetBox(self.controlArea, "Selection bookmarks") self.selectionSetsWidget = SelectionSetsWidget(self) self.selectionSetsWidget.setSizePolicy( QSizePolicy.Preferred, QSizePolicy.Maximum) def store_selections(modified): if not modified: self.storedSelections = self.selectionSetsWidget.selections self.selectionSetsWidget.selectionModified.connect(store_selections) b.layout().addWidget(self.selectionSetsWidget) gui.separator(self.controlArea) b = gui.widgetBox(self.controlArea, "Sort output columns") self.columnsSortingWidget = SortedListWidget(self) self.columnsSortingWidget.setSizePolicy( QSizePolicy.Preferred, QSizePolicy.Maximum) box = gui.widgetBox(self.controlArea, 'Experiment name') self.experimentNameCB = gui.comboBox( box, self, "exnamei", items=SORTING_MODEL_LIST) b.layout().addWidget(self.columnsSortingWidget) sorting_model = QStringListModel(SORTING_MODEL_LIST) self.columnsSortingWidget.setModel(sorting_model) self.columnsSortingWidget.sortingOrder = self.storedSortOrder def store_sort_order(): self.storedSortOrder = self.columnsSortingWidget.sortingOrder self.columnsSortingWidget.sortingOrderChanged.connect(store_sort_order) gui.separator(self.controlArea) box = gui.widgetBox(self.controlArea, 'Expression Type') self.expressionTypesCB = gui.comboBox( box, self, "rtypei", items=[], callback=self.UpdateResultsList) gui.checkBox(self.controlArea, self, "excludeconstant", "Exclude labels with constant values") gui.checkBox(self.controlArea, self, "joinreplicates", "Average replicates (use median)") gui.checkBox(self.controlArea, self, "log2", "Logarithmic (base 2) transformation") gui.checkBox(self.controlArea, self, "transpose", "Genes as columns") self.commit_button = gui.button(self.controlArea, self, "&Commit", callback=self.Commit) self.commit_button.setDisabled(True) gui.rubber(self.controlArea) box = gui.widgetBox(self.controlArea, 'Server') gui.comboBox(box, self, "serveri", items=[title for url, title in self.servers], callback=self.ServerChosen) gui.lineEdit(box, self, "username", "Username:"******"password", "Password:"******"Clear cache", callback=self.clear_cache) gui.lineEdit(self.mainArea, self, "searchString", "Search", callbackOnType=True, callback=self.SearchUpdate) self.headerLabels = [t[1] for t in HEADER] self.experimentsWidget = QTreeWidget() self.experimentsWidget.setHeaderLabels(self.headerLabels) self.experimentsWidget.setSelectionMode(QTreeWidget.ExtendedSelection) self.experimentsWidget.setRootIsDecorated(False) self.experimentsWidget.setSortingEnabled(True) contextEventFilter = gui.VisibleHeaderSectionContextEventFilter( self.experimentsWidget, self.experimentsWidget) self.experimentsWidget.header().installEventFilter(contextEventFilter) self.experimentsWidget.setItemDelegateForColumn( 0, gui.IndicatorItemDelegate(self, role=Qt.DisplayRole)) self.experimentsWidget.setAlternatingRowColors(True) self.experimentsWidget.selectionModel().selectionChanged.connect( self.onSelectionChanged) self.selectionSetsWidget.setSelectionModel( self.experimentsWidget.selectionModel()) self.selectionSetsWidget.setSelections(self.storedSelections) self.mainArea.layout().addWidget(self.experimentsWidget) self.restoreHeaderState() self.experimentsWidget.header().geometriesChanged.connect( self.saveHeaderState) self.dbc = None self.AuthSet() QTimer.singleShot(100, self.ConnectAndUpdate) def sizeHint(self): return QSize(800, 600) def AuthSet(self): if len(self.username): self.passf.setDisabled(False) else: self.passf.setDisabled(True) def AuthChanged(self): self.AuthSet() self.ConnectAndUpdate() def ConnectAndUpdate(self): self.Connect() if self.dbc: def get_data_count(project_id): # XXX: is there a better way? # Note: limit 0 would return all objects return self.dbc.gen.api.data.get(case_ids__contains=project_id, type__startswith='data:expression:', limit=1)['meta']['total_count'] self.projects = sorted([p for p in self.dbc.projects().items() if get_data_count(p[0]) > 0], key=lambda x: x[1]) self.UpdateProjects() self.ProjectChosen() self.UpdateExperimentTypes() def Connect(self): self.error(1) self.warning(1) username = '******' password = '******' url = self.servers[self.serveri][0] if self.username: username = self.username password = self.password if username.lower() in ['*****@*****.**', 'anonymous']: username = '******' password = '******' self.dbc = None self.projects = [] self.result_types = [] try: self.dbc = Genesis( address=url, username=username, password=password, cache=self.buffer) except requests.exceptions.ConnectionError: self.dbc = Genesis( address=url, username=username, password=password, connect=False, cache=self.buffer) self.warning(1, "Could not connect to server, working from cache.") except Exception: self.error(1, "Wrong username or password.") self.UpdateProjects() self.UpdateExperimentTypes() # clear lists def Reload(self): self.UpdateExperiments(reload=True) def clear_cache(self): self.buffer.clear() self.Reload() def rtype(self): """Return selected result template type """ if self.result_types: return self.result_types[self.rtypei] else: return None def UpdateExperimentTypes(self): self.expressionTypesCB.clear() items = [self.result_types_labels[desc] for desc in self.result_types] self.expressionTypesCB.addItems(items) #do not update anything if the list is empty if len(self.result_types): self.rtypei = max(0, min(self.rtypei, len(self.result_types) - 1)) def UpdateProjects(self): self.projectCB.clear() items = [desc for pid, desc in self.projects] self.projectCB.addItems(items) #do not update anything if the list if empty if len(self.projects) > 0: self.projecti = max(0, min(self.projecti, len(self.projects) - 1)) def UpdateExperiments(self, reload=False): self.experimentsWidget.clear() if not self.dbc or not self.dbc.projectid: # the connection did not succeed return self.items = [] self.progressBarInit() result_types = [] result_types_labels = [] sucind = False # success indicator for database index try: result_types, result_types_labels = self.dbc.result_types(reload=reload) sucind = True except Exception: try: result_types, result_types_labels = self.dbc.result_types() self.warning(0, "Can not access database - using cached data.") sucind = True except Exception: self.error(0, "Can not access database.") if sucind: self.warning(0) self.error(0) self.result_types = result_types self.result_types_labels = result_types_labels self.UpdateExperimentTypes() self.UpdateResultsList(reload=reload) self.progressBarFinished() if self.currentSelection: self.currentSelection.select(self.experimentsWidget.selectionModel()) self.handle_commit_button() def ProjectChosen(self, reload=False): if self.projects: self.dbc.projectid = self.projects[self.projecti][0] else: self.dbc.projectid = None self.UpdateExperiments(reload=reload) def ServerChosen(self): self.ConnectAndUpdate() def UpdateResultsList(self, reload=False): results_list = self.dbc.results_list(self.rtype(), reload=reload) try: results_list = self.dbc.results_list(self.rtype(), reload=reload) except Exception: try: results_list = self.dbc.results_list(self.rtype()) except Exception: self.error(0, "Can not access database.") self.results_list = results_list #softly change the view so that the selection stays the same items_shown = {} for i, item in enumerate(self.items): c = str(item.text(ID_INDEX)) items_shown[c] = i items_to_show = set(id_ for id_ in self.results_list) add_items = set(items_to_show) - set(items_shown) delete_items = set(items_shown) - set(items_to_show) i = 0 while i < self.experimentsWidget.topLevelItemCount(): it = self.experimentsWidget.topLevelItem(i) if str(it.text(ID_INDEX)) in delete_items: self.experimentsWidget.takeTopLevelItem(i) else: i += 1 delete_ind = set([items_shown[i] for i in delete_items]) self.items = [it for i, it in enumerate(self.items) if i not in delete_ind] for r_annot in add_items: d = defaultdict(lambda: "?", self.results_list[r_annot]) row_items = [""] + [to_text(d.get(key, "?")) for key, _ in HEADER[1:]] row_items[ID_INDEX] = r_annot ci = MyTreeWidgetItem(self.experimentsWidget, row_items) self.items.append(ci) for i in range(len(self.headerLabels)): self.experimentsWidget.resizeColumnToContents(i) self.wantbufver = lambda x: self.results_list[x]["date_modified"] self.UpdateCached() def UpdateCached(self): if self.wantbufver and self.dbc: for item in self.items: id = str(item.text(ID_INDEX)) version = self.dbc._in_buffer(id + "|||" + self.rtype()) value = " " if version == self.wantbufver(id) else "" item.setData(0, Qt.DisplayRole, value) def SearchUpdate(self, string=""): for item in self.items: item.setHidden( not all(s in item for s in self.searchString.split())) def Commit(self): pb = gui.ProgressBar(self, iterations=100) table = None ids = [] for item in self.experimentsWidget.selectedItems(): unique_id = str(item.text(ID_INDEX)) ids.append(unique_id) transfn = None if self.log2: transfn = lambda x: math.log(x + 1.0, 2) reverse_header_dict = {name: name for key, name in HEADER} reverse_header_dict["ID"] = "id" allowed_labels = None def namefn(a): name = SORTING_MODEL_LIST[self.exnamei] name = reverse_header_dict.get(name, "id") return dict(a)[name] if len(ids): table = self.dbc.get_data( ids=ids, result_type=self.rtype(), callback=pb.advance, exclude_constant_labels=self.excludeconstant, bufver=self.wantbufver, transform=transfn, allowed_labels=allowed_labels, namefn=namefn) if self.joinreplicates: table = dicty.join_replicates(table, ignorenames=self.dbc.IGNORE_REPLICATE, namefn="name", avg=dicty.median, fnshow=lambda x: " | ".join(map(str, x))) # Sort attributes sortOrder = self.columnsSortingWidget.sortingOrder all_values = defaultdict(set) for at in table.domain.attributes: atts = at.attributes for name in sortOrder: all_values[name].add(atts.get(reverse_header_dict[name], "")) isnum = {} for at, vals in all_values.items(): vals = filter(None, vals) try: for a in vals: float(a) isnum[at] = True except ValueError: isnum[at] = False def optfloat(x, at): if x == "": return "" else: return float(x) if isnum[at] else x def sorting_key(attr): atts = attr.attributes return tuple([optfloat(atts.get(reverse_header_dict[name], ""), name) for name in sortOrder]) attributes = sorted(table.domain.attributes, key=sorting_key) domain = Orange.data.Domain( attributes, table.domain.class_vars, table.domain.metas) table = Orange.data.Table.from_table(domain, table) table = Orange.data.Table(domain, table) if self.transpose: experiments = [at for at in table.domain.variables] attr = [compat.ContinuousVariable.make(ex['DDB'].value) for ex in table] metavars = sorted(table.domain.variables[0].attributes.keys()) metavars = [compat.StringVariable.make(name) for name in metavars] domain = compat.create_domain(attr, None, metavars) metas = [[exp.attributes[var.name] for var in metavars] for exp in experiments] table = compat.create_table(domain, table.X.transpose(), None, metas) data_hints.set_hint(table, "taxid", "352472") data_hints.set_hint(table, "genesinrows", False) self.send("Data", table) self.UpdateCached() pb.finish() def onSelectionChanged(self, selected, deselected): self.handle_commit_button() def handle_commit_button(self): self.currentSelection = \ SelectionByKey(self.experimentsWidget.selectionModel().selection(), key=(ID_INDEX,)) self.commit_button.setDisabled(not len(self.currentSelection)) def saveHeaderState(self): hview = self.experimentsWidget.header() for i, label in enumerate(self.headerLabels): self.experimentsHeaderState[label] = hview.isSectionHidden(i) def restoreHeaderState(self): hview = self.experimentsWidget.header() state = self.experimentsHeaderState for i, label in enumerate(self.headerLabels): hview.setSectionHidden(i, state.get(label, True)) self.experimentsWidget.resizeColumnToContents(i)
class OWGEODatasets(OWWidget): name = "GEO Data Sets" description = DESCRIPTION icon = "../widgets/icons/GEODataSets.svg" priority = PRIORITY inputs = [] outputs = [("Expression Data", Orange.data.Table)] settingsList = ["outputRows", "mergeSpots", "gdsSelectionStates", "splitterSettings", "currentGds", "autoCommit", "datasetNames"] outputRows = Setting(True) mergeSpots = Setting(True) gdsSelectionStates = Setting({}) currentGds = Setting(None) datasetNames = Setting({}) splitterSettings = Setting( (b'\x00\x00\x00\xff\x00\x00\x00\x00\x00\x00\x00\x02\x00\x00\x01\xea\x00\x00\x00\xd7\x01\x00\x00\x00\x07\x01\x00\x00\x00\x02', b'\x00\x00\x00\xff\x00\x00\x00\x00\x00\x00\x00\x02\x00\x00\x01\xb5\x00\x00\x02\x10\x01\x00\x00\x00\x07\x01\x00\x00\x00\x01') ) autoCommit = Setting(False) def __init__(self, parent=None, signalManager=None, name=" GEO Data Sets"): OWWidget.__init__(self, parent, signalManager, name) self.selectionChanged = False self.filterString = "" self.datasetName = "" ## GUI box = gui.widgetBox(self.controlArea, "Info", addSpace=True) self.infoBox = gui.widgetLabel(box, "Initializing\n\n") box = gui.widgetBox(self.controlArea, "Output", addSpace=True) gui.radioButtonsInBox(box, self, "outputRows", ["Genes in rows", "Samples in rows"], "Rows", callback=self.commitIf) gui.checkBox(box, self, "mergeSpots", "Merge spots of same gene", callback=self.commitIf) gui.separator(box) self.nameEdit = gui.lineEdit( box, self, "datasetName", "Data set name", tooltip="Override the default output data set name", callback=self.onNameEdited ) self.nameEdit.setPlaceholderText("") if sys.version_info < (3, ): box = gui.widgetBox(self.controlArea, "Commit", addSpace=True) self.commitButton = gui.button( box, self, "Commit", callback=self.commit) cb = gui.checkBox(box, self, "autoCommit", "Commit on any change") gui.setStopper(self, self.commitButton, cb, "selectionChanged", self.commit) else: gui.auto_commit(self.controlArea, self, "autoCommit", "Commit", box="Commit") self.commitIf = self.commit gui.rubber(self.controlArea) gui.widgetLabel(self.mainArea, "Filter") self.filterLineEdit = QLineEdit( textChanged=self.filter ) self.completer = TokenListCompleter( self, caseSensitivity=Qt.CaseInsensitive ) self.filterLineEdit.setCompleter(self.completer) self.mainArea.layout().addWidget(self.filterLineEdit) splitter = QSplitter(Qt.Vertical, self.mainArea) self.mainArea.layout().addWidget(splitter) self.treeWidget = QTreeView(splitter) self.treeWidget.setSelectionMode(QTreeView.SingleSelection) self.treeWidget.setRootIsDecorated(False) self.treeWidget.setSortingEnabled(True) self.treeWidget.setAlternatingRowColors(True) self.treeWidget.setUniformRowHeights(True) self.treeWidget.setEditTriggers(QTreeView.NoEditTriggers) linkdelegate = LinkStyledItemDelegate(self.treeWidget) self.treeWidget.setItemDelegateForColumn(1, linkdelegate) self.treeWidget.setItemDelegateForColumn(8, linkdelegate) self.treeWidget.setItemDelegateForColumn( 0, gui.IndicatorItemDelegate(self.treeWidget, role=Qt.DisplayRole)) proxyModel = MySortFilterProxyModel(self.treeWidget) self.treeWidget.setModel(proxyModel) self.treeWidget.selectionModel().selectionChanged.connect( self.updateSelection ) self.treeWidget.viewport().setMouseTracking(True) splitterH = QSplitter(Qt.Horizontal, splitter) box = gui.widgetBox(splitterH, "Description") self.infoGDS = gui.widgetLabel(box, "") self.infoGDS.setWordWrap(True) gui.rubber(box) box = gui.widgetBox(splitterH, "Sample Annotations") self.annotationsTree = QTreeWidget(box) self.annotationsTree.setHeaderLabels( ["Type (Sample annotations)", "Sample count"] ) self.annotationsTree.setRootIsDecorated(True) box.layout().addWidget(self.annotationsTree) self.annotationsTree.itemChanged.connect( self.annotationSelectionChanged ) self._annotationsUpdating = False self.splitters = splitter, splitterH for sp, setting in zip(self.splitters, self.splitterSettings): sp.splitterMoved.connect(self.splitterMoved) sp.restoreState(setting) self.searchKeys = ["dataset_id", "title", "platform_organism", "description"] self.gds = [] self.gds_info = None self.resize(1000, 600) self.setBlocking(True) self.setEnabled(False) self.progressBarInit() self._executor = ThreadExecutor() func = partial(get_gds_model, methodinvoke(self, "_setProgress", (float,))) self._inittask = Task(function=func) self._inittask.finished.connect(self._initializemodel) self._executor.submit(self._inittask) self._datatask = None @Slot(float) def _setProgress(self, value): self.progressBarValue = value def _initializemodel(self): assert self.thread() is QThread.currentThread() model, self.gds_info, self.gds = self._inittask.result() model.setParent(self) proxy = self.treeWidget.model() proxy.setFilterKeyColumn(0) proxy.setFilterRole(TextFilterRole) proxy.setFilterCaseSensitivity(False) proxy.setFilterFixedString(self.filterString) proxy.setSourceModel(model) proxy.sort(0, Qt.DescendingOrder) self.progressBarFinished() self.setBlocking(False) self.setEnabled(True) filter_items = " ".join( gds[key] for gds in self.gds for key in self.searchKeys ) tr_chars = ",.:;!?(){}[]_-+\\|/%#@$^&*<>~`" tr_table = str.maketrans(tr_chars, " " * len(tr_chars)) filter_items = filter_items.translate(tr_table) filter_items = sorted(set(filter_items.split(" "))) filter_items = [item for item in filter_items if len(item) > 3] self.completer.setTokenList(filter_items) if self.currentGds: current_id = self.currentGds["dataset_id"] gdss = [(i, qunpack(proxy.data(proxy.index(i, 1), Qt.DisplayRole))) for i in range(proxy.rowCount())] current = [i for i, data in gdss if data and data == current_id] if current: current_index = proxy.index(current[0], 0) self.treeWidget.selectionModel().select( current_index, QItemSelectionModel.Select | QItemSelectionModel.Rows ) self.treeWidget.scrollTo( current_index, QTreeView.PositionAtCenter) for i in range(8): self.treeWidget.resizeColumnToContents(i) self.treeWidget.setColumnWidth( 1, min(self.treeWidget.columnWidth(1), 300)) self.treeWidget.setColumnWidth( 2, min(self.treeWidget.columnWidth(2), 200)) self.updateInfo() def updateInfo(self): gds_info = self.gds_info text = ("%i datasets\n%i datasets cached\n" % (len(gds_info), len(glob.glob(serverfiles.localpath("GEO") + "/GDS*")))) filtered = self.treeWidget.model().rowCount() if len(self.gds) != filtered: text += ("%i after filtering") % filtered self.infoBox.setText(text) def updateSelection(self, *args): current = self.treeWidget.selectedIndexes() mapToSource = self.treeWidget.model().mapToSource current = [mapToSource(index).row() for index in current] if current: self.currentGds = self.gds[current[0]] self.setAnnotations(self.currentGds) self.infoGDS.setText(self.currentGds.get("description", "")) self.nameEdit.setPlaceholderText(self.currentGds["title"]) self.datasetName = \ self.datasetNames.get(self.currentGds["dataset_id"], "") else: self.currentGds = None self.nameEdit.setPlaceholderText("") self.datasetName = "" self.commitIf() def setAnnotations(self, gds): self._annotationsUpdating = True self.annotationsTree.clear() annotations = defaultdict(set) subsetscount = {} for desc in gds["subsets"]: annotations[desc["type"]].add(desc["description"]) subsetscount[desc["description"]] = str(len(desc["sample_id"])) for type, subsets in annotations.items(): key = (gds["dataset_id"], type) subsetItem = QTreeWidgetItem(self.annotationsTree, [type]) subsetItem.setFlags(subsetItem.flags() | Qt.ItemIsUserCheckable | Qt.ItemIsTristate) subsetItem.setCheckState( 0, self.gdsSelectionStates.get(key, Qt.Checked) ) subsetItem.key = key for subset in subsets: key = (gds["dataset_id"], type, subset) item = QTreeWidgetItem( subsetItem, [subset, subsetscount.get(subset, "")] ) item.setFlags(item.flags() | Qt.ItemIsUserCheckable) item.setCheckState( 0, self.gdsSelectionStates.get(key, Qt.Checked) ) item.key = key self._annotationsUpdating = False self.annotationsTree.expandAll() for i in range(self.annotationsTree.columnCount()): self.annotationsTree.resizeColumnToContents(i) def annotationSelectionChanged(self, item, column): if self._annotationsUpdating: return for i in range(self.annotationsTree.topLevelItemCount()): item = self.annotationsTree.topLevelItem(i) self.gdsSelectionStates[item.key] = item.checkState(0) for j in range(item.childCount()): child = item.child(j) self.gdsSelectionStates[child.key] = child.checkState(0) def filter(self): filter_string = unicode(self.filterLineEdit.text()) proxyModel = self.treeWidget.model() if proxyModel: strings = filter_string.lower().strip().split() proxyModel.setFilterFixedStrings(strings) self.updateInfo() def selectedSamples(self): """ Return the currently selected sample annotations. The return value is a list of selected (sample type, sample value) tuples. .. note:: if some Sample annotation type has no selected values. this method will return all values for it. """ samples = [] unused_types = [] used_types = [] for stype in childiter(self.annotationsTree.invisibleRootItem()): selected_values = [] all_values = [] for sval in childiter(stype): value = (str(stype.text(0)), str(sval.text(0))) if self.gdsSelectionStates.get(sval.key, True): selected_values.append(value) all_values.append(value) if selected_values: samples.extend(selected_values) used_types.append(str(stype.text(0))) else: # If no sample of sample type is selected we don't filter # on it. samples.extend(all_values) unused_types.append(str(stype.text(0))) return samples, used_types def commitIf(self): if self.autoCommit: self.commit() else: self.selectionChanged = True @Slot(int, int) def progressCompleted(self, value, total): if total > 0: self.progressBarSet(100. * value / total, processEvents=False) else: pass # TODO: report 'indeterminate progress' def commit(self): if self.currentGds: self.error(0) sample_type = None self.progressBarInit(processEvents=None) _, groups = self.selectedSamples() if len(groups) == 1 and self.outputRows: sample_type = groups[0] self.setEnabled(False) self.setBlocking(True) progress = methodinvoke(self, "progressCompleted", (int, int)) def get_data(gds_id, report_genes, transpose, sample_type, title): gds_ensure_downloaded(gds_id, progress) gds = geo.GDS(gds_id) data = gds.getdata( report_genes=report_genes, transpose=transpose, sample_type=sample_type ) data.name = title return data get_data = partial( get_data, self.currentGds["dataset_id"], report_genes=self.mergeSpots, transpose=self.outputRows, sample_type=sample_type, title=self.datasetName or self.currentGds["title"] ) self._datatask = Task(function=get_data) self._datatask.finished.connect(self._on_dataready) self._executor.submit(self._datatask) def _on_dataready(self): self.setEnabled(True) self.setBlocking(False) self.progressBarFinished(processEvents=False) try: data = self._datatask.result() except urlrequest.URLError as error: self.error(0, ("Error while connecting to the NCBI ftp server! " "'%s'" % error)) sys.excepthook(type(error), error, getattr(error, "__traceback__")) return finally: self._datatask = None data_name = data.name samples, _ = self.selectedSamples() self.warning(0) message = None if self.outputRows: def samplesinst(ex): out = [] for meta in data.domain.metas: out.append((meta.name, ex[meta].value)) if data.domain.class_var.name != 'class': out.append((data.domain.class_var.name, ex[data.domain.class_var].value)) return out samples = set(samples) mask = [samples.issuperset(samplesinst(ex)) for ex in data] data = data[numpy.array(mask, dtype=bool)] if len(data) == 0: message = "No samples with selected sample annotations." else: samples = set(samples) domain = Orange.data.Domain( [attr for attr in data.domain.attributes if samples.issuperset(attr.attributes.items())], data.domain.class_var, data.domain.metas ) # domain.addmetas(data.domain.getmetas()) if len(domain.attributes) == 0: message = "No samples with selected sample annotations." stypes = set(s[0] for s in samples) for attr in domain.attributes: attr.attributes = dict( (key, value) for key, value in attr.attributes.items() if key in stypes ) data = Orange.data.Table(domain, data) if message is not None: self.warning(0, message) data_hints.set_hint(data, "taxid", self.currentGds.get("taxid", ""), 10.0) data_hints.set_hint(data, "genesinrows", self.outputRows, 10.0) data.name = data_name self.send("Expression Data", data) model = self.treeWidget.model().sourceModel() row = self.gds.index(self.currentGds) model.setData(model.index(row, 0), " ", Qt.DisplayRole) self.updateInfo() self.selectionChanged = False def splitterMoved(self, *args): self.splitterSettings = [bytes(sp.saveState()) for sp in self.splitters] def send_report(self): self.report_items("GEO Dataset", [("ID", self.currentGds['dataset_id']), ("Title", self.currentGds['title']), ("Organism", self.currentGds['sample_organism'])]) self.report_items("Data", [("Samples", self.currentGds['sample_count']), ("Features", self.currentGds['feature_count']), ("Genes", self.currentGds['gene_count'])]) self.report_name("Sample annotations") subsets = defaultdict(list) for subset in self.currentGds['subsets']: subsets[subset['type']].append((subset['description'], len(subset['sample_id']))) self.report_html += "<ul>" for type in subsets: self.report_html += "<b>" + type + ":</b></br>" for desc, count in subsets[type]: self.report_html += 9 * " " + "<b>{}:</b> {}</br>".format(desc, count) self.report_html += "</ul>" def onDeleteWidget(self): if self._inittask: self._inittask.future().cancel() self._inittask.finished.disconnect(self._initializemodel) if self._datatask: self._datatask.future().cancel() self._datatask.finished.disconnect(self._on_dataready) self._executor.shutdown(wait=False) super(OWGEODatasets, self).onDeleteWidget() def onNameEdited(self): if self.currentGds: gds_id = self.currentGds["dataset_id"] self.datasetNames[gds_id] = unicode(self.nameEdit.text()) self.commitIf()
class OWKEGGPathwayBrowser(widget.OWWidget): name = "KEGG Pathways" description = "Browse KEGG pathways that include an input set of genes." icon = "../widgets/icons/OWKEGGPathwayBrowser.svg" priority = 70 inputs = [("Data", Orange.data.Table, "SetData", widget.Default), ("Reference", Orange.data.Table, "SetRefData")] outputs = [("Selected Data", Orange.data.Table, widget.Default), ("Unselected Data", Orange.data.Table)] autoCommit = settings.Setting(False) autoResize = settings.Setting(True) useReference = settings.Setting(False) showOrthology = settings.Setting(True) Ready, Initializing, Running = 0, 1, 2 class Error(widget.OWWidget.Error): missing_annotation = widget.Msg(ERROR_ON_MISSING_ANNOTATION) missing_gene_id = widget.Msg(ERROR_ON_MISSING_GENE_ID) missing_tax_id = widget.Msg(ERROR_ON_MISSING_TAX_ID) def __init__(self, parent=None): super().__init__(parent) self._changedFlag = False self.__invalidated = False self.__runstate = OWKEGGPathwayBrowser.Initializing self.__in_setProgress = False self.controlArea.setMaximumWidth(250) box = gui.widgetBox(self.controlArea, "Info") self.infoLabel = gui.widgetLabel(box, "No data on input\n") gui.separator(self.controlArea) gui.checkBox(self.controlArea, self, "useReference", "From signal", box="Reference", callback=self.Update) gui.separator(self.controlArea) gui.checkBox( self.controlArea, self, "showOrthology", "Show pathways in full orthology", box="Orthology", callback=self.UpdateListView, ) gui.checkBox( self.controlArea, self, "autoResize", "Resize to fit", box="Image", callback=self.UpdatePathwayViewTransform, ) box = gui.widgetBox(self.controlArea, "Cache Control") gui.button( box, self, "Clear cache", callback=self.ClearCache, tooltip="Clear all locally cached KEGG data.", default=False, autoDefault=False, ) gui.separator(self.controlArea) gui.auto_commit(self.controlArea, self, "autoCommit", "Commit") gui.rubber(self.controlArea) spliter = QSplitter(Qt.Vertical, self.mainArea) self.pathwayView = PathwayView(self, spliter) self.pathwayView.scene().selectionChanged.connect( self._onSelectionChanged) self.mainArea.layout().addWidget(spliter) self.listView = QTreeWidget(allColumnsShowFocus=True, selectionMode=QTreeWidget.SingleSelection, sortingEnabled=True, maximumHeight=200) spliter.addWidget(self.listView) self.listView.setColumnCount(4) self.listView.setHeaderLabels( ["Pathway", "P value", "Genes", "Reference"]) self.listView.itemSelectionChanged.connect(self.UpdatePathwayView) select = QAction("Select All", self, shortcut=QKeySequence.SelectAll) select.triggered.connect(self.selectAll) self.addAction(select) self.data = None self.input_genes = [] self.tax_id = None self.use_attr_names = None self.gene_id_attribute = None self.gene_id_column = None self.ref_data = None self.ref_genes = [] self.ref_tax_id = None self.ref_use_attr_names = None self.ref_gene_id_attribute = None self.ref_gene_id_column = None self.pathways = {} self.ncbi_gene_map = [] self.org = None self._executor = concurrent.ThreadExecutor() self.setEnabled(False) self.setBlocking(True) progress = concurrent.methodinvoke(self, "setProgress", (float, )) def get_genome(): """Return a KEGGGenome with the common org entries precached.""" genome = kegg.KEGGGenome() essential = genome.essential_organisms() common = genome.common_organisms() # Remove duplicates of essential from common. # (essential + common list as defined here will be used in the # GUI.) common = [c for c in common if c not in essential] # TODO: Add option to specify additional organisms not # in the common list. keys = list(map(genome.org_code_to_entry_key, essential + common)) genome.pre_cache(keys, progress_callback=progress) return (keys, genome) self._genomeTask = task = concurrent.Task(function=get_genome) task.finished.connect(self.__initialize_finish) self.progressBarInit() self.infoLabel.setText("Fetching organism definitions\n") self._executor.submit(task) def __initialize_finish(self): if self.__runstate != OWKEGGPathwayBrowser.Initializing: return try: keys, genome = self._genomeTask.result() except Exception as err: self.error(0, str(err)) raise self.progressBarFinished() self.setEnabled(True) self.setBlocking(False) self.infoLabel.setText("No data on input\n") def clear(self): """ Clear the widget state. """ self.pathways = {} self.org = None self.infoLabel.setText("No data on input\n") self.listView.clear() self.pathwayView.SetPathway(None) self.send("Selected Data", None) self.send("Unselected Data", None) def SetData(self, data=None): if self.__runstate == OWKEGGPathwayBrowser.Initializing: self.__initialize_finish() self.Error.clear() if data: self.data = data self.tax_id = str(self.data.attributes.get(TAX_ID, None)) self.use_attr_names = self.data.attributes.get( GENE_AS_ATTRIBUTE_NAME, None) self.gene_id_attribute = self.data.attributes.get( GENE_ID_ATTRIBUTE, None) self.gene_id_column = self.data.attributes.get( GENE_ID_COLUMN, None) if not (self.use_attr_names is not None and ((self.gene_id_attribute is None) ^ (self.gene_id_column is None))): if self.tax_id is None: self.Error.missing_annotation() return self.Error.missing_gene_id() return elif self.tax_id is None: self.Error.missing_tax_id() return self.warning(0) self.error(0) self.information(0) self.__invalidated = True else: self.clear() def SetRefData(self, data=None): self.information(1) if data is not None and self.useReference: self.ref_data = data self.ref_tax_id = str(self.ref_data.attributes.get(TAX_ID, None)) self.ref_use_attr_names = self.ref_data.attributes.get( GENE_AS_ATTRIBUTE_NAME, None) self.ref_gene_id_attribute = self.ref_data.attributes.get( GENE_ID_ATTRIBUTE, None) self.ref_gene_id_column = self.ref_data.attributes.get( GENE_ID_COLUMN, None) if not (self.ref_use_attr_names is not None and ((self.ref_gene_id_attribute is None) ^ (self.ref_gene_id_column is None))): if self.ref_tax_id is None: self.Error.missing_annotation() return self.Error.missing_gene_id() return elif self.ref_tax_id is None: self.Error.missing_tax_id() return self.__invalidated = True def handleNewSignals(self): if self.__invalidated: self.Update() self.__invalidated = False def UpdateListView(self): self.bestPValueItem = None self.listView.clear() if not self.data: return allPathways = self.org.pathways() allRefPathways = kegg.pathways("map") items = [] kegg_pathways = kegg.KEGGPathways() org_code = self.org.org_code if self.showOrthology: self.koOrthology = kegg.KEGGBrite("ko00001") self.listView.setRootIsDecorated(True) path_ids = {s[-5:] for s in self.pathways.keys()} def _walkCollect(koEntry): num = koEntry.title[:5] if koEntry.title else None if num in path_ids: return [koEntry] + reduce( lambda li, c: li + _walkCollect(c), [child for child in koEntry.entries], []) else: c = reduce(lambda li, c: li + _walkCollect(c), [child for child in koEntry.entries], []) return c + (c and [koEntry] or []) allClasses = reduce(lambda li1, li2: li1 + li2, [_walkCollect(c) for c in self.koOrthology], []) def _walkCreate(koEntry, lvItem): item = QTreeWidgetItem(lvItem) id = "path:" + org_code + koEntry.title[:5] if koEntry.title[:5] in path_ids: p = kegg_pathways.get_entry(id) if p is None: # In case the genesets still have obsolete entries name = koEntry.title else: name = p.name genes, p_value, ref = self.pathways[id] item.setText(0, name) item.setText(1, "%.5f" % p_value) item.setText( 2, "%i of %i" % (len(genes), len(self.input_genes))) item.setText(3, "%i of %i" % (ref, len(self.ref_genes))) item.pathway_id = id if p is not None else None else: if id in allPathways: text = kegg_pathways.get_entry(id).name else: text = koEntry.title item.setText(0, text) if id in allPathways: item.pathway_id = id elif "path:map" + koEntry.title[:5] in allRefPathways: item.pathway_id = "path:map" + koEntry.title[:5] else: item.pathway_id = None for child in koEntry.entries: if child in allClasses: _walkCreate(child, item) for koEntry in self.koOrthology: if koEntry in allClasses: _walkCreate(koEntry, self.listView) self.listView.update() else: self.listView.setRootIsDecorated(False) pathways = self.pathways.items() pathways = sorted(pathways, key=lambda item: item[1][1]) for id, (genes, p_value, ref) in pathways: item = QTreeWidgetItem(self.listView) item.setText(0, kegg_pathways.get_entry(id).name) item.setText(1, "%.5f" % p_value) item.setText(2, "%i of %i" % (len(genes), len(self.input_genes))) item.setText(3, "%i of %i" % (ref, len(self.ref_genes))) item.pathway_id = id items.append(item) self.bestPValueItem = items and items[0] or None self.listView.expandAll() for i in range(4): self.listView.resizeColumnToContents(i) if self.bestPValueItem: index = self.listView.indexFromItem(self.bestPValueItem) self.listView.selectionModel().select( index, QItemSelectionModel.ClearAndSelect) def UpdatePathwayView(self): items = self.listView.selectedItems() if len(items) > 0: item = items[0] else: item = None self.commit() item = item or self.bestPValueItem if not item or not item.pathway_id: self.pathwayView.SetPathway(None) return def get_kgml_and_image(pathway_id): """Return an initialized KEGGPathway with pre-cached data""" p = kegg.KEGGPathway(pathway_id) p._get_kgml() # makes sure the kgml file is downloaded p._get_image_filename() # makes sure the image is downloaded return (pathway_id, p) self.setEnabled(False) self._pathwayTask = concurrent.Task( function=lambda: get_kgml_and_image(item.pathway_id)) self._pathwayTask.finished.connect(self._onPathwayTaskFinshed) self._executor.submit(self._pathwayTask) def _onPathwayTaskFinshed(self): self.setEnabled(True) pathway_id, self.pathway = self._pathwayTask.result() objects = self.pathways.get(pathway_id, [[]])[0] # [ncbi_gene_id] # map ncbi_gene_id to keg_id for display objects = flatten(relation_map(self.ncbi_gene_map, objects)) self.pathwayView.SetPathway(self.pathway, objects) def UpdatePathwayViewTransform(self): self.pathwayView.updateTransform() def Update(self): """ Update (recompute enriched pathways) the widget state. """ if not self.data: return self.error(0) self.information(0) # XXX: Check data in setData, do not even allow this to be executed if # data has no genes try: self.__get_input_genes() self.input_genes = set(self.input_genes) except ValueError: self.error(0, "Cannot extract gene names from input.") self.information(1) self.org = kegg.KEGGOrganism(kegg.from_taxid(self.tax_id)) if self.useReference and self.ref_data: self.__get_ref_genes() self.ref_genes = set(self.ref_genes) else: self.ref_genes = self.org.get_ncbi_ids() def run_enrichment(org_code, genes, reference, progress=None): # We use the kegg pathway gene sets provided by 'geneset' for # the enrichment calculation. kegg_api = kegg.api.CachedKeggApi() link_map = kegg_api.link(org_code, "pathway") # [(pathway_id, kegg_gene_id)] ncbi_gene_map = kegg_api.conv( org_code, 'ncbi-geneid') # [(ncbi_gene_id, kegg_gene_id)] ncbi_gene_map = [(_1.split(":", 1)[1], _2) for _1, _2 in ncbi_gene_map] link_map = relation_join( link_map, [(_2, _1) for _1, _2 in ncbi_gene_map]) # [(pathway_id, ncbi_gene_id)] kegg_sets = relation_list_to_multimap( link_map) # {pathway_id -> [ncbi_gene_ids]} # map kegg gene ids to ncbi_gene_ids. kegg_sets = geneset.GeneSets(sets=[ geneset.GeneSet(gs_id=ddi, genes=set(genes)) for ddi, genes in kegg_sets.items() ]) pathways = pathway_enrichment(kegg_sets, genes, reference, callback=progress) # Ensure that pathway entries are pre-cached for later use in the # list/tree view kegg_pathways = kegg.KEGGPathways() kegg_pathways.pre_cache(pathways.keys(), progress_callback=progress) return pathways, ncbi_gene_map self.progressBarInit() self.setEnabled(False) self.infoLabel.setText("Retrieving...\n") progress = concurrent.methodinvoke(self, "setProgress", (float, )) run_func = partial(run_enrichment, self.org.org_code, self.input_genes, self.ref_genes, progress) self._enrichTask = concurrent.Task(function=run_func) self._enrichTask.finished.connect(self._onEnrichTaskFinished) self._executor.submit(self._enrichTask) def _onEnrichTaskFinished(self): self.setEnabled(True) self.setBlocking(False) try: pathways, ncbi_gene_map = self._enrichTask.result() except Exception: raise self.progressBarFinished() self.pathways = pathways self.ncbi_gene_map = ncbi_gene_map if not self.pathways: self.warning(0, "No enriched pathways found.") else: self.warning(0) self.infoLabel.setText("{} unique gene names on input\n".format( len(set(self.input_genes)))) self.UpdateListView() @Slot(float) def setProgress(self, value): if self.__in_setProgress: return self.__in_setProgress = True self.progressBarSet(value) self.__in_setProgress = False def __get_input_genes(self): """ Extract and return gene names from `data`. """ self.input_genes = [] if self.use_attr_names: for variable in self.data.domain.attributes: self.input_genes.append( str(variable.attributes.get(self.gene_id_attribute, '?'))) else: genes, _ = self.data.get_column_view(self.gene_id_column) self.input_genes = [str(g) for g in genes] if len(self.input_genes) <= 0: raise ValueError("No gene names in data.") def __get_ref_genes(self): """ Extract and return gene names from `data`. """ self.ref_genes = [] if self.ref_use_attr_names: for variable in self.ref_data.domain.attributes: self.ref_genes.append( str( variable.attributes.get(self.ref_gene_id_attribute, '?'))) else: genes, _ = self.ref_data.get_column_view(self.ref_gene_id_column) self.ref_genes = [str(g) for g in genes] def selectAll(self): """ Select all items in the pathway view. """ changed = False scene = self.pathwayView.scene() with disconnected(scene.selectionChanged, self._onSelectionChanged): for item in scene.items(): if item.flags( ) & QGraphicsItem.ItemIsSelectable and not item.isSelected(): item.setSelected(True) changed = True if changed: self._onSelectionChanged() def _onSelectionChanged(self): # Item selection in the pathwayView/scene has changed self.commit() def commit(self): if self.data: selectedItems = self.pathwayView.scene().selectedItems() selectedGenes = reduce( set.union, [item.marked_objects for item in selectedItems], set()) # map kegg_ids back to ncbi_gene_id backmap = [(_2, _1) for _1, _2 in self.ncbi_gene_map] selectedGenes = set(flatten(relation_map(backmap, selectedGenes))) if self.use_attr_names: selected = [ column for column in self.data.domain.attributes if self.gene_id_attribute in column.attributes and str(column.attributes[ self.gene_id_attribute]) in selectedGenes ] data = self.data[:, selected] self.send("Selected Data", data) else: selected_indices = [] other_indices = [] for row_index, row in enumerate(self.data): gene_in_row = str(row[self.gene_id_column]) if gene_in_row in self.input_genes and gene_in_row in selectedGenes: selected_indices.append(row_index) else: other_indices.append(row_index) if selected_indices: selected = self.data[selected_indices] else: selected = None if other_indices: other = self.data[other_indices] else: other = None self.send("Selected Data", selected) self.send("Unselected Data", other) else: self.send("Selected Data", None) self.send("Unselected Data", None) def ClearCache(self): kegg.caching.clear_cache() def onDeleteWidget(self): """ Called before the widget is removed from the canvas. """ super().onDeleteWidget() self.org = None self._executor.shutdown(wait=False) gc.collect() # Force collection (WHY?) def sizeHint(self): return QSize(1024, 720)
class OWKEGGPathwayBrowser(widget.OWWidget): name = "KEGG Pathways" description = "Browse KEGG pathways that include an input set of genes." icon = "../widgets/icons/KEGGPathways.svg" priority = 2030 inputs = [("Data", Orange.data.Table, "SetData", widget.Default), ("Reference", Orange.data.Table, "SetRefData")] outputs = [("Selected Data", Orange.data.Table, widget.Default), ("Unselected Data", Orange.data.Table)] settingsHandler = settings.DomainContextHandler() organismIndex = settings.ContextSetting(0) geneAttrIndex = settings.ContextSetting(0) useAttrNames = settings.ContextSetting(False) autoCommit = settings.Setting(False) autoResize = settings.Setting(True) useReference = settings.Setting(False) showOrthology = settings.Setting(True) Ready, Initializing, Running = 0, 1, 2 def __init__(self, parent=None): super().__init__(parent) self.organismCodes = [] self._changedFlag = False self.__invalidated = False self.__runstate = OWKEGGPathwayBrowser.Initializing self.__in_setProgress = False self.controlArea.setMaximumWidth(250) box = gui.widgetBox(self.controlArea, "Info") self.infoLabel = gui.widgetLabel(box, "No data on input\n") # Organism selection. box = gui.widgetBox(self.controlArea, "Organism") self.organismComboBox = gui.comboBox( box, self, "organismIndex", items=[], callback=self.Update, addSpace=True, tooltip="Select the organism of the input genes", ) # Selection of genes attribute box = gui.widgetBox(self.controlArea, "Gene attribute") self.geneAttrCandidates = itemmodels.VariableListModel(parent=self) self.geneAttrCombo = gui.comboBox(box, self, "geneAttrIndex", callback=self.Update) self.geneAttrCombo.setModel(self.geneAttrCandidates) gui.checkBox( box, self, "useAttrNames", "Use variable names", disables=[(-1, self.geneAttrCombo)], callback=self.Update ) self.geneAttrCombo.setDisabled(bool(self.useAttrNames)) gui.separator(self.controlArea) gui.checkBox(self.controlArea, self, "useReference", "From signal", box="Reference", callback=self.Update) gui.separator(self.controlArea) gui.checkBox( self.controlArea, self, "showOrthology", "Show pathways in full orthology", box="Orthology", callback=self.UpdateListView, ) gui.checkBox( self.controlArea, self, "autoResize", "Resize to fit", box="Image", callback=self.UpdatePathwayViewTransform ) box = gui.widgetBox(self.controlArea, "Cache Control") gui.button( box, self, "Clear cache", callback=self.ClearCache, tooltip="Clear all locally cached KEGG data.", default=False, autoDefault=False, ) gui.separator(self.controlArea) gui.auto_commit(self.controlArea, self, "autoCommit", "Commit") gui.rubber(self.controlArea) spliter = QSplitter(Qt.Vertical, self.mainArea) self.pathwayView = PathwayView(self, spliter) self.pathwayView.scene().selectionChanged.connect(self._onSelectionChanged) self.mainArea.layout().addWidget(spliter) self.listView = QTreeWidget( allColumnsShowFocus=True, selectionMode=QTreeWidget.SingleSelection, sortingEnabled=True, maximumHeight=200 ) spliter.addWidget(self.listView) self.listView.setColumnCount(4) self.listView.setHeaderLabels(["Pathway", "P value", "Genes", "Reference"]) self.listView.itemSelectionChanged.connect(self.UpdatePathwayView) select = QAction("Select All", self, shortcut=QKeySequence.SelectAll) select.triggered.connect(self.selectAll) self.addAction(select) self.data = None self.refData = None self._executor = concurrent.ThreadExecutor() self.setEnabled(False) self.setBlocking(True) progress = concurrent.methodinvoke(self, "setProgress", (float,)) def get_genome(): """Return a KEGGGenome with the common org entries precached.""" genome = kegg.KEGGGenome() essential = genome.essential_organisms() common = genome.common_organisms() # Remove duplicates of essential from common. # (essential + common list as defined here will be used in the # GUI.) common = [c for c in common if c not in essential] # TODO: Add option to specify additional organisms not # in the common list. keys = list(map(genome.org_code_to_entry_key, essential + common)) genome.pre_cache(keys, progress_callback=progress) return (keys, genome) self._genomeTask = task = concurrent.Task(function=get_genome) task.finished.connect(self.__initialize_finish) self.progressBarInit() self.infoLabel.setText("Fetching organism definitions\n") self._executor.submit(task) def __initialize_finish(self): if self.__runstate != OWKEGGPathwayBrowser.Initializing: return try: keys, genome = self._genomeTask.result() except Exception as err: self.error(0, str(err)) raise self.progressBarFinished() self.setEnabled(True) self.setBlocking(False) entries = [genome[key] for key in keys] items = [entry.definition for entry in entries] codes = [entry.organism_code for entry in entries] self.organismCodes = codes self.organismComboBox.clear() self.organismComboBox.addItems(items) self.organismComboBox.setCurrentIndex(self.organismIndex) self.infoLabel.setText("No data on input\n") def Clear(self): """ Clear the widget state. """ self.queryGenes = [] self.referenceGenes = [] self.genes = {} self.uniqueGenesDict = {} self.revUniqueGenesDict = {} self.pathways = {} self.org = None self.geneAttrCandidates[:] = [] self.infoLabel.setText("No data on input\n") self.listView.clear() self.pathwayView.SetPathway(None) self.send("Selected Data", None) self.send("Unselected Data", None) def SetData(self, data=None): if self.__runstate == OWKEGGPathwayBrowser.Initializing: self.__initialize_finish() self.closeContext() self.data = data self.warning(0) self.error(0) self.information(0) if data is not None: vars = data.domain.variables + data.domain.metas vars = [var for var in vars if isinstance(var, Orange.data.StringVariable)] self.geneAttrCandidates[:] = vars # Try to guess the gene name variable if vars: names_lower = [v.name.lower() for v in vars] scores = [(name == "gene", "gene" in name) for name in names_lower] imax, _ = max(enumerate(scores), key=itemgetter(1)) else: imax = -1 self.geneAttrIndex = imax taxid = data_hints.get_hint(data, "taxid", None) if taxid: try: code = kegg.from_taxid(taxid) self.organismIndex = self.organismCodes.index(code) except Exception as ex: print(ex, taxid) self.useAttrNames = data_hints.get_hint(data, "genesinrows", self.useAttrNames) self.openContext(data) if len(self.geneAttrCandidates) == 0: self.useAttrNames = True self.geneAttrIndex = -1 else: self.geneAttrIndex = min(self.geneAttrIndex, len(self.geneAttrCandidates) - 1) else: self.Clear() self.__invalidated = True def SetRefData(self, data=None): self.refData = data self.information(1) if data is not None and self.useReference: self.__invalidated = True def handleNewSignals(self): if self.__invalidated: self.Update() self.__invalidated = False def UpdateListView(self): self.bestPValueItem = None self.listView.clear() if not self.data: return allPathways = self.org.pathways() allRefPathways = kegg.pathways("map") items = [] kegg_pathways = kegg.KEGGPathways() org_code = self.organismCodes[min(self.organismIndex, len(self.organismCodes) - 1)] if self.showOrthology: self.koOrthology = kegg.KEGGBrite("ko00001") self.listView.setRootIsDecorated(True) path_ids = set([s[-5:] for s in self.pathways.keys()]) def _walkCollect(koEntry): num = koEntry.title[:5] if koEntry.title else None if num in path_ids: return [koEntry] + reduce( lambda li, c: li + _walkCollect(c), [child for child in koEntry.entries], [] ) else: c = reduce(lambda li, c: li + _walkCollect(c), [child for child in koEntry.entries], []) return c + (c and [koEntry] or []) allClasses = reduce(lambda li1, li2: li1 + li2, [_walkCollect(c) for c in self.koOrthology], []) def _walkCreate(koEntry, lvItem): item = QTreeWidgetItem(lvItem) id = "path:" + org_code + koEntry.title[:5] if koEntry.title[:5] in path_ids: p = kegg_pathways.get_entry(id) if p is None: # In case the genesets still have obsolete entries name = koEntry.title else: name = p.name genes, p_value, ref = self.pathways[id] item.setText(0, name) item.setText(1, "%.5f" % p_value) item.setText(2, "%i of %i" % (len(genes), len(self.genes))) item.setText(3, "%i of %i" % (ref, len(self.referenceGenes))) item.pathway_id = id if p is not None else None else: if id in allPathways: text = kegg_pathways.get_entry(id).name else: text = koEntry.title item.setText(0, text) if id in allPathways: item.pathway_id = id elif "path:map" + koEntry.title[:5] in allRefPathways: item.pathway_id = "path:map" + koEntry.title[:5] else: item.pathway_id = None for child in koEntry.entries: if child in allClasses: _walkCreate(child, item) for koEntry in self.koOrthology: if koEntry in allClasses: _walkCreate(koEntry, self.listView) self.listView.update() else: self.listView.setRootIsDecorated(False) pathways = self.pathways.items() pathways = sorted(pathways, key=lambda item: item[1][1]) for id, (genes, p_value, ref) in pathways: item = QTreeWidgetItem(self.listView) item.setText(0, kegg_pathways.get_entry(id).name) item.setText(1, "%.5f" % p_value) item.setText(2, "%i of %i" % (len(genes), len(self.genes))) item.setText(3, "%i of %i" % (ref, len(self.referenceGenes))) item.pathway_id = id items.append(item) self.bestPValueItem = items and items[0] or None self.listView.expandAll() for i in range(4): self.listView.resizeColumnToContents(i) if self.bestPValueItem: index = self.listView.indexFromItem(self.bestPValueItem) self.listView.selectionModel().select(index, QItemSelectionModel.ClearAndSelect) def UpdatePathwayView(self): items = self.listView.selectedItems() if len(items) > 0: item = items[0] else: item = None self.commit() item = item or self.bestPValueItem if not item or not item.pathway_id: self.pathwayView.SetPathway(None) return def get_kgml_and_image(pathway_id): """Return an initialized KEGGPathway with pre-cached data""" p = kegg.KEGGPathway(pathway_id) p._get_kgml() # makes sure the kgml file is downloaded p._get_image_filename() # makes sure the image is downloaded return (pathway_id, p) self.setEnabled(False) self._pathwayTask = concurrent.Task(function=lambda: get_kgml_and_image(item.pathway_id)) self._pathwayTask.finished.connect(self._onPathwayTaskFinshed) self._executor.submit(self._pathwayTask) def _onPathwayTaskFinshed(self): self.setEnabled(True) pathway_id, self.pathway = self._pathwayTask.result() self.pathwayView.SetPathway(self.pathway, self.pathways.get(pathway_id, [[]])[0]) def UpdatePathwayViewTransform(self): self.pathwayView.updateTransform() def Update(self): """ Update (recompute enriched pathways) the widget state. """ if not self.data: return self.error(0) self.information(0) # XXX: Check data in setData, do not even alow this to be executed if # data has no genes try: genes = self.GeneNamesFromData(self.data) except ValueError: self.error(0, "Cannot extract gene names from input.") genes = [] if not self.useAttrNames and any("," in gene for gene in genes): genes = reduce(add, (split_and_strip(gene, ",") for gene in genes), []) self.information(0, "Separators detected in input gene names. " "Assuming multiple genes per instance.") self.queryGenes = genes self.information(1) reference = None if self.useReference and self.refData: reference = self.GeneNamesFromData(self.refData) if not self.useAttrNames and any("," in gene for gene in reference): reference = reduce(add, (split_and_strip(gene, ",") for gene in reference), []) self.information( 1, "Separators detected in reference gene " "names. Assuming multiple genes per " "instance." ) org_code = self.SelectedOrganismCode() def run_enrichment(org_code, genes, reference=None, progress=None): org = kegg.KEGGOrganism(org_code) if reference is None: reference = org.get_genes() # Map 'genes' and 'reference' sets to unique KEGG identifiers unique_genes, _, _ = org.get_unique_gene_ids(set(genes)) unique_ref_genes, _, _ = org.get_unique_gene_ids(set(reference)) taxid = kegg.to_taxid(org.org_code) # Map the taxid back to standard 'common' taxids # (as used by 'geneset') if applicable r_tax_map = dict((v, k) for k, v in kegg.KEGGGenome.TAXID_MAP.items()) if taxid in r_tax_map: taxid = r_tax_map[taxid] # We use the kegg pathway gene sets provided by 'geneset' for # the enrichment calculation. # Ensure we are using the latest genesets # TODO: ?? Is updating the index enough? serverfiles.update(geneset.sfdomain, "index.pck") kegg_gs_collections = geneset.collections((("KEGG", "pathways"), taxid)) pathways = pathway_enrichment( kegg_gs_collections, unique_genes.keys(), unique_ref_genes.keys(), callback=progress ) # Ensure that pathway entries are pre-cached for later use in the # list/tree view kegg_pathways = kegg.KEGGPathways() kegg_pathways.pre_cache(pathways.keys(), progress_callback=progress) return pathways, org, unique_genes, unique_ref_genes self.progressBarInit() self.setEnabled(False) self.infoLabel.setText("Retrieving...\n") progress = concurrent.methodinvoke(self, "setProgress", (float,)) self._enrichTask = concurrent.Task(function=lambda: run_enrichment(org_code, genes, reference, progress)) self._enrichTask.finished.connect(self._onEnrichTaskFinished) self._executor.submit(self._enrichTask) def _onEnrichTaskFinished(self): self.setEnabled(True) self.setBlocking(False) try: pathways, org, unique_genes, unique_ref_genes = self._enrichTask.result() except Exception: raise self.progressBarFinished() self.org = org self.genes = unique_genes.keys() self.uniqueGenesDict = unique_genes self.revUniqueGenesDict = dict([(val, key) for key, val in self.uniqueGenesDict.items()]) self.referenceGenes = unique_ref_genes.keys() self.pathways = pathways if not self.pathways: self.warning(0, "No enriched pathways found.") else: self.warning(0) count = len(set(self.queryGenes)) self.infoLabel.setText( "%i unique gene names on input\n" "%i (%.1f%%) genes names matched" % (count, len(unique_genes), 100.0 * len(unique_genes) / count if count else 0.0) ) self.UpdateListView() @Slot(float) def setProgress(self, value): if self.__in_setProgress: return self.__in_setProgress = True self.progressBarSet(value) self.__in_setProgress = False def GeneNamesFromData(self, data): """ Extract and return gene names from `data`. """ if self.useAttrNames: genes = [str(v.name).strip() for v in data.domain.attributes] elif self.geneAttrCandidates: assert 0 <= self.geneAttrIndex < len(self.geneAttrCandidates) geneAttr = self.geneAttrCandidates[self.geneAttrIndex] genes = [str(e[geneAttr]) for e in data if not numpy.isnan(e[geneAttr])] else: raise ValueError("No gene names in data.") return genes def SelectedOrganismCode(self): """ Return the selected organism code. """ return self.organismCodes[min(self.organismIndex, len(self.organismCodes) - 1)] def selectAll(self): """ Select all items in the pathway view. """ changed = False scene = self.pathwayView.scene() with disconnected(scene.selectionChanged, self._onSelectionChanged): for item in scene.items(): if item.flags() & QGraphicsItem.ItemIsSelectable and not item.isSelected(): item.setSelected(True) changed = True if changed: self._onSelectionChanged() def _onSelectionChanged(self): # Item selection in the pathwayView/scene has changed self.commit() def commit(self): if self.data: selectedItems = self.pathwayView.scene().selectedItems() selectedGenes = reduce(set.union, [item.marked_objects for item in selectedItems], set()) if self.useAttrNames: selected = [self.data.domain[self.uniqueGenesDict[gene]] for gene in selectedGenes] # newDomain = Orange.data.Domain(selectedVars, 0) data = self.data[:, selected] # data = Orange.data.Table(newDomain, self.data) self.send("Selected Data", data) elif self.geneAttrCandidates: assert 0 <= self.geneAttrIndex < len(self.geneAttrCandidates) geneAttr = self.geneAttrCandidates[self.geneAttrIndex] selectedIndices = [] otherIndices = [] for i, ex in enumerate(self.data): names = [ self.revUniqueGenesDict.get(name, None) for name in split_and_strip(str(ex[geneAttr]), ",") ] if any(name and name in selectedGenes for name in names): selectedIndices.append(i) else: otherIndices.append(i) if selectedIndices: selected = self.data[selectedIndices] else: selected = None if otherIndices: other = self.data[otherIndices] else: other = None self.send("Selected Data", selected) self.send("Unselected Data", other) else: self.send("Selected Data", None) self.send("Unselected Data", None) def ClearCache(self): kegg.caching.clear_cache() def onDeleteWidget(self): """ Called before the widget is removed from the canvas. """ super().onDeleteWidget() self.org = None self._executor.shutdown(wait=False) gc.collect() # Force collection (WHY?) def sizeHint(self): return QSize(1024, 720)
class OWKEGGPathwayBrowser(widget.OWWidget): name = "KEGG Pathways" description = "Browse KEGG pathways that include an input set of genes." icon = "../widgets/icons/OWKEGGPathwayBrowser.svg" priority = 8 inputs = [("Data", Orange.data.Table, "SetData", widget.Default), ("Reference", Orange.data.Table, "SetRefData")] outputs = [("Selected Data", Orange.data.Table, widget.Default), ("Unselected Data", Orange.data.Table)] settingsHandler = settings.DomainContextHandler() organismIndex = settings.ContextSetting(0) geneAttrIndex = settings.ContextSetting(0) useAttrNames = settings.ContextSetting(False) autoCommit = settings.Setting(False) autoResize = settings.Setting(True) useReference = settings.Setting(False) showOrthology = settings.Setting(True) Ready, Initializing, Running = 0, 1, 2 def __init__(self, parent=None): super().__init__(parent) self.organismCodes = [] self._changedFlag = False self.__invalidated = False self.__runstate = OWKEGGPathwayBrowser.Initializing self.__in_setProgress = False self.controlArea.setMaximumWidth(250) box = gui.widgetBox(self.controlArea, "Info") self.infoLabel = gui.widgetLabel(box, "No data on input\n") # Organism selection. box = gui.widgetBox(self.controlArea, "Organism") self.organismComboBox = gui.comboBox( box, self, "organismIndex", items=[], callback=self.Update, addSpace=True, tooltip="Select the organism of the input genes") # Selection of genes attribute box = gui.widgetBox(self.controlArea, "Gene attribute") self.geneAttrCandidates = itemmodels.VariableListModel(parent=self) self.geneAttrCombo = gui.comboBox( box, self, "geneAttrIndex", callback=self.Update) self.geneAttrCombo.setModel(self.geneAttrCandidates) gui.checkBox(box, self, "useAttrNames", "Use variable names", disables=[(-1, self.geneAttrCombo)], callback=self.Update) self.geneAttrCombo.setDisabled(bool(self.useAttrNames)) gui.separator(self.controlArea) gui.checkBox(self.controlArea, self, "useReference", "From signal", box="Reference", callback=self.Update) gui.separator(self.controlArea) gui.checkBox(self.controlArea, self, "showOrthology", "Show pathways in full orthology", box="Orthology", callback=self.UpdateListView) gui.checkBox(self.controlArea, self, "autoResize", "Resize to fit", box="Image", callback=self.UpdatePathwayViewTransform) box = gui.widgetBox(self.controlArea, "Cache Control") gui.button(box, self, "Clear cache", callback=self.ClearCache, tooltip="Clear all locally cached KEGG data.", default=False, autoDefault=False) gui.separator(self.controlArea) gui.auto_commit(self.controlArea, self, "autoCommit", "Commit") gui.rubber(self.controlArea) spliter = QSplitter(Qt.Vertical, self.mainArea) self.pathwayView = PathwayView(self, spliter) self.pathwayView.scene().selectionChanged.connect( self._onSelectionChanged ) self.mainArea.layout().addWidget(spliter) self.listView = QTreeWidget( allColumnsShowFocus=True, selectionMode=QTreeWidget.SingleSelection, sortingEnabled=True, maximumHeight=200) spliter.addWidget(self.listView) self.listView.setColumnCount(4) self.listView.setHeaderLabels( ["Pathway", "P value", "Genes", "Reference"]) self.listView.itemSelectionChanged.connect(self.UpdatePathwayView) select = QAction( "Select All", self, shortcut=QKeySequence.SelectAll ) select.triggered.connect(self.selectAll) self.addAction(select) self.data = None self.refData = None self._executor = concurrent.ThreadExecutor() self.setEnabled(False) self.setBlocking(True) progress = concurrent.methodinvoke(self, "setProgress", (float,)) def get_genome(): """Return a KEGGGenome with the common org entries precached.""" genome = kegg.KEGGGenome() essential = genome.essential_organisms() common = genome.common_organisms() # Remove duplicates of essential from common. # (essential + common list as defined here will be used in the # GUI.) common = [c for c in common if c not in essential] # TODO: Add option to specify additional organisms not # in the common list. keys = list(map(genome.org_code_to_entry_key, essential + common)) genome.pre_cache(keys, progress_callback=progress) return (keys, genome) self._genomeTask = task = concurrent.Task(function=get_genome) task.finished.connect(self.__initialize_finish) self.progressBarInit() self.infoLabel.setText("Fetching organism definitions\n") self._executor.submit(task) def __initialize_finish(self): if self.__runstate != OWKEGGPathwayBrowser.Initializing: return try: keys, genome = self._genomeTask.result() except Exception as err: self.error(0, str(err)) raise self.progressBarFinished() self.setEnabled(True) self.setBlocking(False) entries = [genome[key] for key in keys] items = [entry.definition for entry in entries] codes = [entry.organism_code for entry in entries] self.organismCodes = codes self.organismComboBox.clear() self.organismComboBox.addItems(items) self.organismComboBox.setCurrentIndex(self.organismIndex) self.infoLabel.setText("No data on input\n") def Clear(self): """ Clear the widget state. """ self.queryGenes = [] self.referenceGenes = [] self.genes = {} self.uniqueGenesDict = {} self.revUniqueGenesDict = {} self.pathways = {} self.org = None self.geneAttrCandidates[:] = [] self.infoLabel.setText("No data on input\n") self.listView.clear() self.pathwayView.SetPathway(None) self.send("Selected Data", None) self.send("Unselected Data", None) def SetData(self, data=None): if self.__runstate == OWKEGGPathwayBrowser.Initializing: self.__initialize_finish() self.data = data self.warning(0) self.error(0) self.information(0) if data is not None: vars = data.domain.variables + data.domain.metas vars = [var for var in vars if isinstance(var, Orange.data.StringVariable)] self.geneAttrCandidates[:] = vars # Try to guess the gene name variable if vars: names_lower = [v.name.lower() for v in vars] scores = [(name == "gene", "gene" in name) for name in names_lower] imax, _ = max(enumerate(scores), key=itemgetter(1)) else: imax = -1 self.geneAttrIndex = imax taxid = data_hints.get_hint(data, TAX_ID, None) if taxid: try: code = kegg.from_taxid(taxid) self.organismIndex = self.organismCodes.index(code) except Exception as ex: print(ex, taxid) self.useAttrNames = data_hints.get_hint(data, GENE_NAME, self.useAttrNames) if len(self.geneAttrCandidates) == 0: self.useAttrNames = True self.geneAttrIndex = -1 else: self.geneAttrIndex = min(self.geneAttrIndex, len(self.geneAttrCandidates) - 1) else: self.Clear() self.__invalidated = True def SetRefData(self, data=None): self.refData = data self.information(1) if data is not None and self.useReference: self.__invalidated = True def handleNewSignals(self): if self.__invalidated: self.Update() self.__invalidated = False def UpdateListView(self): self.bestPValueItem = None self.listView.clear() if not self.data: return allPathways = self.org.pathways() allRefPathways = kegg.pathways("map") items = [] kegg_pathways = kegg.KEGGPathways() org_code = self.organismCodes[min(self.organismIndex, len(self.organismCodes) - 1)] if self.showOrthology: self.koOrthology = kegg.KEGGBrite("ko00001") self.listView.setRootIsDecorated(True) path_ids = set([s[-5:] for s in self.pathways.keys()]) def _walkCollect(koEntry): num = koEntry.title[:5] if koEntry.title else None if num in path_ids: return ([koEntry] + reduce(lambda li, c: li + _walkCollect(c), [child for child in koEntry.entries], [])) else: c = reduce(lambda li, c: li + _walkCollect(c), [child for child in koEntry.entries], []) return c + (c and [koEntry] or []) allClasses = reduce(lambda li1, li2: li1 + li2, [_walkCollect(c) for c in self.koOrthology], []) def _walkCreate(koEntry, lvItem): item = QTreeWidgetItem(lvItem) id = "path:" + org_code + koEntry.title[:5] if koEntry.title[:5] in path_ids: p = kegg_pathways.get_entry(id) if p is None: # In case the genesets still have obsolete entries name = koEntry.title else: name = p.name genes, p_value, ref = self.pathways[id] item.setText(0, name) item.setText(1, "%.5f" % p_value) item.setText(2, "%i of %i" % (len(genes), len(self.genes))) item.setText(3, "%i of %i" % (ref, len(self.referenceGenes))) item.pathway_id = id if p is not None else None else: if id in allPathways: text = kegg_pathways.get_entry(id).name else: text = koEntry.title item.setText(0, text) if id in allPathways: item.pathway_id = id elif "path:map" + koEntry.title[:5] in allRefPathways: item.pathway_id = "path:map" + koEntry.title[:5] else: item.pathway_id = None for child in koEntry.entries: if child in allClasses: _walkCreate(child, item) for koEntry in self.koOrthology: if koEntry in allClasses: _walkCreate(koEntry, self.listView) self.listView.update() else: self.listView.setRootIsDecorated(False) pathways = self.pathways.items() pathways = sorted(pathways, key=lambda item: item[1][1]) for id, (genes, p_value, ref) in pathways: item = QTreeWidgetItem(self.listView) item.setText(0, kegg_pathways.get_entry(id).name) item.setText(1, "%.5f" % p_value) item.setText(2, "%i of %i" % (len(genes), len(self.genes))) item.setText(3, "%i of %i" % (ref, len(self.referenceGenes))) item.pathway_id = id items.append(item) self.bestPValueItem = items and items[0] or None self.listView.expandAll() for i in range(4): self.listView.resizeColumnToContents(i) if self.bestPValueItem: index = self.listView.indexFromItem(self.bestPValueItem) self.listView.selectionModel().select( index, QItemSelectionModel.ClearAndSelect ) def UpdatePathwayView(self): items = self.listView.selectedItems() if len(items) > 0: item = items[0] else: item = None self.commit() item = item or self.bestPValueItem if not item or not item.pathway_id: self.pathwayView.SetPathway(None) return def get_kgml_and_image(pathway_id): """Return an initialized KEGGPathway with pre-cached data""" p = kegg.KEGGPathway(pathway_id) p._get_kgml() # makes sure the kgml file is downloaded p._get_image_filename() # makes sure the image is downloaded return (pathway_id, p) self.setEnabled(False) self._pathwayTask = concurrent.Task( function=lambda: get_kgml_and_image(item.pathway_id) ) self._pathwayTask.finished.connect(self._onPathwayTaskFinshed) self._executor.submit(self._pathwayTask) def _onPathwayTaskFinshed(self): self.setEnabled(True) pathway_id, self.pathway = self._pathwayTask.result() self.pathwayView.SetPathway( self.pathway, self.pathways.get(pathway_id, [[]])[0] ) def UpdatePathwayViewTransform(self): self.pathwayView.updateTransform() def Update(self): """ Update (recompute enriched pathways) the widget state. """ if not self.data: return self.error(0) self.information(0) # XXX: Check data in setData, do not even allow this to be executed if # data has no genes try: genes = self.GeneNamesFromData(self.data) except ValueError: self.error(0, "Cannot extract gene names from input.") genes = [] if not self.useAttrNames and any("," in gene for gene in genes): genes = reduce(add, (split_and_strip(gene, ",") for gene in genes), []) self.information(0, "Separators detected in input gene names. " "Assuming multiple genes per instance.") self.queryGenes = genes self.information(1) reference = None if self.useReference and self.refData: reference = self.GeneNamesFromData(self.refData) if not self.useAttrNames \ and any("," in gene for gene in reference): reference = reduce(add, (split_and_strip(gene, ",") for gene in reference), []) self.information(1, "Separators detected in reference gene " "names. Assuming multiple genes per " "instance.") org_code = self.SelectedOrganismCode() from orangecontrib.bioinformatics.ncbi.gene import GeneMatcher gm = GeneMatcher(kegg.to_taxid(org_code)) gm.genes = genes gm.run_matcher() mapped_genes = {gene: str(ncbi_id) for gene, ncbi_id in gm.map_input_to_ncbi().items()} def run_enrichment(org_code, genes, reference=None, progress=None): org = kegg.KEGGOrganism(org_code) if reference is None: reference = org.get_ncbi_ids() # This is here just to keep widget working without any major changes. # map not needed, geneMatcher will not work on widget level. unique_genes = genes unique_ref_genes = dict([(gene, gene) for gene in set(reference)]) taxid = kegg.to_taxid(org.org_code) # Map the taxid back to standard 'common' taxids # (as used by 'geneset') if applicable r_tax_map = dict((v, k) for k, v in kegg.KEGGGenome.TAXID_MAP.items()) if taxid in r_tax_map: taxid = r_tax_map[taxid] # We use the kegg pathway gene sets provided by 'geneset' for # the enrichment calculation. kegg_api = kegg.api.CachedKeggApi() linkmap = kegg_api.link(org.org_code, "pathway") converted_ids = kegg_api.conv(org.org_code, 'ncbi-geneid') kegg_sets = relation_list_to_multimap(linkmap, dict((gene.upper(), ncbi.split(':')[-1]) for ncbi, gene in converted_ids)) kegg_sets = geneset.GeneSets(input=kegg_sets) pathways = pathway_enrichment( kegg_sets, unique_genes.values(), unique_ref_genes.keys(), callback=progress ) # Ensure that pathway entries are pre-cached for later use in the # list/tree view kegg_pathways = kegg.KEGGPathways() kegg_pathways.pre_cache( pathways.keys(), progress_callback=progress ) return pathways, org, unique_genes, unique_ref_genes self.progressBarInit() self.setEnabled(False) self.infoLabel.setText("Retrieving...\n") progress = concurrent.methodinvoke(self, "setProgress", (float,)) self._enrichTask = concurrent.Task( function=lambda: run_enrichment(org_code, mapped_genes, reference, progress) ) self._enrichTask.finished.connect(self._onEnrichTaskFinished) self._executor.submit(self._enrichTask) def _onEnrichTaskFinished(self): self.setEnabled(True) self.setBlocking(False) try: pathways, org, unique_genes, unique_ref_genes = \ self._enrichTask.result() except Exception: raise self.progressBarFinished() self.org = org self.genes = unique_genes.keys() self.uniqueGenesDict = {ncbi_id: input_name for input_name, ncbi_id in unique_genes.items()} self.revUniqueGenesDict = dict([(val, key) for key, val in self.uniqueGenesDict.items()]) self.referenceGenes = unique_ref_genes.keys() self.pathways = pathways if not self.pathways: self.warning(0, "No enriched pathways found.") else: self.warning(0) count = len(set(self.queryGenes)) self.infoLabel.setText( "%i unique gene names on input\n" "%i (%.1f%%) genes names matched" % (count, len(unique_genes), 100.0 * len(unique_genes) / count if count else 0.0) ) self.UpdateListView() @Slot(float) def setProgress(self, value): if self.__in_setProgress: return self.__in_setProgress = True self.progressBarSet(value) self.__in_setProgress = False def GeneNamesFromData(self, data): """ Extract and return gene names from `data`. """ if self.useAttrNames: genes = [str(v.name).strip() for v in data.domain.attributes] elif self.geneAttrCandidates: assert 0 <= self.geneAttrIndex < len(self.geneAttrCandidates) geneAttr = self.geneAttrCandidates[self.geneAttrIndex] genes = [str(e[geneAttr]) for e in data if not numpy.isnan(e[geneAttr])] else: raise ValueError("No gene names in data.") return genes def SelectedOrganismCode(self): """ Return the selected organism code. """ return self.organismCodes[min(self.organismIndex, len(self.organismCodes) - 1)] def selectAll(self): """ Select all items in the pathway view. """ changed = False scene = self.pathwayView.scene() with disconnected(scene.selectionChanged, self._onSelectionChanged): for item in scene.items(): if item.flags() & QGraphicsItem.ItemIsSelectable and \ not item.isSelected(): item.setSelected(True) changed = True if changed: self._onSelectionChanged() def _onSelectionChanged(self): # Item selection in the pathwayView/scene has changed self.commit() def commit(self): if self.data: selectedItems = self.pathwayView.scene().selectedItems() selectedGenes = reduce(set.union, [item.marked_objects for item in selectedItems], set()) if self.useAttrNames: selected = [self.data.domain[self.uniqueGenesDict[gene]] for gene in selectedGenes] # newDomain = Orange.data.Domain(selectedVars, 0) data = self.data[:, selected] # data = Orange.data.Table(newDomain, self.data) self.send("Selected Data", data) elif self.geneAttrCandidates: assert 0 <= self.geneAttrIndex < len(self.geneAttrCandidates) geneAttr = self.geneAttrCandidates[self.geneAttrIndex] selectedIndices = [] otherIndices = [] for i, ex in enumerate(self.data): names = [self.revUniqueGenesDict.get(name, None) for name in split_and_strip(str(ex[geneAttr]), ",")] if any(name and name in selectedGenes for name in names): selectedIndices.append(i) else: otherIndices.append(i) if selectedIndices: selected = self.data[selectedIndices] else: selected = None if otherIndices: other = self.data[otherIndices] else: other = None self.send("Selected Data", selected) self.send("Unselected Data", other) else: self.send("Selected Data", None) self.send("Unselected Data", None) def ClearCache(self): kegg.caching.clear_cache() def onDeleteWidget(self): """ Called before the widget is removed from the canvas. """ super().onDeleteWidget() self.org = None self._executor.shutdown(wait=False) gc.collect() # Force collection (WHY?) def sizeHint(self): return QSize(1024, 720)
class OWGEODatasets(OWWidget, ConcurrentWidgetMixin): name = "GEO Data Sets" description = "Access to Gene Expression Omnibus data sets." icon = "icons/OWGEODatasets.svg" priority = 2 class Outputs: gds_data = Output("Expression Data", Table) search_pattern = Setting('') auto_commit = Setting(True) genes_as_rows = Setting(False) mergeSpots = Setting(True) selected_gds = Setting(None) gdsSelectionStates = Setting({}) splitter_settings = Setting(( b'\x00\x00\x00\xff\x00\x00\x00\x00\x00\x00\x00\x02\x00\x00\x01\xea\x00\x00\x00\xd7\x01\x00\x00\x00\x07\x01\x00\x00\x00\x02', b'\x00\x00\x00\xff\x00\x00\x00\x00\x00\x00\x00\x02\x00\x00\x01\xb5\x00\x00\x02\x10\x01\x00\x00\x00\x07\x01\x00\x00\x00\x01', )) def __init__(self): OWWidget.__init__(self) ConcurrentWidgetMixin.__init__(self) self.gds_info: Optional[GDSInfo] = GDSInfo( ) # TODO: handle possible exceptions self.gds_data: Optional[Table] = None # Control area box = widgetBox(self.controlArea, 'Info', addSpace=True) self.infoBox = widgetLabel(box, 'Initializing\n\n') box = widgetBox(self.controlArea, 'Output', addSpace=True) radioButtonsInBox(box, self, 'genes_as_rows', ['Samples in rows', 'Genes in rows'], callback=self._run) separator(box) rubber(self.controlArea) auto_commit(self.controlArea, self, 'auto_commit', '&Commit', box=False) # Main Area # Filter widget self.filter = lineEdit(self.mainArea, self, 'search_pattern', 'Filter:', callbackOnType=True, callback=self._apply_filter) self.mainArea.layout().addWidget(self.filter) splitter_vertical = QSplitter(Qt.Vertical, self.mainArea) self.mainArea.layout().addWidget(splitter_vertical) # set table view self.table_view = QTableView(splitter_vertical) self.table_view.setShowGrid(False) self.table_view.setSortingEnabled(True) self.table_view.sortByColumn(1, Qt.AscendingOrder) self.table_view.setAlternatingRowColors(True) self.table_view.verticalHeader().setVisible(False) self.table_view.setSelectionBehavior(QAbstractItemView.SelectRows) self.table_view.setSelectionMode(QAbstractItemView.SingleSelection) self.table_view.viewport().setMouseTracking(True) self.table_view.setSizeAdjustPolicy( QAbstractScrollArea.AdjustToContents) self.table_model = GEODatasetsModel() self.table_model.initialize(self.gds_info) self.table_view.setModel(self.table_model) self.table_model.show_table() self.table_view.horizontalHeader().setStretchLastSection(True) self.table_view.resizeColumnsToContents() v_header = self.table_view.verticalHeader() option = self.table_view.viewOptions() size = self.table_view.style().sizeFromContents( QStyle.CT_ItemViewItem, option, QSize(20, 20), self.table_view) v_header.setDefaultSectionSize(size.height() + 2) v_header.setMinimumSectionSize(5) # set item delegates self.table_view.setItemDelegateForColumn( self.table_model.pubmedid_col, LinkStyledItemDelegate(self.table_view)) self.table_view.setItemDelegateForColumn( self.table_model.gds_id_col, LinkStyledItemDelegate(self.table_view)) self.table_view.setItemDelegateForColumn( self.table_model.indicator_col, IndicatorItemDelegate(self.table_view, role=Qt.DisplayRole), ) splitter_horizontal = QSplitter(Qt.Horizontal, splitter_vertical) # Description Widget box = widgetBox(splitter_horizontal, 'Description') self.description_widget = widgetLabel(box, '') self.description_widget.setWordWrap(True) rubber(box) # Sample Annotations Widget box = widgetBox(splitter_horizontal, 'Sample Annotations') self.annotations_widget = QTreeWidget(box) self.annotations_widget.setHeaderLabels( ['Type (Sample annotations)', 'Sample count']) self.annotations_widget.setRootIsDecorated(True) box.layout().addWidget(self.annotations_widget) self._annotations_updating = False self.annotations_widget.itemChanged.connect( self.on_annotation_selection_changed) self.splitters = splitter_vertical, splitter_horizontal for sp, setting in zip(self.splitters, self.splitter_settings): sp.splitterMoved.connect(self._splitter_moved) sp.restoreState(setting) self.table_view.selectionModel().selectionChanged.connect( self.on_gds_selection_changed) self._apply_filter() self._run() def _splitter_moved(self, *args): self.splitter_settings = [ bytes(sp.saveState()) for sp in self.splitters ] def _set_description_widget(self): self.description_widget.setText( self.selected_gds.get('description', 'Description not available.')) def _set_annotations_widget(self, gds): self._annotations_updating = True self.annotations_widget.clear() annotations = defaultdict(set) subsets_count = {} for desc in gds['subsets']: annotations[desc['type']].add(desc['description']) subsets_count[desc['description']] = str(len(desc['sample_id'])) for _type, subsets in annotations.items(): key = (gds["name"], _type) parent = QTreeWidgetItem(self.annotations_widget, [_type]) parent.key = key for subset in subsets: key = (gds['name'], _type, subset) item = QTreeWidgetItem( parent, [subset, subsets_count.get(subset, '')]) item.setFlags(item.flags() | Qt.ItemIsUserCheckable) item.setCheckState( 0, self.gdsSelectionStates.get(key, Qt.Checked)) item.key = key self._annotations_updating = False self.annotations_widget.expandAll() for i in range(self.annotations_widget.columnCount()): self.annotations_widget.resizeColumnToContents(i) def _set_selection(self): if self.selected_gds is not None: index = self.table_model.get_row_index( self.selected_gds.get('name')) if index is not None: self.table_view.selectionModel().blockSignals(True) self.table_view.selectRow(index) self._handle_selection_changed() self.table_view.selectionModel().blockSignals(False) def _handle_selection_changed(self): if self.table_model.table is not None: selection = self.table_view.selectionModel().selectedRows( self.table_model.gds_id_col) selected_gds_name = selection[0].data( ) if len(selection) > 0 else None if selected_gds_name: self.selected_gds = self.table_model.info.get( selected_gds_name) self._set_annotations_widget(self.selected_gds) self._set_description_widget() else: self.annotations_widget.clear() self.description_widget.clear() self.update_info() def _apply_filter(self): if self.table_model.table is not None: self.table_model.show_table( filter_pattern=str(self.search_pattern)) self._set_selection() self.update_info() def _run(self): if self.selected_gds is not None: self.gds_data = None self.start(run_download_task, self.selected_gds.get('name'), self.get_selected_samples(), self.genes_as_rows) def on_gds_selection_changed(self): self._handle_selection_changed() self._run() def on_annotation_selection_changed(self): if self._annotations_updating: return for i in range(self.annotations_widget.topLevelItemCount()): item = self.annotations_widget.topLevelItem(i) if 'key' in item.__dict__: self.gdsSelectionStates[item.key] = item.checkState(0) for j in range(item.childCount()): child = item.child(j) if 'key' in child.__dict__: self.gdsSelectionStates[child.key] = child.checkState(0) self._run() def update_info(self): all_gds = len(self.table_model.info) text = "{} datasets\n{} datasets cached\n".format( all_gds, len(local_files.listfiles())) filtered = self.table_view.model().rowCount() if all_gds != filtered: text += "{} after filtering".format(filtered) self.infoBox.setText(text) def get_selected_samples(self): """ Return the currently selected sample annotations. The return value is a list of selected (sample type, sample value) tuples. .. note:: if some Sample annotation type has no selected values. this method will return all values for it. TODO: this could probably be simplified. """ def childiter(item): """ Iterate over the children of an QTreeWidgetItem instance. """ for i in range(item.childCount()): yield item.child(i) samples = [] unused_types = [] used_types = [] for stype in childiter(self.annotations_widget.invisibleRootItem()): selected_values = [] all_values = [] for sval in childiter(stype): value = (str(stype.text(0)), str(sval.text(0))) if self.gdsSelectionStates.get(sval.key, True): selected_values.append(value) all_values.append(value) if selected_values: samples.extend(selected_values) used_types.append(str(stype.text(0))) else: # If no sample of sample type is selected we don't filter on it. samples.extend(all_values) unused_types.append(str(stype.text(0))) _samples = defaultdict(list) for sample, sample_type in samples: _samples[sample].append(sample_type) return _samples def commit(self): self.Outputs.gds_data.send(self.gds_data) def on_done(self, result: Result): assert isinstance(result.gds_dataset, Table) self.gds_data = result.gds_dataset self.commit() if self.gds_info: self.table_model.initialize(self.gds_info) self._apply_filter() def on_partial_result(self, result: Any) -> None: pass def onDeleteWidget(self): self.shutdown() super().onDeleteWidget() def send_report(self): self.report_items( "GEO Dataset", [ ("ID", self.selected_gds['name']), ("Title", self.selected_gds['title']), ("Organism", self.selected_gds['sample_organism']), ], ) self.report_items( "Data", [ ("Samples", self.selected_gds['sample_count']), ("Features", self.selected_gds['variables']), ("Genes", self.selected_gds['genes']), ], ) self.report_name("Sample annotations") subsets = defaultdict(list) for subset in self.selected_gds['subsets']: subsets[subset['type']].append( (subset['description'], len(subset['sample_id']))) self.report_html += "<ul>" for _type in subsets: self.report_html += "<b>" + _type + ":</b></br>" for desc, count in subsets[_type]: self.report_html += 9 * " " + "<b>{}:</b> {}</br>".format( desc, count) self.report_html += "</ul>"
class OWPIPAx(widget.OWWidget): name = "PIPAx" description = "Access data from PIPA RNA-Seq database." icon = "../widgets/icons/PIPA.svg" priority = 35 inputs = [] outputs = [("Data", Orange.data.Table)] username = settings.Setting("") password = settings.Setting("") log2 = settings.Setting(False) rtypei = settings.Setting(5) # hardcoded rpkm mapability polya excludeconstant = settings.Setting(False) joinreplicates = settings.Setting(False) #: The stored current selection (in experiments view) #: SelectionByKey | None currentSelection = settings.Setting(None) #: Stored selections (presets) #: list of SelectionByKey storedSelections = settings.Setting([]) #: Stored column sort keys (from Sort view) #: list of strings storedSortingOrder = settings.Setting( ["Strain", "Experiment", "Genotype", "Timepoint"]) experimentsHeaderState = settings.Setting( {name: False for _, name in HEADER[:ID_INDEX + 1]}) def __init__(self, parent=None, signalManager=None, name="PIPAx"): super().__init__(parent) self.selectedExperiments = [] self.buffer = dicty.CacheSQLite(bufferfile) self.searchString = "" self.result_types = [] self.mappings = {} self.controlArea.setMaximumWidth(250) self.controlArea.setMinimumWidth(250) gui.button(self.controlArea, self, "Reload", callback=self.Reload) gui.button(self.controlArea, self, "Clear cache", callback=self.clear_cache) b = gui.widgetBox(self.controlArea, "Experiment Sets") self.selectionSetsWidget = SelectionSetsWidget(self) self.selectionSetsWidget.setSizePolicy(QSizePolicy.Preferred, QSizePolicy.Maximum) def store_selections(modified): if not modified: self.storedSelections = self.selectionSetsWidget.selections self.selectionSetsWidget.selectionModified.connect(store_selections) b.layout().addWidget(self.selectionSetsWidget) gui.separator(self.controlArea) b = gui.widgetBox(self.controlArea, "Sort output columns") self.columnsSortingWidget = SortedListWidget(self) self.columnsSortingWidget.setSizePolicy(QSizePolicy.Preferred, QSizePolicy.Maximum) def store_sort_order(): self.storedSortingOrder = self.columnsSortingWidget.sortingOrder self.columnsSortingWidget.sortingOrderChanged.connect(store_sort_order) b.layout().addWidget(self.columnsSortingWidget) sorting_model = QStringListModel(SORTING_MODEL_LIST) self.columnsSortingWidget.setModel(sorting_model) gui.separator(self.controlArea) box = gui.widgetBox(self.controlArea, 'Expression Type') self.expressionTypesCB = gui.comboBox(box, self, "rtypei", items=[], callback=self.UpdateResultsList) gui.checkBox(self.controlArea, self, "excludeconstant", "Exclude labels with constant values") gui.checkBox(self.controlArea, self, "joinreplicates", "Average replicates (use median)") gui.checkBox(self.controlArea, self, "log2", "Logarithmic (base 2) transformation") self.commit_button = gui.button(self.controlArea, self, "&Commit", callback=self.Commit) self.commit_button.setDisabled(True) gui.rubber(self.controlArea) box = gui.widgetBox(self.controlArea, "Authentication") gui.lineEdit(box, self, "username", "Username:"******"password", "Password:"******"searchString", "Search", callbackOnType=True, callback=self.SearchUpdate) self.headerLabels = [t[1] for t in HEADER] self.experimentsWidget = QTreeWidget() self.experimentsWidget.setHeaderLabels(self.headerLabels) self.experimentsWidget.setSelectionMode(QTreeWidget.ExtendedSelection) self.experimentsWidget.setRootIsDecorated(False) self.experimentsWidget.setSortingEnabled(True) contextEventFilter = gui.VisibleHeaderSectionContextEventFilter( self.experimentsWidget, self.experimentsWidget) self.experimentsWidget.header().installEventFilter(contextEventFilter) self.experimentsWidget.setItemDelegateForColumn( 0, gui.IndicatorItemDelegate(self, role=Qt.DisplayRole)) self.experimentsWidget.setAlternatingRowColors(True) self.experimentsWidget.selectionModel().selectionChanged.connect( self.onSelectionChanged) self.selectionSetsWidget.setSelectionModel( self.experimentsWidget.selectionModel()) self.mainArea.layout().addWidget(self.experimentsWidget) # Restore the selection states from the stored settings self.selectionSetsWidget.selections = self.storedSelections self.columnsSortingWidget.sortingOrder = self.storedSortingOrder self.restoreHeaderState() self.experimentsWidget.header().geometriesChanged.connect( self.saveHeaderState) self.dbc = None self.AuthSet() QTimer.singleShot(100, self.UpdateExperiments) def sizeHint(self): return QSize(800, 600) def AuthSet(self): if len(self.username): self.passf.setDisabled(False) else: self.passf.setDisabled(True) def AuthChanged(self): self.AuthSet() self.ConnectAndUpdate() def ConnectAndUpdate(self): self.Connect() self.UpdateExperiments(reload=True) def Connect(self): self.error(1) self.warning(1) def en(x): return x if len(x) else None self.dbc = dicty.PIPAx(cache=self.buffer, username=en(self.username), password=self.password) # check password if en(self.username) != None: try: self.dbc.mappings(reload=True) except dicty.AuthenticationError: self.error(1, "Wrong username or password") self.dbc = None except Exception as ex: print("Error when contacting the PIPA database", ex) sys.excepthook(*sys.exc_info()) try: # maybe cached? self.dbc.mappings() self.warning( 1, "Can not access database - using cached data.") except Exception as ex: self.dbc = None self.error(1, "Can not access database.") def Reload(self): self.UpdateExperiments(reload=True) def clear_cache(self): self.buffer.clear() self.Reload() def rtype(self): """Return selected result template type """ if self.result_types: return self.result_types[self.rtypei][0] else: return "-1" def UpdateExperimentTypes(self): self.expressionTypesCB.clear() items = [desc for _, desc in self.result_types] self.expressionTypesCB.addItems(items) self.rtypei = max(0, min(self.rtypei, len(self.result_types) - 1)) def UpdateExperiments(self, reload=False): self.experimentsWidget.clear() self.items = [] self.progressBarInit() if not self.dbc: self.Connect() mappings = {} result_types = [] sucind = False # success indicator for database index try: mappings = self.dbc.mappings(reload=reload) result_types = self.dbc.result_types(reload=reload) sucind = True except Exception as ex: try: mappings = self.dbc.mappings() result_types = self.dbc.result_types() self.warning(0, "Can not access database - using cached data.") sucind = True except Exception as ex: self.error(0, "Can not access database.") if sucind: self.warning(0) self.error(0) self.mappings = mappings self.result_types = result_types self.UpdateExperimentTypes() self.UpdateResultsList(reload=reload) self.progressBarFinished() if self.currentSelection: self.currentSelection.select( self.experimentsWidget.selectionModel()) self.handle_commit_button() def UpdateResultsList(self, reload=False): results_list = {} try: results_list = self.dbc.results_list(self.rtype(), reload=reload) except Exception as ex: try: results_list = self.dbc.results_list(self.rtype()) except Exception as ex: self.error(0, "Can not access database.") self.results_list = results_list mappings_key_dict = dict(((m["data_id"], m["id"]), key) \ for key, m in self.mappings.items()) def mapping_unique_id(annot): """Map annotations dict from results_list to unique `mappings` ids. """ data_id, mappings_id = annot["data_id"], annot["mappings_id"] return mappings_key_dict[data_id, mappings_id] elements = [] # softly change the view so that the selection stays the same items_shown = {} for i, item in enumerate(self.items): c = str(item.text(10)) items_shown[c] = i items_to_show = dict((mapping_unique_id(annot), annot) for annot in self.results_list.values()) add_items = set(items_to_show) - set(items_shown) delete_items = set(items_shown) - set(items_to_show) i = 0 while i < self.experimentsWidget.topLevelItemCount(): it = self.experimentsWidget.topLevelItem(i) if str(it.text(10)) in delete_items: self.experimentsWidget.takeTopLevelItem(i) else: i += 1 delete_ind = set([items_shown[i] for i in delete_items]) self.items = [ it for i, it in enumerate(self.items) if i not in delete_ind ] for r_annot in [items_to_show[i] for i in add_items]: d = defaultdict(lambda: "?", r_annot) row_items = [""] + [d.get(key, "?") for key, _ in HEADER[1:]] try: time_dict = literal_eval(row_items[DATE_INDEX]) date_rna = date( time_dict["fullYearUTC"], time_dict["monthUTC"] + 1, # Why is month 0 based? time_dict["dateUTC"]) row_items[DATE_INDEX] = date_rna.strftime("%x") except Exception: row_items[DATE_INDEX] = '' row_items[ID_INDEX] = mapping_unique_id(r_annot) elements.append(row_items) ci = MyTreeWidgetItem(self.experimentsWidget, row_items) self.items.append(ci) for i in range(len(self.headerLabels)): self.experimentsWidget.resizeColumnToContents(i) # which is the ok buffer version # FIXME: what attribute to use for version? self.wantbufver = \ lambda x, ad=self.results_list: \ defaultdict(lambda: "?", ad[x])["date"] self.wantbufver = lambda x: "0" self.UpdateCached() def UpdateCached(self): if self.wantbufver and self.dbc: fn = self.dbc.download_key_function() result_id_key = dict(((m["data_id"], m["mappings_id"]), key) \ for key, m in self.results_list.items()) for item in self.items: c = str(item.text(10)) mapping = self.mappings[c] data_id, mappings_id = mapping["data_id"], mapping["id"] r_id = result_id_key[data_id, mappings_id] # Get the buffered version buffered = self.dbc.inBuffer(fn(r_id)) value = " " if buffered == self.wantbufver(r_id) else "" item.setData(0, Qt.DisplayRole, value) def SearchUpdate(self, string=""): for item in self.items: item.setHidden(not all(s in item \ for s in self.searchString.split()) ) def Commit(self): if not self.dbc: self.Connect() pb = gui.ProgressBar(self, iterations=100) table = None ids = [] for item in self.experimentsWidget.selectedItems(): unique_id = str(item.text(10)) annots = self.mappings[unique_id] ids.append((annots["data_id"], annots["id"])) transfn = None if self.log2: transfn = lambda x: math.log(x + 1.0, 2) reverse_header_dict = dict((name, key) for key, name in HEADER) hview = self.experimentsWidget.header() shownHeaders = [label for i, label in \ list(enumerate(self.headerLabels))[1:] \ if not hview.isSectionHidden(i) ] allowed_labels = [reverse_header_dict.get(label, label) \ for label in shownHeaders] if self.joinreplicates and "id" not in allowed_labels: # need 'id' labels in join_replicates for attribute names allowed_labels.append("id") if len(ids): table = self.dbc.get_data( ids=ids, result_type=self.rtype(), callback=pb.advance, exclude_constant_labels=self.excludeconstant, # bufver=self.wantbufver, transform=transfn, allowed_labels=allowed_labels) if self.joinreplicates: table = dicty.join_replicates(table, ignorenames=[ "replicate", "data_id", "mappings_id", "data_name", "id", "unique_id" ], namefn=None, avg=dicty.median) # Sort attributes sortOrder = self.columnsSortingWidget.sortingOrder all_values = defaultdict(set) for at in table.domain.attributes: atts = at.attributes for name in sortOrder: all_values[name].add( atts.get(reverse_header_dict[name], "")) isnum = {} for at, vals in all_values.items(): vals = filter(None, vals) try: for a in vals: float(a) isnum[at] = True except: isnum[at] = False def optfloat(x, at): if x == "": return "" else: return float(x) if isnum[at] else x def sorting_key(attr): atts = attr.attributes return tuple([optfloat(atts.get(reverse_header_dict[name], ""), name) \ for name in sortOrder]) attributes = sorted(table.domain.attributes, key=sorting_key) domain = Orange.data.Domain(attributes, table.domain.class_var, table.domain.metas) table = table.from_table(domain, table) data_hints.set_hint(table, "taxid", "352472") data_hints.set_hint(table, "genesinrows", False) self.send("Data", table) self.UpdateCached() pb.finish() def onSelectionChanged(self, selected, deselected): self.handle_commit_button() def handle_commit_button(self): self.currentSelection = \ SelectionByKey(self.experimentsWidget.selectionModel().selection(), key=(1, 2, 3, 10)) self.commit_button.setDisabled(not len(self.currentSelection)) def saveHeaderState(self): hview = self.experimentsWidget.header() for i, label in enumerate(self.headerLabels): self.experimentsHeaderState[label] = hview.isSectionHidden(i) def restoreHeaderState(self): hview = self.experimentsWidget.header() state = self.experimentsHeaderState for i, label in enumerate(self.headerLabels): hview.setSectionHidden(i, state.get(label, True)) self.experimentsWidget.resizeColumnToContents(i)