def main(servername, credentialsfilename, catalog): def apply(catalog, goal): """ Apply the goal configuration to live catalog """ print 'applying...' counter = 0 ready = False while ready == False: try: catalog.applyCatalogConfig(goal) ready = True except HTTPError as err: print err print err.errno if err.errno == CONFLICT: et, ev, tb = sys.exc_info() print 'Conflict Exception "%s"' % str(ev) counter = counter + 1 if counter >= 5: print '%s' % str(traceback.format_exception(et, ev, tb)) ready = True else: print 'Retrying...' except: et, ev, tb = sys.exc_info() print str(et) print 'Exception "%s"' % str(ev) print '%s' % str(traceback.format_exception(et, ev, tb)) ready = True credentials = json.load(open(credentialsfilename)) catalog = ErmrestCatalog('https', servername, catalog, credentials) try: goal = catalog.get_catalog_model() except AttributeError: try: goal = catalog.getCatalogModel() except: et, ev, tb = sys.exc_info() print 'got exception "%s"' % str(ev) print '%s' % str(traceback.format_exception(et, ev, tb)) sys.exit(1) schema_name = 'Microscopy' table_name = 'Slide' column_name = 'Label' goal.column(schema_name, table_name, column_name).column_display.update({ "compact": {"markdown_pattern": "{{#Label}}:::iframe [](/chaise/PrintLabel.html?label=/microscopy/printer/slide/job?{{{Label}}}){height=75 width=150 style=\"border-style: none; border-color: rgb(153, 153, 153);\" .iframe} \n:::{{/Label}}"}, "detailed": {"markdown_pattern": "{{#Label}}:::iframe [](/chaise/PrintLabel.html?label=/microscopy/printer/slide/job?{{{Label}}}){height=75 width=150 style=\"border-style: none; border-color: rgb(153, 153, 153);\" .iframe} \n:::{{/Label}}"} }) apply(catalog, goal) print 'Successfully updated the URL annotation for the column %s' % column_name
RMT = '&RMT::geq::%s' % (urlquote(options.RMT)) """ Get the non NULL "Thumbnail" values from the "Scan" table. """ servername = options.server credentialsfilename = options.credentials catalog = 1 schema = 'Microscopy' table = 'Scan' column = 'Thumbnail' prefix = '/var/www/html' output = '%s_add_border.sh' % servername.split('.')[0] credentials = json.load(open(credentialsfilename)) catalog = ErmrestCatalog('https', servername, catalog, credentials) url = '/attribute/%s:%s/!%s::null::%s/%s' % (urlquote(schema), urlquote(table), urlquote(column), RMT, urlquote(column)) print ('Query URL: "https://%s/ermrest/catalog/1%s"' % (servername, url)) resp = catalog.get(url) resp.raise_for_status() rows = resp.json() thumbnails = [] for row in rows: thumbnails.append(row[column]) """ Generate the shell script. """
fkey_defs=fkey_defs, annotations=table_annotations, acls=table_acls, acl_bindings=table_acl_bindings, comment=table_comment, provide_system=True) def main(catalog, mode, replace=False, really=False): updater = CatalogUpdater(catalog) table_def['column_annotations'] = column_annotations table_def['column_comment'] = column_comment updater.update_table(mode, schema_name, table_def, replace=replace, really=really) if __name__ == "__main__": host = 'pdb.isrd.isi.edu' catalog_id = 99 mode, replace, host, catalog_id = parse_args(host, catalog_id, is_table=True) catalog = ErmrestCatalog('https', host, catalog_id=catalog_id, credentials=get_credential(host)) main(catalog, mode, replace)
print 'ERROR: Missing credentials file' sys.exit() servername = options.server credentialsfilename = options.credentials catalog = 1 schema = 'Microscopy' table = 'Scan' acquisition = 'Acquisition Date' czi = 'HTTP URL' rid = 'RID' rct = 'RCT' filename = 'filename' credentials = json.load(open(credentialsfilename)) catalog = ErmrestCatalog('https', servername, catalog, credentials) hatrac_store = HatracStore( 'https', servername, {'cookie': credentials['cookie']} ) url = '/attribute/%s:%s/%s::null::/%s,%s,%s,%s' % (urlquote(schema), urlquote(table), urlquote(acquisition), urlquote(rid), urlquote(rct), urlquote(filename), urlquote(czi)) print 'Query URL: "%s"' % url resp = catalog.get(url) resp.raise_for_status() rows = resp.json() entities = [] for row in rows:
class MainWindow(QMainWindow): config = None credential = None config_path = None store = None catalog = None identity = None attributes = None server = None tempdir = None progress_update_signal = pyqtSignal(str) use_3D_viewer = False curator_mode = False def __init__(self, config_path=None): super(MainWindow, self).__init__() self.ui = MainWindowUI(self) self.configure(config_path) self.authWindow = EmbeddedAuthWindow( self, config=self.config.get("server"), cookie_persistence=False, authentication_success_callback=self.onLoginSuccess ) self.getSession() if not self.identity: self.ui.actionLaunch.setEnabled(False) self.ui.actionRefresh.setEnabled(False) self.ui.actionOptions.setEnabled(False) self.ui.actionLogout.setEnabled(False) def configure(self, config_path): # configure logging self.ui.logTextBrowser.widget.log_update_signal.connect(self.updateLog) self.ui.logTextBrowser.setFormatter(logging.Formatter("%(asctime)s - %(levelname)s - %(message)s")) logging.getLogger().addHandler(self.ui.logTextBrowser) logging.getLogger().setLevel(logging.INFO) # configure Ermrest/Hatrac if not config_path: config_path = os.path.join(os.path.expanduser( os.path.normpath("~/.deriva/synapse/synspy-launcher")), "config.json") self.config_path = config_path config = read_config(self.config_path, create_default=True, default=DEFAULT_CONFIG) protocol = config["server"]["protocol"] self.server = config["server"]["host"] catalog_id = config["server"]["catalog_id"] session_config = config.get("session") self.catalog = ErmrestCatalog(protocol, self.server, catalog_id, self.credential, session_config=session_config) self.store = HatracStore(protocol, self.server, self.credential, session_config=session_config) # create working dir (tempdir) self.tempdir = tempfile.mkdtemp(prefix="synspy_") # determine viewer mode self.use_3D_viewer = True if config.get("viewer_mode", "2d").lower() == "3d" else False # curator mode? curator_mode = config.get("curator_mode") if not curator_mode: config["curator_mode"] = False self.curator_mode = config.get("curator_mode") # save config self.config = config write_config(self.config_path, self.config) def getSession(self): qApp.setOverrideCursor(Qt.WaitCursor) self.updateStatus("Validating session.") queryTask = SessionQueryTask(self.catalog) queryTask.status_update_signal.connect(self.onSessionResult) queryTask.query() def onLoginSuccess(self, **kwargs): self.authWindow.hide() self.credential = kwargs["credential"] self.catalog.set_credentials(self.credential, self.server) self.store.set_credentials(self.credential, self.server) self.getSession() def enableControls(self): self.ui.actionLaunch.setEnabled(True) self.ui.actionRefresh.setEnabled(True) self.ui.actionOptions.setEnabled(self.authWindow.authenticated()) self.ui.actionLogin.setEnabled(not self.authWindow.authenticated()) self.ui.actionLogout.setEnabled(self.authWindow.authenticated()) self.ui.actionExit.setEnabled(True) self.ui.workList.setEnabled(True) def disableControls(self): self.ui.actionLaunch.setEnabled(False) self.ui.actionRefresh.setEnabled(False) self.ui.actionOptions.setEnabled(False) self.ui.actionLogin.setEnabled(False) self.ui.actionLogout.setEnabled(False) self.ui.actionExit.setEnabled(False) self.ui.workList.setEnabled(False) def closeEvent(self, event=None): self.disableControls() self.cancelTasks() shutil.rmtree(self.tempdir) if event: event.accept() def cancelTasks(self): Task.shutdown_all() self.statusBar().showMessage("Waiting for background tasks to terminate...") while True: qApp.processEvents() if QThreadPool.globalInstance().waitForDone(10): break self.statusBar().showMessage("All background tasks terminated successfully") def is_curator(self): for attr in self.attributes: if attr.get('id') == CURATORS: return True return False def displayWorklist(self, worklist): keys = [ "RID", "RCT", "Source Image", "Classifier", "Due Date", "Accepted?", "Status", "URL", "Npz URL", "ZYX Slice", "Segmentation Mode", "Segments URL", "Segments Filtered URL", "Subject", ] self.ui.workList.clear() self.ui.workList.setRowCount(0) self.ui.workList.setColumnCount(0) displayed = ["RID", "RCT", "Segmentation Mode", "Classifier", "Due Date", "Accepted?", "Status"] self.ui.workList.setRowCount(len(worklist)) self.ui.workList.setColumnCount(len(keys)) self.ui.workList.removeAction(self.ui.markIncompleteAction) if self.is_curator() and self.curator_mode: self.ui.workList.addAction(self.ui.markIncompleteAction) rows = 0 for row in worklist: value = row.get("Status") if not (value == "analysis pending" or value == "analysis in progress") \ and not (self.is_curator() and self.curator_mode): self.ui.workList.hideRow(rows) cols = 0 for key in keys: item = QTableWidgetItem() if key == "Classifier": value = "%s (%s)" % (row['user'][0]['Full_Name'], row['user'][0]['Display_Name']) item.setData(Qt.UserRole, row['Classifier']) elif key == "URL" or key == "Subject": value = row["source_image"][0].get(key) else: value = row.get(key) if isinstance(value, bool): value = str(value) if isinstance(value, str) and key == 'RCT': value = value.replace('T', ' ')[0:19] # drop fractional seconds and TZ if isinstance(value, str): item.setText(value) item.setToolTip(value) self.ui.workList.setItem(rows, cols, item) cols += 1 rows += 1 cols = 0 for key in keys: if key not in displayed: self.ui.workList.hideColumn(cols) cols += 1 self.ui.workList.setHorizontalHeaderLabels(keys) # add header names self.ui.workList.horizontalHeader().setDefaultAlignment(Qt.AlignLeft) # set alignment for col in range(len(displayed)): self.ui.workList.resizeColumnToContents(col) self.ui.workList.sortByColumn(2, Qt.DescendingOrder) def getCacheDir(self): cwd = os.getcwd() cache_dir = os.path.expanduser(self.config.get("cache_dir", cwd)) if not os.path.isdir(cache_dir): try: os.makedirs(cache_dir) except OSError as error: if error.errno != errno.EEXIST: logging.error(format_exception(error)) cache_dir = cwd return cache_dir def downloadCallback(self, **kwargs): status = kwargs.get("progress") if status: self.progress_update_signal.emit(status) return True def uploadCallback(self, **kwargs): completed = kwargs.get("completed") total = kwargs.get("total") file_path = kwargs.get("file_path") if completed and total: file_path = " [%s]" % os.path.basename(file_path) if file_path else "" status = "Uploading file%s: %d%% complete" % (file_path, round(((completed / total) % 100) * 100)) else: summary = kwargs.get("summary", "") file_path = "Uploaded file: [%s] " % os.path.basename(file_path) if file_path else "" status = file_path # + summary if status: self.progress_update_signal.emit(status) return True def serverProblemMessageBox(self, text, detail): msg = QMessageBox() msg.setIcon(QMessageBox.Warning) msg.setWindowTitle("Confirm Action") msg.setText(text) msg.setInformativeText(detail + "\n\nWould you like to remove this item from the current worklist?") msg.setStandardButtons(QMessageBox.Yes | QMessageBox.No) ret = msg.exec_() if ret == QMessageBox.No: return else: row = self.ui.workList.getCurrentTableRow() self.ui.workList.removeRow(row) return def retrieveFiles(self): # if there is an existing segments file, download it first, otherwise just initiate the input file download seg_mode = self.ui.workList.getCurrentTableItemTextByName("Segmentation Mode") segments_url = self.ui.workList.getCurrentTableItemTextByName("Segments Filtered URL") if segments_url: segments_filename = 'ROI_%s_%s_only.csv' % ( self.ui.workList.getCurrentTableItemTextByName("RID"), seg_mode) segments_destfile = os.path.abspath(os.path.join(self.tempdir, segments_filename)) self.updateStatus("Downloading file: [%s]" % segments_destfile) downloadTask = FileRetrieveTask(self.store) downloadTask.status_update_signal.connect(self.onRetrieveAnalysisFileResult) self.progress_update_signal.connect(self.updateProgress) downloadTask.retrieve( segments_url, destfile=segments_destfile, progress_callback=self.downloadCallback) else: self.retrieveInputFile() def retrieveInputFile(self): # get the main TIFF file for analysis if not already cached if self.use_3D_viewer: url = self.ui.workList.getCurrentTableItemTextByName("URL") filename = 'Image_%s.ome.tiff' % self.ui.workList.getCurrentTableItemTextByName("Source Image") else: url = self.ui.workList.getCurrentTableItemTextByName("Npz URL") filename = 'ROI_%s.npz' % self.ui.workList.getCurrentTableItemTextByName("RID") destfile = os.path.abspath(os.path.join(self.getCacheDir(), filename)) if not url and not self.use_3D_viewer: self.resetUI("Unable to launch 2D viewer due to missing NPZ file for %s." % self.ui.workList.getCurrentTableItemTextByName("RID")) self.serverProblemMessageBox( "2D viewer requires NPZ data to be present!", "The launcher is currently configured to execute the 2D viewer, which requires NPZ files for input. " + "No NPZ file could be found on the server for this task.") return if not os.path.isfile(destfile): self.updateStatus("Downloading file: [%s]" % destfile) downloadTask = FileRetrieveTask(self.store) downloadTask.status_update_signal.connect(self.onRetrieveInputFileResult) self.progress_update_signal.connect(self.updateProgress) downloadTask.retrieve( url, destfile=destfile, progress_callback=self.downloadCallback) else: self.onRetrieveInputFileResult(True, "The file [%s] already exists" % destfile, None, destfile) def getSubprocessPath(self): executable = "synspy-viewer" if self.use_3D_viewer else "synspy-viewer2d" base_path = None return os.path.normpath(resource_path(executable, base_path)) def executeViewer(self, file_path): self.updateStatus("Executing viewer...") env = os.environ env["SYNSPY_AUTO_DUMP_LOAD"] = "true" env["DUMP_PREFIX"] = "./ROI_%s" % self.ui.workList.getCurrentTableItemTextByName("RID") env["ZYX_SLICE"] = self.ui.workList.getCurrentTableItemTextByName("ZYX Slice") env["ZYX_IMAGE_GRID"] = "0.4, 0.26, 0.26" env["SYNSPY_DETECT_NUCLEI"] = str( "nucleic" == self.ui.workList.getCurrentTableItemTextByName("Segmentation Mode")).lower() output_path = os.path.join(os.path.dirname(self.config_path), "viewer.log") classifier = self.ui.workList.getTableItemByName( self.ui.workList.getCurrentTableRow(), "Classifier").data(Qt.UserRole) viewerTask = ViewerTask(self.getSubprocessPath(), self.identity == classifier, proc_output_path=output_path) viewerTask.status_update_signal.connect(self.onSubprocessExecuteResult) viewerTask.run(file_path, self.tempdir, env) def uploadAnalysisResult(self, update_state): qApp.setOverrideCursor(Qt.WaitCursor) # generate hatrac upload params basename = "ROI_%s" % self.ui.workList.getCurrentTableItemTextByName("RID") match = r"%s_.*\.csv$" % basename output_files = [f for f in os.listdir(self.tempdir) if os.path.isfile(os.path.join(self.tempdir, f)) and re.match(match, f)] if not output_files: self.resetUI("Could not locate output file from viewer subprocess -- aborting.") return seg_mode = self.ui.workList.getCurrentTableItemTextByName("Segmentation Mode") if seg_mode == "synaptic": extension = "_synaptic_only.csv" elif seg_mode == "nucleic": extension = "_nucleic_only.csv" else: self.updateStatus("Unknown segmentation mode \"%s\" -- aborting." % seg_mode) return file_name = basename + extension hatrac_path = HATRAC_UPDATE_URL_TEMPLATE % \ (self.ui.workList.getCurrentTableItemTextByName("Subject"), file_name) file_path = os.path.abspath(os.path.join(self.tempdir, file_name)) # upload to object store self.updateStatus("Uploading file %s to server..." % file_name) self.progress_update_signal.connect(self.updateProgress) uploadTask = FileUploadTask(self.store) uploadTask.status_update_signal.connect(self.onUploadFileResult) uploadTask.upload(hatrac_path, file_path, update_state, callback=self.uploadCallback) def markIncomplete(self): RID = self.ui.workList.getCurrentTableItemTextByName("RID") body = [{"RID": RID, "Status": "analysis in progress"}] self.updateStatus("Updating task status for %s..." % RID) updateTask = CatalogUpdateTask(self.catalog) updateTask.status_update_signal.connect(self.onCatalogUpdateResult) updateTask.update(WORKLIST_STATUS_UPDATE, json=body) @pyqtSlot() def taskTriggered(self): self.ui.logTextBrowser.widget.clear() self.disableControls() @pyqtSlot(str) def updateProgress(self, status): self.statusBar().showMessage(status) @pyqtSlot(str, str) def updateStatus(self, status, detail=None): logging.info(status + ((": %s" % detail) if detail else "")) self.statusBar().showMessage(status) @pyqtSlot(str, str) def resetUI(self, status, detail=None): qApp.restoreOverrideCursor() self.updateStatus(status, detail) self.enableControls() @pyqtSlot(str) def updateLog(self, text): self.ui.logTextBrowser.widget.appendPlainText(text) @pyqtSlot(bool, str, str, object) def onSessionResult(self, success, status, detail, result): qApp.restoreOverrideCursor() if success: self.identity = result["client"]["id"] self.attributes = result["attributes"] display_name = result["client"]["full_name"] self.setWindowTitle("%s (%s - %s)" % (self.windowTitle(), self.server, display_name)) self.ui.actionLaunch.setEnabled(True) self.ui.actionLogout.setEnabled(True) self.ui.actionLogin.setEnabled(False) if not self.is_curator(): self.curator_mode = self.config["curator_mode"] = False self.on_actionRefresh_triggered() else: self.updateStatus("Login required.") @pyqtSlot() def on_actionLaunch_triggered(self): self.disableControls() qApp.setOverrideCursor(Qt.WaitCursor) # create working dir (tempdir) if self.tempdir: shutil.rmtree(self.tempdir) self.tempdir = tempfile.mkdtemp(prefix="synspy_") self.retrieveFiles() @pyqtSlot(bool, str, str, object) def onRetrieveAnalysisFileResult(self, success, status, detail, file_path): if not success: try: os.remove(file_path) except Exception as e: logging.warning("Unable to remove file [%s]: %s" % (file_path, format_exception(e))) self.resetUI(status, detail) self.serverProblemMessageBox( "Unable to download required input file", "The in-progress analysis file was not downloaded successfully.") return self.retrieveInputFile() @pyqtSlot(bool, str, str, object) def onRetrieveInputFileResult(self, success, status, detail, file_path): if not success: try: os.remove(file_path) except Exception as e: logging.warning("Unable to remove file [%s]: %s" % (file_path, format_exception(e))) self.resetUI(status, detail) self.serverProblemMessageBox( "Unable to download required input file", "The image input file was not downloaded successfully.") return self.executeViewer(file_path) @pyqtSlot(bool, str, str, object) def onSubprocessExecuteResult(self, success, status, detail, is_owner): qApp.restoreOverrideCursor() if not success: self.resetUI(status, detail) return if not bool(is_owner) or self.curator_mode: self.resetUI(status, detail) return # prompt for save/complete/discard msg = QMessageBox() msg.setIcon(QMessageBox.Information) msg.setWindowTitle("Confirm Action") msg.setText("How would you like to proceed?") msg.setInformativeText( "Select \"Save Progress\" to save your progress and upload the output to the server.\n\n" "Select \"Complete\" to upload the output to the server and mark this task as completed.\n\n" "Select \"Discard\" to abort the process and leave the task state unchanged.") saveButton = msg.addButton("Save Progress", QMessageBox.ActionRole) completeButton = msg.addButton("Complete", QMessageBox.ActionRole) discardButton = msg.addButton("Discard", QMessageBox.RejectRole) msg.exec_() if msg.clickedButton() == discardButton: self.resetUI("Aborted.") return update_state = None if msg.clickedButton() == saveButton: update_state = ("incomplete", "analysis in progress") elif msg.clickedButton() == completeButton: update_state = ("complete", "analysis complete") self.uploadAnalysisResult(update_state) @pyqtSlot(bool, str, str, object) def onUploadFileResult(self, success, status, detail, result): if not success: self.resetUI(status, detail) self.serverProblemMessageBox( "Unable to upload required file(s)", "One or more required files were not uploaded successfully.") return state = result[0] RID = self.ui.workList.getCurrentTableItemTextByName("RID") body = [{"RID": RID, "Segments Filtered URL": result[1], "Status": state[1]}] self.updateStatus("Updating task status for %s..." % RID) updateTask = CatalogUpdateTask(self.catalog) updateTask.status_update_signal.connect(self.onCatalogUpdateResult) updateTask.update(WORKLIST_UPDATE, json=body) @pyqtSlot(bool, str, str, object) def onCatalogUpdateResult(self, success, status, detail, result): if not success: self.resetUI(status, detail) self.serverProblemMessageBox( "Unable to update catalog data", "The catalog state was not updated successfully.") return qApp.restoreOverrideCursor() self.on_actionRefresh_triggered() @pyqtSlot() def on_actionRefresh_triggered(self): if not self.identity: self.updateStatus("Unable to get worklist -- not logged in.") return qApp.setOverrideCursor(Qt.WaitCursor) self.disableControls() self.updateStatus("Refreshing worklist...") queryTask = CatalogQueryTask(self.catalog) queryTask.status_update_signal.connect(self.onRefreshResult) if self.is_curator() and self.curator_mode: queryTask.query(WORKLIST_CURATOR_QUERY) else: queryTask.query(WORKLIST_QUERY % urlquote(self.identity, "")) @pyqtSlot(bool, str, str, object) def onRefreshResult(self, success, status, detail, result): if success: self.displayWorklist(result) self.resetUI("Ready.") else: self.resetUI(status, detail) if (self.ui.workList.rowCount() > 0) and self.identity: self.ui.actionLaunch.setEnabled(True) else: self.ui.actionLaunch.setEnabled(False) @pyqtSlot() def on_actionLogin_triggered(self): self.authWindow.show() self.authWindow.login() @pyqtSlot() def on_actionLogout_triggered(self): self.authWindow.logout() self.setWindowTitle("%s %s (synspy: %s)" % (self.ui.title, launcher_version, synspy_version)) self.ui.workList.clearContents() self.ui.workList.setRowCount(0) self.identity = None self.ui.actionLaunch.setEnabled(False) self.ui.actionLogout.setEnabled(False) self.ui.actionLogin.setEnabled(True) @pyqtSlot() def on_actionHelp_triggered(self): pass @pyqtSlot() def on_actionOptions_triggered(self): OptionsDialog.getOptions(self) @pyqtSlot() def on_actionExit_triggered(self): self.closeEvent() QCoreApplication.quit()
] } comment = None schema_def = em.Schema.define( 'isa', comment=comment, acls=acls, annotations=annotations, ) def main(catalog, mode, replace=False): updater = CatalogUpdater(catalog) updater.update_catalog.update_schema(mode, schema_name, schema_def, replace=replace) if __name__ == "__main__": host = 'pbcconsortium.isrd.isi.edu' catalog_id = 1 mode, replace, host, catalog_id = parse_args(host, catalog_id, is_catalog=True) credential = get_credential(host) catalog = ErmrestCatalog('https', host, catalog_id, credentials=credential) main(catalog, mode, replace)
class Annotations: def __init__(self, server, catalog, credentials, config): self.annotations = {} self.ignored_schema_patterns = [] ip = config.get("ignored_schema_patterns") if ip is not None: for p in ip: self.ignored_schema_patterns.append(re.compile(p)) self.types = set() self.managed_attributes = [] self.ignore_unmanaged = True self.ignored_attributes = [] self.consolidated_annotations = {} self.annotations_to_delete = [] if config is not None: known_attributes = config.get("known_attributes") if known_attributes is not None: self.managed_attributes = known_attributes.get("managed", []) self.ignored_attributes = known_attributes.get("ignored", []) self.annotations_to_delete = known_attributes.get("to_delete", []) self.ignore_unmanaged = known_attributes.get("ignore_all_unmanaged", True) self.annotations["known_attributes"] = known_attributes else: self.annotations["known_attributes"] = {'managed': [], 'ignore_all_unmanaged': True} self.consolidated_annotations = config.get("consolidated_annotations", {}) for k in AttrSpecList.SPEC_TYPES: d = self.consolidated_annotations.get(k) if d is None: d = [dict()] self.consolidated_annotations[k] = AttrSpecList(known_attributes, d) self.annotations[k] = self.munge_specs(self.consolidated_annotations[k]) for k in self.managed_attributes: if k in self.annotations_to_delete: raise ValueError("{k} is both 'managed' and 'to_delete'".format(k=k)) self.catalog = ErmrestCatalog('https', server, catalog, credentials) self.catalog_config = self.catalog.getCatalogConfig() if self.catalog_config.annotations is not None: self.add_catalog_annotations(self.catalog_config) if self.catalog_config.schemas is not None: for s in self.catalog_config.schemas.values(): self.add_schema_annotations(s) def munge_specs(self, annotation_list): speclist = [] if annotation_list is not None: if isinstance(annotation_list, AttrSpecList): annotation_list = annotation_list.get_specs() for spec in annotation_list: speclist.append(spec.config_format()) return speclist def consolidated_schema_annotation(self, annotation): matches = [] for c in self.consolidated_annotations["schema_annotations"].get_specs(): if c.schema_entry_matches(annotation.get("schema"), key=annotation.get("uri")): matches.append(c) return self.check_consolidation(matches, annotation.get("value")) def consolidated_table_annotation(self, annotation): matches = [] for c in self.consolidated_annotations["table_annotations"].get_specs(): if c.table_entry_matches(annotation.get("schema"), annotation.get("table"), key=annotation.get("uri")): matches.append(c) return self.check_consolidation(matches, annotation.get("value")) def consolidated_column_annotation(self, annotation): matches = [] for c in self.consolidated_annotations["column_annotations"].get_specs(): if c.column_entry_matches(annotation.get("schema"), annotation.get("table"), annotation.get("column"), key=annotation.get("uri")): matches.append(c) return self.check_consolidation(matches, annotation.get("value")) def consolidated_foreign_key_annotation(self, annotation): matches = [] for c in self.consolidated_annotations["foreign_key_annotations"].get_specs(): if c.foreign_key_entry_matches(annotation.get("schema"), annotation.get("table"), annotation.get("foreign_key_schema"), annotation.get("foreign_key"), key=annotation.get("uri")): matches.append(c) return self.check_consolidation(matches, annotation.get("value")) def check_consolidation(self, matches, value): if len(matches) != 1: # Zero or more than one matching pattern, so we need the exact spec to disambiguate return False match = matches[0] # if match.get("override") == True: # # We don't care what the original version was. We want to go with the pattern match # return True return match.get("value") == value def add_catalog_annotations(self, catalog): annotations = self.find_relevant_annotations(catalog.annotations) if annotations is not None: for v in annotations: self.annotations["catalog_annotations"].append(v) def add_schema_annotations(self, schema): annotations = self.find_relevant_annotations(schema.annotations) if annotations is not None: for v in annotations: v["schema"] = schema.name if not self.consolidated_schema_annotation(v): self.annotations["schema_annotations"].append(v) for table in schema.tables.values(): self.add_table_annotations(table) def add_table_annotations(self, table): annotations = self.find_relevant_annotations(table.annotations) if annotations is not None: for v in annotations: v["schema"] = table.sname v["table"] = table.name if not self.consolidated_table_annotation(v): self.annotations["table_annotations"].append(v) for column in table.column_definitions: self.add_column_annotations(table, column) for fkey in table.foreign_keys: self.add_foreign_key_annotations(fkey) def add_column_annotations(self, table, column): annotations = self.find_relevant_annotations(column.annotations) if annotations is not None: for v in annotations: v["schema"] = table.sname v["table"] = table.name v["column"] = column.name if not self.consolidated_column_annotation(v): self.annotations["column_annotations"].append(v) def add_foreign_key_annotations(self, fkey): annotations = self.find_relevant_annotations(fkey.annotations) if annotations is not None: if len(fkey.names) < 1: raise ValueError("foreign key without a name") for v in annotations: v["schema"] = fkey.sname v["table"] = fkey.tname v["foreign_key_schema"] = fkey.names[0][0] v["foreign_key"] = fkey.names[0][1] if not self.consolidated_foreign_key_annotation(v): self.annotations["foreign_key_annotations"].append(v) def find_relevant_annotations(self, annotations): if annotations is None or len(annotations) == 0: return None new = [] if self.managed_attributes is None: for k in annotations.keys(): if k not in self.annotations_to_delete: new.append({"uri": k, "value": annotations[k]}) self.types.add(k) else: for k in annotations.keys(): if k in self.managed_attributes: new.append({"uri": k, "value": annotations[k]}) self.types.add(k) if len(new) == 0: return None return new def dumps(self): return json.dumps(self.annotations, indent=4, sort_keys=True) def types_list(self): types = list(self.types) types.sort() return types
import deriva.core.ermrest_model as em import argparse parser = argparse.ArgumentParser() parser.add_argument('hostname') parser.add_argument('catalog_number') parser.add_argument('schema_name') args = parser.parse_args() hostname = args.hostname schema_name = args.schema_name catalog_number = args.catalog_number term_table = 'Instance_Level' term_comment = '' credential = get_credential(hostname) catalog = ErmrestCatalog('https', hostname, catalog_number, credentials=credential) def create_vocabulary_table(catalog, term_table, term_comment): model_root = catalog.getCatalogModel() new_vocab_table = \ model_root.schemas[schema_name].create_table(catalog, em.Table.define_vocabulary(term_table,'CORE:{RID}',comment=term_comment) ) create_vocabulary_table(catalog, term_table, term_comment)
with open(pickle_file, 'rb') as pickle_file: vocab_list = pickle.load(pickle_file) filename_list = os.listdir(files_dir) filename_list.sort() credential = get_credential(hostname) server = DerivaServer('https', hostname, credential) catalog = server.connect_ermrest(catalog_number) pb = catalog.getPathBuilder() map_term_value = {} catalog_ermrest = ErmrestCatalog('https', hostname, catalog_number, credentials=credential) model = catalog_ermrest.getCatalogModel() # map to ID of Vocabulary table for vocab_dict in vocab_list: for k, v in vocab_dict.items(): vocab_table_name = '{}_{}_term'.format(k[0], k[1]) vocab_table_name = vocab_table_name[-50:] vocab_table = pb.schemas[vocab_schema_name].tables[vocab_table_name] entities = vocab_table.path.entities() for entity in entities: term_data_map[(k[0], k[1], entity['Name'])] = entity['ID'] # schema = model.schemas[schema_name] # for tab_name in schema.tables.keys():