def main(servername, credentialsfilename, catalog):
def apply(catalog, goal):
"""
Apply the goal configuration to live catalog
"""
print 'applying...'
counter = 0
ready = False
while ready == False:
try:
catalog.applyCatalogConfig(goal)
ready = True
except HTTPError as err:
print err
print err.errno
if err.errno == CONFLICT:
et, ev, tb = sys.exc_info()
print 'Conflict Exception "%s"' % str(ev)
counter = counter + 1
if counter >= 5:
print '%s' % str(traceback.format_exception(et, ev, tb))
ready = True
else:
print 'Retrying...'
except:
et, ev, tb = sys.exc_info()
print str(et)
print 'Exception "%s"' % str(ev)
print '%s' % str(traceback.format_exception(et, ev, tb))
ready = True
credentials = json.load(open(credentialsfilename))
catalog = ErmrestCatalog('https', servername, catalog, credentials)
try:
goal = catalog.get_catalog_model()
except AttributeError:
try:
goal = catalog.getCatalogModel()
except:
et, ev, tb = sys.exc_info()
print 'got exception "%s"' % str(ev)
print '%s' % str(traceback.format_exception(et, ev, tb))
sys.exit(1)
schema_name = 'Microscopy'
table_name = 'Slide'
column_name = 'Label'
goal.column(schema_name, table_name, column_name).column_display.update({
"compact": {"markdown_pattern": "{{#Label}}:::iframe [](/chaise/PrintLabel.html?label=/microscopy/printer/slide/job?{{{Label}}}){height=75 width=150 style=\"border-style: none; border-color: rgb(153, 153, 153);\" .iframe} \n:::{{/Label}}"},
"detailed": {"markdown_pattern": "{{#Label}}:::iframe [](/chaise/PrintLabel.html?label=/microscopy/printer/slide/job?{{{Label}}}){height=75 width=150 style=\"border-style: none; border-color: rgb(153, 153, 153);\" .iframe} \n:::{{/Label}}"}
})
apply(catalog, goal)
print 'Successfully updated the URL annotation for the column %s' % column_name
RMT = '&RMT::geq::%s' % (urlquote(options.RMT))
"""
Get the non NULL "Thumbnail" values from the "Scan" table.
"""
servername = options.server
credentialsfilename = options.credentials
catalog = 1
schema = 'Microscopy'
table = 'Scan'
column = 'Thumbnail'
prefix = '/var/www/html'
output = '%s_add_border.sh' % servername.split('.')[0]
credentials = json.load(open(credentialsfilename))
catalog = ErmrestCatalog('https', servername, catalog, credentials)
url = '/attribute/%s:%s/!%s::null::%s/%s' % (urlquote(schema), urlquote(table), urlquote(column), RMT, urlquote(column))
print ('Query URL: "https://%s/ermrest/catalog/1%s"' % (servername, url))
resp = catalog.get(url)
resp.raise_for_status()
rows = resp.json()
thumbnails = []
for row in rows:
thumbnails.append(row[column])
"""
Generate the shell script.
"""
fkey_defs=fkey_defs,
annotations=table_annotations,
acls=table_acls,
acl_bindings=table_acl_bindings,
comment=table_comment,
provide_system=True)
def main(catalog, mode, replace=False, really=False):
updater = CatalogUpdater(catalog)
table_def['column_annotations'] = column_annotations
table_def['column_comment'] = column_comment
updater.update_table(mode,
schema_name,
table_def,
replace=replace,
really=really)
if __name__ == "__main__":
host = 'pdb.isrd.isi.edu'
catalog_id = 99
mode, replace, host, catalog_id = parse_args(host,
catalog_id,
is_table=True)
catalog = ErmrestCatalog('https',
host,
catalog_id=catalog_id,
credentials=get_credential(host))
main(catalog, mode, replace)
print 'ERROR: Missing credentials file'
sys.exit()
servername = options.server
credentialsfilename = options.credentials
catalog = 1
schema = 'Microscopy'
table = 'Scan'
acquisition = 'Acquisition Date'
czi = 'HTTP URL'
rid = 'RID'
rct = 'RCT'
filename = 'filename'
credentials = json.load(open(credentialsfilename))
catalog = ErmrestCatalog('https', servername, catalog, credentials)
hatrac_store = HatracStore(
'https',
servername,
{'cookie': credentials['cookie']}
)
url = '/attribute/%s:%s/%s::null::/%s,%s,%s,%s' % (urlquote(schema), urlquote(table), urlquote(acquisition), urlquote(rid), urlquote(rct), urlquote(filename), urlquote(czi))
print 'Query URL: "%s"' % url
resp = catalog.get(url)
resp.raise_for_status()
rows = resp.json()
entities = []
for row in rows:
class MainWindow(QMainWindow):
config = None
credential = None
config_path = None
store = None
catalog = None
identity = None
attributes = None
server = None
tempdir = None
progress_update_signal = pyqtSignal(str)
use_3D_viewer = False
curator_mode = False
def __init__(self, config_path=None):
super(MainWindow, self).__init__()
self.ui = MainWindowUI(self)
self.configure(config_path)
self.authWindow = EmbeddedAuthWindow(
self,
config=self.config.get("server"),
cookie_persistence=False,
authentication_success_callback=self.onLoginSuccess
)
self.getSession()
if not self.identity:
self.ui.actionLaunch.setEnabled(False)
self.ui.actionRefresh.setEnabled(False)
self.ui.actionOptions.setEnabled(False)
self.ui.actionLogout.setEnabled(False)
def configure(self, config_path):
# configure logging
self.ui.logTextBrowser.widget.log_update_signal.connect(self.updateLog)
self.ui.logTextBrowser.setFormatter(logging.Formatter("%(asctime)s - %(levelname)s - %(message)s"))
logging.getLogger().addHandler(self.ui.logTextBrowser)
logging.getLogger().setLevel(logging.INFO)
# configure Ermrest/Hatrac
if not config_path:
config_path = os.path.join(os.path.expanduser(
os.path.normpath("~/.deriva/synapse/synspy-launcher")), "config.json")
self.config_path = config_path
config = read_config(self.config_path, create_default=True, default=DEFAULT_CONFIG)
protocol = config["server"]["protocol"]
self.server = config["server"]["host"]
catalog_id = config["server"]["catalog_id"]
session_config = config.get("session")
self.catalog = ErmrestCatalog(protocol, self.server, catalog_id, self.credential, session_config=session_config)
self.store = HatracStore(protocol, self.server, self.credential, session_config=session_config)
# create working dir (tempdir)
self.tempdir = tempfile.mkdtemp(prefix="synspy_")
# determine viewer mode
self.use_3D_viewer = True if config.get("viewer_mode", "2d").lower() == "3d" else False
# curator mode?
curator_mode = config.get("curator_mode")
if not curator_mode:
config["curator_mode"] = False
self.curator_mode = config.get("curator_mode")
# save config
self.config = config
write_config(self.config_path, self.config)
def getSession(self):
qApp.setOverrideCursor(Qt.WaitCursor)
self.updateStatus("Validating session.")
queryTask = SessionQueryTask(self.catalog)
queryTask.status_update_signal.connect(self.onSessionResult)
queryTask.query()
def onLoginSuccess(self, **kwargs):
self.authWindow.hide()
self.credential = kwargs["credential"]
self.catalog.set_credentials(self.credential, self.server)
self.store.set_credentials(self.credential, self.server)
self.getSession()
def enableControls(self):
self.ui.actionLaunch.setEnabled(True)
self.ui.actionRefresh.setEnabled(True)
self.ui.actionOptions.setEnabled(self.authWindow.authenticated())
self.ui.actionLogin.setEnabled(not self.authWindow.authenticated())
self.ui.actionLogout.setEnabled(self.authWindow.authenticated())
self.ui.actionExit.setEnabled(True)
self.ui.workList.setEnabled(True)
def disableControls(self):
self.ui.actionLaunch.setEnabled(False)
self.ui.actionRefresh.setEnabled(False)
self.ui.actionOptions.setEnabled(False)
self.ui.actionLogin.setEnabled(False)
self.ui.actionLogout.setEnabled(False)
self.ui.actionExit.setEnabled(False)
self.ui.workList.setEnabled(False)
def closeEvent(self, event=None):
self.disableControls()
self.cancelTasks()
shutil.rmtree(self.tempdir)
if event:
event.accept()
def cancelTasks(self):
Task.shutdown_all()
self.statusBar().showMessage("Waiting for background tasks to terminate...")
while True:
qApp.processEvents()
if QThreadPool.globalInstance().waitForDone(10):
break
self.statusBar().showMessage("All background tasks terminated successfully")
def is_curator(self):
for attr in self.attributes:
if attr.get('id') == CURATORS:
return True
return False
def displayWorklist(self, worklist):
keys = [
"RID",
"RCT",
"Source Image",
"Classifier",
"Due Date",
"Accepted?",
"Status",
"URL",
"Npz URL",
"ZYX Slice",
"Segmentation Mode",
"Segments URL",
"Segments Filtered URL",
"Subject",
]
self.ui.workList.clear()
self.ui.workList.setRowCount(0)
self.ui.workList.setColumnCount(0)
displayed = ["RID", "RCT", "Segmentation Mode", "Classifier", "Due Date", "Accepted?", "Status"]
self.ui.workList.setRowCount(len(worklist))
self.ui.workList.setColumnCount(len(keys))
self.ui.workList.removeAction(self.ui.markIncompleteAction)
if self.is_curator() and self.curator_mode:
self.ui.workList.addAction(self.ui.markIncompleteAction)
rows = 0
for row in worklist:
value = row.get("Status")
if not (value == "analysis pending" or value == "analysis in progress") \
and not (self.is_curator() and self.curator_mode):
self.ui.workList.hideRow(rows)
cols = 0
for key in keys:
item = QTableWidgetItem()
if key == "Classifier":
value = "%s (%s)" % (row['user'][0]['Full_Name'], row['user'][0]['Display_Name'])
item.setData(Qt.UserRole, row['Classifier'])
elif key == "URL" or key == "Subject":
value = row["source_image"][0].get(key)
else:
value = row.get(key)
if isinstance(value, bool):
value = str(value)
if isinstance(value, str) and key == 'RCT':
value = value.replace('T', ' ')[0:19] # drop fractional seconds and TZ
if isinstance(value, str):
item.setText(value)
item.setToolTip(value)
self.ui.workList.setItem(rows, cols, item)
cols += 1
rows += 1
cols = 0
for key in keys:
if key not in displayed:
self.ui.workList.hideColumn(cols)
cols += 1
self.ui.workList.setHorizontalHeaderLabels(keys) # add header names
self.ui.workList.horizontalHeader().setDefaultAlignment(Qt.AlignLeft) # set alignment
for col in range(len(displayed)):
self.ui.workList.resizeColumnToContents(col)
self.ui.workList.sortByColumn(2, Qt.DescendingOrder)
def getCacheDir(self):
cwd = os.getcwd()
cache_dir = os.path.expanduser(self.config.get("cache_dir", cwd))
if not os.path.isdir(cache_dir):
try:
os.makedirs(cache_dir)
except OSError as error:
if error.errno != errno.EEXIST:
logging.error(format_exception(error))
cache_dir = cwd
return cache_dir
def downloadCallback(self, **kwargs):
status = kwargs.get("progress")
if status:
self.progress_update_signal.emit(status)
return True
def uploadCallback(self, **kwargs):
completed = kwargs.get("completed")
total = kwargs.get("total")
file_path = kwargs.get("file_path")
if completed and total:
file_path = " [%s]" % os.path.basename(file_path) if file_path else ""
status = "Uploading file%s: %d%% complete" % (file_path, round(((completed / total) % 100) * 100))
else:
summary = kwargs.get("summary", "")
file_path = "Uploaded file: [%s] " % os.path.basename(file_path) if file_path else ""
status = file_path # + summary
if status:
self.progress_update_signal.emit(status)
return True
def serverProblemMessageBox(self, text, detail):
msg = QMessageBox()
msg.setIcon(QMessageBox.Warning)
msg.setWindowTitle("Confirm Action")
msg.setText(text)
msg.setInformativeText(detail + "\n\nWould you like to remove this item from the current worklist?")
msg.setStandardButtons(QMessageBox.Yes | QMessageBox.No)
ret = msg.exec_()
if ret == QMessageBox.No:
return
else:
row = self.ui.workList.getCurrentTableRow()
self.ui.workList.removeRow(row)
return
def retrieveFiles(self):
# if there is an existing segments file, download it first, otherwise just initiate the input file download
seg_mode = self.ui.workList.getCurrentTableItemTextByName("Segmentation Mode")
segments_url = self.ui.workList.getCurrentTableItemTextByName("Segments Filtered URL")
if segments_url:
segments_filename = 'ROI_%s_%s_only.csv' % (
self.ui.workList.getCurrentTableItemTextByName("RID"), seg_mode)
segments_destfile = os.path.abspath(os.path.join(self.tempdir, segments_filename))
self.updateStatus("Downloading file: [%s]" % segments_destfile)
downloadTask = FileRetrieveTask(self.store)
downloadTask.status_update_signal.connect(self.onRetrieveAnalysisFileResult)
self.progress_update_signal.connect(self.updateProgress)
downloadTask.retrieve(
segments_url,
destfile=segments_destfile,
progress_callback=self.downloadCallback)
else:
self.retrieveInputFile()
def retrieveInputFile(self):
# get the main TIFF file for analysis if not already cached
if self.use_3D_viewer:
url = self.ui.workList.getCurrentTableItemTextByName("URL")
filename = 'Image_%s.ome.tiff' % self.ui.workList.getCurrentTableItemTextByName("Source Image")
else:
url = self.ui.workList.getCurrentTableItemTextByName("Npz URL")
filename = 'ROI_%s.npz' % self.ui.workList.getCurrentTableItemTextByName("RID")
destfile = os.path.abspath(os.path.join(self.getCacheDir(), filename))
if not url and not self.use_3D_viewer:
self.resetUI("Unable to launch 2D viewer due to missing NPZ file for %s." %
self.ui.workList.getCurrentTableItemTextByName("RID"))
self.serverProblemMessageBox(
"2D viewer requires NPZ data to be present!",
"The launcher is currently configured to execute the 2D viewer, which requires NPZ files for input. " +
"No NPZ file could be found on the server for this task.")
return
if not os.path.isfile(destfile):
self.updateStatus("Downloading file: [%s]" % destfile)
downloadTask = FileRetrieveTask(self.store)
downloadTask.status_update_signal.connect(self.onRetrieveInputFileResult)
self.progress_update_signal.connect(self.updateProgress)
downloadTask.retrieve(
url,
destfile=destfile,
progress_callback=self.downloadCallback)
else:
self.onRetrieveInputFileResult(True, "The file [%s] already exists" % destfile, None, destfile)
def getSubprocessPath(self):
executable = "synspy-viewer" if self.use_3D_viewer else "synspy-viewer2d"
base_path = None
return os.path.normpath(resource_path(executable, base_path))
def executeViewer(self, file_path):
self.updateStatus("Executing viewer...")
env = os.environ
env["SYNSPY_AUTO_DUMP_LOAD"] = "true"
env["DUMP_PREFIX"] = "./ROI_%s" % self.ui.workList.getCurrentTableItemTextByName("RID")
env["ZYX_SLICE"] = self.ui.workList.getCurrentTableItemTextByName("ZYX Slice")
env["ZYX_IMAGE_GRID"] = "0.4, 0.26, 0.26"
env["SYNSPY_DETECT_NUCLEI"] = str(
"nucleic" == self.ui.workList.getCurrentTableItemTextByName("Segmentation Mode")).lower()
output_path = os.path.join(os.path.dirname(self.config_path), "viewer.log")
classifier = self.ui.workList.getTableItemByName(
self.ui.workList.getCurrentTableRow(), "Classifier").data(Qt.UserRole)
viewerTask = ViewerTask(self.getSubprocessPath(), self.identity == classifier, proc_output_path=output_path)
viewerTask.status_update_signal.connect(self.onSubprocessExecuteResult)
viewerTask.run(file_path, self.tempdir, env)
def uploadAnalysisResult(self, update_state):
qApp.setOverrideCursor(Qt.WaitCursor)
# generate hatrac upload params
basename = "ROI_%s" % self.ui.workList.getCurrentTableItemTextByName("RID")
match = r"%s_.*\.csv$" % basename
output_files = [f for f in os.listdir(self.tempdir)
if os.path.isfile(os.path.join(self.tempdir, f)) and re.match(match, f)]
if not output_files:
self.resetUI("Could not locate output file from viewer subprocess -- aborting.")
return
seg_mode = self.ui.workList.getCurrentTableItemTextByName("Segmentation Mode")
if seg_mode == "synaptic":
extension = "_synaptic_only.csv"
elif seg_mode == "nucleic":
extension = "_nucleic_only.csv"
else:
self.updateStatus("Unknown segmentation mode \"%s\" -- aborting." % seg_mode)
return
file_name = basename + extension
hatrac_path = HATRAC_UPDATE_URL_TEMPLATE % \
(self.ui.workList.getCurrentTableItemTextByName("Subject"), file_name)
file_path = os.path.abspath(os.path.join(self.tempdir, file_name))
# upload to object store
self.updateStatus("Uploading file %s to server..." % file_name)
self.progress_update_signal.connect(self.updateProgress)
uploadTask = FileUploadTask(self.store)
uploadTask.status_update_signal.connect(self.onUploadFileResult)
uploadTask.upload(hatrac_path, file_path, update_state, callback=self.uploadCallback)
def markIncomplete(self):
RID = self.ui.workList.getCurrentTableItemTextByName("RID")
body = [{"RID": RID, "Status": "analysis in progress"}]
self.updateStatus("Updating task status for %s..." % RID)
updateTask = CatalogUpdateTask(self.catalog)
updateTask.status_update_signal.connect(self.onCatalogUpdateResult)
updateTask.update(WORKLIST_STATUS_UPDATE, json=body)
@pyqtSlot()
def taskTriggered(self):
self.ui.logTextBrowser.widget.clear()
self.disableControls()
@pyqtSlot(str)
def updateProgress(self, status):
self.statusBar().showMessage(status)
@pyqtSlot(str, str)
def updateStatus(self, status, detail=None):
logging.info(status + ((": %s" % detail) if detail else ""))
self.statusBar().showMessage(status)
@pyqtSlot(str, str)
def resetUI(self, status, detail=None):
qApp.restoreOverrideCursor()
self.updateStatus(status, detail)
self.enableControls()
@pyqtSlot(str)
def updateLog(self, text):
self.ui.logTextBrowser.widget.appendPlainText(text)
@pyqtSlot(bool, str, str, object)
def onSessionResult(self, success, status, detail, result):
qApp.restoreOverrideCursor()
if success:
self.identity = result["client"]["id"]
self.attributes = result["attributes"]
display_name = result["client"]["full_name"]
self.setWindowTitle("%s (%s - %s)" % (self.windowTitle(), self.server, display_name))
self.ui.actionLaunch.setEnabled(True)
self.ui.actionLogout.setEnabled(True)
self.ui.actionLogin.setEnabled(False)
if not self.is_curator():
self.curator_mode = self.config["curator_mode"] = False
self.on_actionRefresh_triggered()
else:
self.updateStatus("Login required.")
@pyqtSlot()
def on_actionLaunch_triggered(self):
self.disableControls()
qApp.setOverrideCursor(Qt.WaitCursor)
# create working dir (tempdir)
if self.tempdir:
shutil.rmtree(self.tempdir)
self.tempdir = tempfile.mkdtemp(prefix="synspy_")
self.retrieveFiles()
@pyqtSlot(bool, str, str, object)
def onRetrieveAnalysisFileResult(self, success, status, detail, file_path):
if not success:
try:
os.remove(file_path)
except Exception as e:
logging.warning("Unable to remove file [%s]: %s" % (file_path, format_exception(e)))
self.resetUI(status, detail)
self.serverProblemMessageBox(
"Unable to download required input file",
"The in-progress analysis file was not downloaded successfully.")
return
self.retrieveInputFile()
@pyqtSlot(bool, str, str, object)
def onRetrieveInputFileResult(self, success, status, detail, file_path):
if not success:
try:
os.remove(file_path)
except Exception as e:
logging.warning("Unable to remove file [%s]: %s" % (file_path, format_exception(e)))
self.resetUI(status, detail)
self.serverProblemMessageBox(
"Unable to download required input file",
"The image input file was not downloaded successfully.")
return
self.executeViewer(file_path)
@pyqtSlot(bool, str, str, object)
def onSubprocessExecuteResult(self, success, status, detail, is_owner):
qApp.restoreOverrideCursor()
if not success:
self.resetUI(status, detail)
return
if not bool(is_owner) or self.curator_mode:
self.resetUI(status, detail)
return
# prompt for save/complete/discard
msg = QMessageBox()
msg.setIcon(QMessageBox.Information)
msg.setWindowTitle("Confirm Action")
msg.setText("How would you like to proceed?")
msg.setInformativeText(
"Select \"Save Progress\" to save your progress and upload the output to the server.\n\n"
"Select \"Complete\" to upload the output to the server and mark this task as completed.\n\n"
"Select \"Discard\" to abort the process and leave the task state unchanged.")
saveButton = msg.addButton("Save Progress", QMessageBox.ActionRole)
completeButton = msg.addButton("Complete", QMessageBox.ActionRole)
discardButton = msg.addButton("Discard", QMessageBox.RejectRole)
msg.exec_()
if msg.clickedButton() == discardButton:
self.resetUI("Aborted.")
return
update_state = None
if msg.clickedButton() == saveButton:
update_state = ("incomplete", "analysis in progress")
elif msg.clickedButton() == completeButton:
update_state = ("complete", "analysis complete")
self.uploadAnalysisResult(update_state)
@pyqtSlot(bool, str, str, object)
def onUploadFileResult(self, success, status, detail, result):
if not success:
self.resetUI(status, detail)
self.serverProblemMessageBox(
"Unable to upload required file(s)",
"One or more required files were not uploaded successfully.")
return
state = result[0]
RID = self.ui.workList.getCurrentTableItemTextByName("RID")
body = [{"RID": RID, "Segments Filtered URL": result[1], "Status": state[1]}]
self.updateStatus("Updating task status for %s..." % RID)
updateTask = CatalogUpdateTask(self.catalog)
updateTask.status_update_signal.connect(self.onCatalogUpdateResult)
updateTask.update(WORKLIST_UPDATE, json=body)
@pyqtSlot(bool, str, str, object)
def onCatalogUpdateResult(self, success, status, detail, result):
if not success:
self.resetUI(status, detail)
self.serverProblemMessageBox(
"Unable to update catalog data",
"The catalog state was not updated successfully.")
return
qApp.restoreOverrideCursor()
self.on_actionRefresh_triggered()
@pyqtSlot()
def on_actionRefresh_triggered(self):
if not self.identity:
self.updateStatus("Unable to get worklist -- not logged in.")
return
qApp.setOverrideCursor(Qt.WaitCursor)
self.disableControls()
self.updateStatus("Refreshing worklist...")
queryTask = CatalogQueryTask(self.catalog)
queryTask.status_update_signal.connect(self.onRefreshResult)
if self.is_curator() and self.curator_mode:
queryTask.query(WORKLIST_CURATOR_QUERY)
else:
queryTask.query(WORKLIST_QUERY % urlquote(self.identity, ""))
@pyqtSlot(bool, str, str, object)
def onRefreshResult(self, success, status, detail, result):
if success:
self.displayWorklist(result)
self.resetUI("Ready.")
else:
self.resetUI(status, detail)
if (self.ui.workList.rowCount() > 0) and self.identity:
self.ui.actionLaunch.setEnabled(True)
else:
self.ui.actionLaunch.setEnabled(False)
@pyqtSlot()
def on_actionLogin_triggered(self):
self.authWindow.show()
self.authWindow.login()
@pyqtSlot()
def on_actionLogout_triggered(self):
self.authWindow.logout()
self.setWindowTitle("%s %s (synspy: %s)" % (self.ui.title, launcher_version, synspy_version))
self.ui.workList.clearContents()
self.ui.workList.setRowCount(0)
self.identity = None
self.ui.actionLaunch.setEnabled(False)
self.ui.actionLogout.setEnabled(False)
self.ui.actionLogin.setEnabled(True)
@pyqtSlot()
def on_actionHelp_triggered(self):
pass
@pyqtSlot()
def on_actionOptions_triggered(self):
OptionsDialog.getOptions(self)
@pyqtSlot()
def on_actionExit_triggered(self):
self.closeEvent()
QCoreApplication.quit()
]
}
comment = None
schema_def = em.Schema.define(
'isa',
comment=comment,
acls=acls,
annotations=annotations,
)
def main(catalog, mode, replace=False):
updater = CatalogUpdater(catalog)
updater.update_catalog.update_schema(mode,
schema_name,
schema_def,
replace=replace)
if __name__ == "__main__":
host = 'pbcconsortium.isrd.isi.edu'
catalog_id = 1
mode, replace, host, catalog_id = parse_args(host,
catalog_id,
is_catalog=True)
credential = get_credential(host)
catalog = ErmrestCatalog('https', host, catalog_id, credentials=credential)
main(catalog, mode, replace)
class Annotations:
def __init__(self, server, catalog, credentials, config):
self.annotations = {}
self.ignored_schema_patterns = []
ip = config.get("ignored_schema_patterns")
if ip is not None:
for p in ip:
self.ignored_schema_patterns.append(re.compile(p))
self.types = set()
self.managed_attributes = []
self.ignore_unmanaged = True
self.ignored_attributes = []
self.consolidated_annotations = {}
self.annotations_to_delete = []
if config is not None:
known_attributes = config.get("known_attributes")
if known_attributes is not None:
self.managed_attributes = known_attributes.get("managed", [])
self.ignored_attributes = known_attributes.get("ignored", [])
self.annotations_to_delete = known_attributes.get("to_delete", [])
self.ignore_unmanaged = known_attributes.get("ignore_all_unmanaged", True)
self.annotations["known_attributes"] = known_attributes
else:
self.annotations["known_attributes"] = {'managed': [], 'ignore_all_unmanaged': True}
self.consolidated_annotations = config.get("consolidated_annotations", {})
for k in AttrSpecList.SPEC_TYPES:
d = self.consolidated_annotations.get(k)
if d is None:
d = [dict()]
self.consolidated_annotations[k] = AttrSpecList(known_attributes, d)
self.annotations[k] = self.munge_specs(self.consolidated_annotations[k])
for k in self.managed_attributes:
if k in self.annotations_to_delete:
raise ValueError("{k} is both 'managed' and 'to_delete'".format(k=k))
self.catalog = ErmrestCatalog('https', server, catalog, credentials)
self.catalog_config = self.catalog.getCatalogConfig()
if self.catalog_config.annotations is not None:
self.add_catalog_annotations(self.catalog_config)
if self.catalog_config.schemas is not None:
for s in self.catalog_config.schemas.values():
self.add_schema_annotations(s)
def munge_specs(self, annotation_list):
speclist = []
if annotation_list is not None:
if isinstance(annotation_list, AttrSpecList):
annotation_list = annotation_list.get_specs()
for spec in annotation_list:
speclist.append(spec.config_format())
return speclist
def consolidated_schema_annotation(self, annotation):
matches = []
for c in self.consolidated_annotations["schema_annotations"].get_specs():
if c.schema_entry_matches(annotation.get("schema"), key=annotation.get("uri")):
matches.append(c)
return self.check_consolidation(matches, annotation.get("value"))
def consolidated_table_annotation(self, annotation):
matches = []
for c in self.consolidated_annotations["table_annotations"].get_specs():
if c.table_entry_matches(annotation.get("schema"),
annotation.get("table"),
key=annotation.get("uri")):
matches.append(c)
return self.check_consolidation(matches, annotation.get("value"))
def consolidated_column_annotation(self, annotation):
matches = []
for c in self.consolidated_annotations["column_annotations"].get_specs():
if c.column_entry_matches(annotation.get("schema"),
annotation.get("table"),
annotation.get("column"),
key=annotation.get("uri")):
matches.append(c)
return self.check_consolidation(matches, annotation.get("value"))
def consolidated_foreign_key_annotation(self, annotation):
matches = []
for c in self.consolidated_annotations["foreign_key_annotations"].get_specs():
if c.foreign_key_entry_matches(annotation.get("schema"),
annotation.get("table"),
annotation.get("foreign_key_schema"),
annotation.get("foreign_key"),
key=annotation.get("uri")):
matches.append(c)
return self.check_consolidation(matches, annotation.get("value"))
def check_consolidation(self, matches, value):
if len(matches) != 1:
# Zero or more than one matching pattern, so we need the exact spec to disambiguate
return False
match = matches[0]
# if match.get("override") == True:
# # We don't care what the original version was. We want to go with the pattern match
# return True
return match.get("value") == value
def add_catalog_annotations(self, catalog):
annotations = self.find_relevant_annotations(catalog.annotations)
if annotations is not None:
for v in annotations:
self.annotations["catalog_annotations"].append(v)
def add_schema_annotations(self, schema):
annotations = self.find_relevant_annotations(schema.annotations)
if annotations is not None:
for v in annotations:
v["schema"] = schema.name
if not self.consolidated_schema_annotation(v):
self.annotations["schema_annotations"].append(v)
for table in schema.tables.values():
self.add_table_annotations(table)
def add_table_annotations(self, table):
annotations = self.find_relevant_annotations(table.annotations)
if annotations is not None:
for v in annotations:
v["schema"] = table.sname
v["table"] = table.name
if not self.consolidated_table_annotation(v):
self.annotations["table_annotations"].append(v)
for column in table.column_definitions:
self.add_column_annotations(table, column)
for fkey in table.foreign_keys:
self.add_foreign_key_annotations(fkey)
def add_column_annotations(self, table, column):
annotations = self.find_relevant_annotations(column.annotations)
if annotations is not None:
for v in annotations:
v["schema"] = table.sname
v["table"] = table.name
v["column"] = column.name
if not self.consolidated_column_annotation(v):
self.annotations["column_annotations"].append(v)
def add_foreign_key_annotations(self, fkey):
annotations = self.find_relevant_annotations(fkey.annotations)
if annotations is not None:
if len(fkey.names) < 1:
raise ValueError("foreign key without a name")
for v in annotations:
v["schema"] = fkey.sname
v["table"] = fkey.tname
v["foreign_key_schema"] = fkey.names[0][0]
v["foreign_key"] = fkey.names[0][1]
if not self.consolidated_foreign_key_annotation(v):
self.annotations["foreign_key_annotations"].append(v)
def find_relevant_annotations(self, annotations):
if annotations is None or len(annotations) == 0:
return None
new = []
if self.managed_attributes is None:
for k in annotations.keys():
if k not in self.annotations_to_delete:
new.append({"uri": k, "value": annotations[k]})
self.types.add(k)
else:
for k in annotations.keys():
if k in self.managed_attributes:
new.append({"uri": k, "value": annotations[k]})
self.types.add(k)
if len(new) == 0:
return None
return new
def dumps(self):
return json.dumps(self.annotations, indent=4, sort_keys=True)
def types_list(self):
types = list(self.types)
types.sort()
return types
import deriva.core.ermrest_model as em
import argparse
parser = argparse.ArgumentParser()
parser.add_argument('hostname')
parser.add_argument('catalog_number')
parser.add_argument('schema_name')
args = parser.parse_args()
hostname = args.hostname
schema_name = args.schema_name
catalog_number = args.catalog_number
term_table = 'Instance_Level'
term_comment = ''
credential = get_credential(hostname)
catalog = ErmrestCatalog('https',
hostname,
catalog_number,
credentials=credential)
def create_vocabulary_table(catalog, term_table, term_comment):
model_root = catalog.getCatalogModel()
new_vocab_table = \
model_root.schemas[schema_name].create_table(catalog, em.Table.define_vocabulary(term_table,'CORE:{RID}',comment=term_comment)
)
create_vocabulary_table(catalog, term_table, term_comment)
with open(pickle_file, 'rb') as pickle_file:
vocab_list = pickle.load(pickle_file)
filename_list = os.listdir(files_dir)
filename_list.sort()
credential = get_credential(hostname)
server = DerivaServer('https', hostname, credential)
catalog = server.connect_ermrest(catalog_number)
pb = catalog.getPathBuilder()
map_term_value = {}
catalog_ermrest = ErmrestCatalog('https',
hostname,
catalog_number,
credentials=credential)
model = catalog_ermrest.getCatalogModel()
# map to ID of Vocabulary table
for vocab_dict in vocab_list:
for k, v in vocab_dict.items():
vocab_table_name = '{}_{}_term'.format(k[0], k[1])
vocab_table_name = vocab_table_name[-50:]
vocab_table = pb.schemas[vocab_schema_name].tables[vocab_table_name]
entities = vocab_table.path.entities()
for entity in entities:
term_data_map[(k[0], k[1], entity['Name'])] = entity['ID']
# schema = model.schemas[schema_name]
# for tab_name in schema.tables.keys():
