def __init__(self, **kwargs):
self.client = None
self.client_id = kwargs.get("client_id") or NATIVE_APP_CLIENT_ID
self.hosts = kwargs.get("hosts")
self.config_file = kwargs.get("config_file")
self.default_scopes = DEFAULT_SCOPES.copy()
if self.client_id == NATIVE_APP_CLIENT_ID:
self.default_scopes.append(GROUPS_SCOPE_NAME)
try:
global GLOBUS_SDK, NATIVE_LOGIN
GLOBUS_SDK = importlib.import_module("globus_sdk")
NATIVE_LOGIN = importlib.import_module("fair_research_login")
except Exception as e:
raise DependencyError("Unable to load a required module: %s" %
format_exception(e))
try:
storage_file = 'globus-credential%s.json' % \
(("-" + self.client_id) if self.client_id != NATIVE_APP_CLIENT_ID else "")
storage = DerivaJSONTokenStorage(
filename=os.path.join(DEFAULT_CONFIG_PATH, storage_file))
self.client = NATIVE_LOGIN.NativeClient(
client_id=self.client_id,
token_storage=storage,
app_name="Login from deriva-client on %s [%s]%s" %
(platform.uname()[1], platform.platform(aliased=True),
" to hosts [%s]" %
", ".join(self.hosts) if self.hosts else ""),
default_scopes=self.default_scopes)
except Exception as e:
logging.error("Unable to instantiate a required class: %s" %
format_exception(e))
def main(self):
args = self.parse_cli()
def _cmd_error_message(emsg):
return "{prog} {subcmd}: {msg}".format(prog=self.parser.prog,
subcmd=args.subcmd,
msg=emsg)
try:
if not hasattr(args, 'func'):
self.parser.print_usage()
return 2
if args.subcmd == "login" or args.subcmd == "logout" or args.subcmd == "user-info":
self.gnl = GlobusNativeLogin(**vars(args))
else:
self.gau = GlobusAuthUtil(**vars(args))
response = args.func(args)
if args.pretty:
if isinstance(response, dict) or isinstance(response, list):
try:
print(json.dumps(response, indent=2))
return
except:
pprint(response)
return
elif not isinstance(response, str):
pprint(response)
return
print(response)
return
except UsageException as e:
eprint("{prog} {subcmd}: {msg}".format(prog=self.parser.prog,
subcmd=args.subcmd,
msg=e))
except ConnectionError as e:
eprint("{prog}: Connection error occurred".format(
prog=self.parser.prog))
except GlobusAuthAPIError as e:
if 401 == e.http_status:
msg = 'Authentication required: %s' % e.message
elif 403 == e.http_status:
msg = 'Permission denied: %s' % e.message
else:
msg = e
logging.debug(format_exception(e))
eprint(_cmd_error_message(msg))
except DependencyError as e:
eprint(_cmd_error_message(e))
except RuntimeError as e:
logging.debug(format_exception(e))
eprint('An unexpected runtime error occurred')
except:
eprint('An unexpected error occurred')
traceback.print_exc()
return 1
def _getFileRecord(self, asset_mapping):
"""
Helper function that queries the catalog to get a record linked to the asset, or create it if it doesn't exist.
:return: the file record
"""
column_map = asset_mapping.get("column_map", {})
rqt = asset_mapping['record_query_template']
try:
path = rqt.format(**self.metadata)
except KeyError as e:
raise ConfigurationError(
"Record query template substitution error: %s" %
format_exception(e))
result = self.catalog.get(path).json()
if result:
self._updateFileMetadata(result[0])
return self.pruneDict(result[0], column_map)
else:
row = self.interpolateDict(self.metadata, column_map)
result = self._catalogRecordCreate(self.metadata['target_table'],
row)
if result:
self._updateFileMetadata(result[0])
return self.interpolateDict(self.metadata,
column_map,
allowNone=True)
def main(self):
args = self.parse_cli()
if args.path is None:
sys.stderr.write("\nError: Input directory not specified.\n")
self.parser.print_usage()
return 2
if not args.quiet:
sys.stderr.write("\n")
try:
DerivaUploadCLI.upload(self.uploader,
os.path.abspath(args.path),
args.host,
args.catalog,
args.token,
args.config_file,
args.credential_file,
args.no_config_update,
args.purge_state)
except (RuntimeError, DerivaUploadError, DerivaUploadConfigurationError, DerivaUploadCatalogCreateError,
DerivaUploadCatalogUpdateError) as e:
sys.stderr.write(("\n" if not args.quiet else "") + format_exception(e))
if args.debug:
traceback.print_exc()
return 1
except:
traceback.print_exc()
return 1
finally:
if not args.quiet:
sys.stderr.write("\n\n")
return 0
def __init__(self, **kwargs):
client_id = kwargs.get("client_id")
client_secret = kwargs.get("client_secret")
if not (client_id and client_secret):
cred_file = kwargs.get("credential_file", CLIENT_CRED_FILE)
creds = read_config(cred_file)
if creds:
client = creds.get("web")
if client:
client_id = client.get('client_id')
client_secret = client.get('client_secret')
try:
global GLOBUS_SDK, GlobusAuthAPIError
GLOBUS_SDK = importlib.import_module("globus_sdk")
GlobusAuthAPIError = GLOBUS_SDK.AuthAPIError
except Exception as e:
raise DependencyError("Unable to load a required module: %s" %
format_exception(e))
if not (client_id and client_secret):
logging.warning(
"Client ID and secret not specified and/or could not be determined."
)
self.client = GLOBUS_SDK.ConfidentialAppAuthClient(
client_id, client_secret)
self.client_id = client_id
def main(self):
args = self.parse_cli()
self.dumpdir = args.dir
self.host = args.host
self.catalog_id = args.catalog
self.graph_format = args.graph_format
if self.host is None:
eprint('Host name must be provided')
return 1
self.catalog = ErmrestCatalog('https', self.host, self.catalog_id, credentials=self._get_credential(self.host))
self.model = self.catalog.getCatalogModel()
self.schemas = [s for s in (args.schemas if args.schemas else self.model.schemas)
if s not in args.skip_schemas
]
try:
os.makedirs(self.dumpdir, exist_ok=True)
except OSError as e:
sys.stderr.write(str(e))
return 1
logger.info('Catalog has {} schema and {} tables'.format(len(self.model.schemas),
sum([len(v.tables) for k, v in
self.model.schemas.items()])))
logger.info('\n'.join([' {} has {} tables'.format(k, len(s.tables))
for k, s in self.model.schemas.items()]))
try:
if args.table:
if ':' not in args.table:
raise DerivaDumpCatalogException('Table name must be in form of schema:table')
[schema_name, table_name] = args.table.split(":")
self._dump_table(schema_name, table_name)
elif args.graph:
self._graph_catalog()
else:
self._dump_catalog()
except DerivaDumpCatalogException as e:
print(e)
except HTTPError as e:
if e.response.status_code == requests.codes.unauthorized:
msg = 'Authentication required for {}'.format(args.server)
elif e.response.status_code == requests.codes.forbidden:
msg = 'Permission denied'
else:
msg = e
logging.debug(format_exception(e))
eprint(msg)
except RuntimeError as e:
sys.stderr.write(str(e))
return 1
except:
traceback.print_exc()
return 1
finally:
sys.stderr.write("\n\n")
return
def main():
cli = AclCLI()
args = cli.parse_cli()
table_name = cli.get_table_arg(args)
schema_names = cli.get_schema_arg_list(args)
credentials = get_credential(args.host, args.credential_file)
save_groups = not (args.dryrun or args.omit_groups)
for schema in schema_names:
acl_config = AclConfig(args.host,
args.catalog,
args.config_file,
credentials,
schema_name=schema,
table_name=table_name,
verbose=args.verbose or args.debug)
try:
if save_groups:
acl_config.save_groups()
save_groups = False
if not args.groups_only:
acl_config.set_acls()
if not args.dryrun:
acl_config.apply_acls()
except HTTPError as e:
print(format_exception(e))
raise
if args.dryrun:
print(acl_config.dumps())
def _catalogRecordCreate(self, catalog_table, row, default_columns=None):
"""
:param catalog_table:
:param row:
:param default_columns:
:return:
"""
if self.cancelled:
return None
try:
missing = self.catalog.validateRowColumns(row, catalog_table)
if missing:
raise CatalogCreateError(
"Unable to update catalog entry because one or more specified columns do not exist in the "
"target table: [%s]" % ','.join(missing))
if not default_columns:
default_columns = self.catalog.getDefaultColumns(
row, catalog_table)
default_param = (
'?defaults=%s' %
','.join(default_columns)) if len(default_columns) > 0 else ''
# for default in default_columns:
# row[default] = None
create_uri = '/entity/%s%s' % (catalog_table, default_param)
logging.debug(
"Attempting catalog record create [%s] with data: %s" %
(create_uri, json.dumps(row)))
return self.catalog.post(create_uri, json=[row]).json()
except:
(etype, value, traceback) = sys.exc_info()
raise CatalogCreateError(format_exception(value))
def createManifestEntry(self, entry):
manifest_entry = dict()
url = entry.get("url")
if not url:
logging.warning(
"Skipping a record due to missing required attribute \"url\" in fetch manifest entry %s" %
json.dumps(entry))
return
ext_url = self.getExternalUrl(url)
length = entry.get("length")
md5 = entry.get("md5")
sha256 = entry.get("sha256")
filename = entry.get("filename")
content_type = entry.get("content_type")
content_disposition = None
# if any required fields are missing from the query result, attempt to get them from the remote server by
# issuing a HEAD request against the supplied URL
if not (length and (md5 or sha256)):
try:
headers = self.headForHeaders(url, raise_for_status=True)
except requests.HTTPError as e:
raise DerivaDownloadError("Exception during HEAD request: %s" % format_exception(e))
length = headers.get("Content-Length")
content_type = headers.get("Content-Type")
content_disposition = headers.get("Content-Disposition")
if not md5:
md5 = headers.get("Content-MD5")
if md5:
md5 = decodeBase64toHex(md5)
if not sha256:
sha256 = headers.get("Content-SHA256")
if sha256:
sha256 = decodeBase64toHex(sha256)
# if content length or both hash values are missing, it is a fatal error
if not length:
raise DerivaDownloadError("Could not determine Content-Length for %s" % ext_url)
if not (md5 or sha256):
raise DerivaDownloadError("Could not locate an MD5 or SHA256 hash for %s" % ext_url)
envvars = self.envars.copy()
envvars.update(entry)
subdir = self.sub_path.format(**envvars)
# if a local filename is not provided, try to construct one using content_disposition, if available
if not filename:
filename = os.path.basename(url).split(":")[0] if not content_disposition else \
parse_content_disposition(content_disposition)
output_path = ''.join([subdir, "/", filename]) if subdir else filename
manifest_entry['url'] = ext_url
manifest_entry['length'] = int(length)
manifest_entry['filename'] = output_path
if md5:
manifest_entry['md5'] = md5
if sha256:
manifest_entry['sha256'] = sha256
if content_type:
manifest_entry["content_type"] = content_type
return manifest_entry
def main(self):
"""Main routine of the CLI.
"""
args = self.parse_cli()
def _resource_error_message(emsg):
return "{prog} {subcmd}: {resource}: {msg}".format(
prog=self.parser.prog, subcmd=args.subcmd, resource=args.resource, msg=emsg)
try:
if not hasattr(args, 'func'):
self.parser.print_usage()
return 1
self._post_parser_init(args)
args.func(args)
return 0
except UsageException as e:
eprint("{prog} {subcmd}: {msg}".format(prog=self.parser.prog, subcmd=args.subcmd, msg=e))
except ConnectionError as e:
eprint("{prog}: Connection error occurred".format(prog=self.parser.prog))
except DerivaPathError as e:
eprint(e)
except HTTPError as e:
if e.response.status_code == requests.codes.unauthorized:
msg = 'Authentication required'
elif e.response.status_code == requests.codes.forbidden:
msg = 'Permission denied'
else:
msg = e
logging.debug(format_exception(e))
eprint(_resource_error_message(msg))
except ResourceException as e:
logging.debug(format_exception(e.cause))
eprint(_resource_error_message(e))
except HatracHashMismatch as e:
logging.debug(format_exception(e))
eprint(_resource_error_message('Checksum verification failed'))
except RuntimeError as e:
logging.debug(format_exception(e))
eprint('Unexpected runtime error occurred')
except:
eprint('Unexpected error occurred')
traceback.print_exc()
return 1
def _queryFileMetadata(self, file_path, asset_mapping, match_groupdict):
"""
Helper function that queries the catalog to get required metadata for a given file/asset
"""
file_name = self.getFileDisplayName(file_path)
logging.info("Computing metadata for file: [%s]." % file_name)
self._initFileMetadata(file_path, asset_mapping, match_groupdict)
logging.info("Computing checksums for file: [%s]. Please wait..." %
file_name)
hashes = self.getFileHashes(
file_path, asset_mapping.get('checksum_types', ['md5', 'sha256']))
for alg, checksum in hashes.items():
alg = alg.lower()
self.metadata[alg] = checksum[0]
self.metadata[alg + "_base64"] = checksum[1]
for uri in asset_mapping.get("metadata_query_templates", []):
try:
path = uri.format(**self.metadata)
except KeyError as e:
raise RuntimeError(
"Metadata query template substitution error: %s" %
format_exception(e))
result = self.catalog.get(path).json()
if result:
self._updateFileMetadata(result[0], True)
self._urlEncodeMetadata(
asset_mapping.get("url_encoding_safe_overrides"))
else:
raise RuntimeError(
"Metadata query did not return any results: %s" % path)
self._getFileExtensionMetadata(self.metadata.get("file_ext"))
for k, v in asset_mapping.get("column_value_templates", {}).items():
try:
self.metadata[k] = v.format(**self.metadata)
except KeyError as e:
logging.warning(
"Column value template substitution error: %s" %
format_exception(e))
continue
self._urlEncodeMetadata(
asset_mapping.get("url_encoding_safe_overrides"))
def GET(self, catalog_id):
warnings.warn(
"The '{}' service is experimental and not intended for production usage."
.format(__name__))
logger.debug("query: {}".format(web.ctx.query))
params = [urlunquote(param) for param in web.ctx.query[1:].split('&')]
logger.debug("params: {}".format(params))
try:
auth_token = web.cookies().get("webauthn")
credentials = format_credential(
token=auth_token) if auth_token else None
return pattern_transformer(catalog_id, params, credentials)
except requests.HTTPError as e:
raise RestException.from_http_error(e)
except ValueError as e:
raise BadRequest(format_exception(e))
except KeyError as e:
raise BadRequest(format_exception(e))
def cleanupTransferState(self):
if self.transfer_state_fp and not self.transfer_state_fp.closed:
try:
self.transfer_state_fp.flush()
self.transfer_state_fp.close()
except Exception as e:
logging.warning(
"Unable to flush/close transfer state file: %s" %
format_exception(e))
def excepthook(etype, value, tb):
traceback.print_tb(tb)
print(format_exception(value), file=sys.stderr)
msg = QMessageBox()
msg.setText(str(value))
msg.setStandardButtons(QMessageBox.Close)
msg.setWindowTitle("Unhandled Exception: %s" % etype.__name__)
msg.setIcon(QMessageBox.Critical)
msg.setDetailedText('\n'.join(traceback.format_exception(etype, value, tb)))
msg.exec_()
def main(self):
try:
args = self.parse_cli()
except ValueError as e:
sys.stderr.write(str(e))
return 2
if not args.quiet:
sys.stderr.write("\n")
try:
try:
downloaded = self.download(args)
sys.stdout.write("\n%s\n" % (json.dumps(downloaded)))
except ConnectionError as e:
raise DerivaDownloadError("Connection error occurred. %s" %
format_exception(e))
except HTTPError as e:
if e.response.status_code == requests.codes.unauthorized:
raise DerivaDownloadAuthenticationError(
"The requested service requires authentication and a valid login session could "
"not be found for the specified host. Server responded: %s"
% e)
elif e.response.status_code == requests.codes.forbidden:
raise DerivaDownloadAuthorizationError(
"A requested operation was forbidden. Server responded: %s"
% e)
except (DerivaDownloadError, DerivaDownloadConfigurationError,
DerivaDownloadAuthenticationError,
DerivaDownloadAuthorizationError) as e:
sys.stderr.write(("\n" if not args.quiet else "") +
format_exception(e))
if args.debug:
traceback.print_exc()
return 1
except:
sys.stderr.write("An unexpected error occurred.")
traceback.print_exc()
return 1
finally:
if not args.quiet:
sys.stderr.write("\n\n")
return 0
def writeTransferState(self):
if not self.transfer_state_fp:
return
try:
self.transfer_state_fp.seek(0, 0)
self.transfer_state_fp.truncate()
json.dump(self.transfer_state, self.transfer_state_fp, indent=2)
self.transfer_state_fp.flush()
except Exception as e:
logging.warning("Unable to write transfer state file: %s" %
format_exception(e))
def getCacheDir(self):
cwd = os.getcwd()
cache_dir = os.path.expanduser(self.config.get("cache_dir", cwd))
if not os.path.isdir(cache_dir):
try:
os.makedirs(cache_dir)
except OSError as error:
if error.errno != errno.EEXIST:
logging.error(format_exception(error))
cache_dir = cwd
return cache_dir
def _onPreAuthContent(self, content):
try:
if not content:
logging.debug("no preauth content")
return
preauth = json.loads(content)
logging.trace("webauthn preauth:\n%s\n", json.dumps(preauth, indent=2))
qApp.setOverrideCursor(Qt.WaitCursor)
self.authn_session_page.setUrl(QUrl(preauth["redirect_url"]))
except (ValueError, Exception) as e:
logging.error(format_exception(e))
self.set_current_html(ERROR_HTML % content)
def onRetrieveInputFileResult(self, success, status, detail, file_path):
if not success:
try:
os.remove(file_path)
except Exception as e:
logging.warning("Unable to remove file [%s]: %s" % (file_path, format_exception(e)))
self.resetUI(status, detail)
self.serverProblemMessageBox(
"Unable to download required input file",
"The image input file was not downloaded successfully.")
return
self.executeViewer(file_path)
def catalogQuery(self, headers=HEADERS, as_file=True):
if as_file:
output_dir = os.path.dirname(self.output_abspath)
self.makeDirs(output_dir)
try:
if as_file:
return self.catalog.getAsFile(self.query,
self.output_abspath,
headers=headers)
else:
return self.catalog.get(self.query, headers=headers).json()
except requests.HTTPError as e:
raise RuntimeError("Unable to execute catalog query: %s" %
format_exception(e))
def logout(self, delete_cookies=False):
if not (self.auth_url and
(self.auth_url.host() and self.auth_url.scheme())):
return
if not self.authenticated():
return
try:
logging.info("Logging out of host: %s" % self.auth_url.toString())
if delete_cookies and self.cookie_persistence:
self.authn_session_page.profile().cookieStore(
).deleteAllCookies()
self._session.delete(self.auth_url.toString() + "/authn/session")
except Exception as e:
logging.warning("Logout error: %s" % format_exception(e))
self._cleanup()
def _getFileHatracMetadata(self, asset_mapping):
try:
hatrac_templates = asset_mapping["hatrac_templates"]
# URI is required
self.metadata["URI"] = hatrac_templates["hatrac_uri"].format(
**self.metadata)
# overridden content-disposition is optional
content_disposition = hatrac_templates.get("content-disposition")
self.metadata["content-disposition"] = \
None if not content_disposition else content_disposition.format(**self.metadata)
self._urlEncodeMetadata(
asset_mapping.get("url_encoding_safe_overrides"))
except KeyError as e:
raise ConfigurationError("Hatrac template substitution error: %s" %
format_exception(e))
def main(self):
args = self.parse_cli()
catalog = DerivaCatalogConfigure(args.host, catalog_id=args.catalog_model)
try:
catalog = DerivaCatalog(args.host, args.catalog_model)
[schema_name, table_name] = args.table_model.split(':')
table = DerivaTable(catalog, schema_name, table_name)
if args.asset_table:
table.create_asset_table(args.asset_table)
if args._visible_columns:
table.create_default_visible_columns(really=args.replace)
if args.configure == 'catalog':
logging.info('Configuring catalog {}:{}'.format(args.host, args.catalog_model))
cfg = DerivaCatalogConfigure(args.host, catalog_id=args.catalog_model)
cfg.configure_baseline_catalog(catalog_name=args.catalog_name,
reader=args.reader, writer=args.writer, curator=args.curator,
admin=args.admin,
set_policy=args.set_policy, public=args.publish)
cfg.apply()
if args.table_model:
[schema_name, table_name] = args.table_model.split(':')
table = catalog.schema(schema_name).table_model(table_name)
table.configure_table_defaults(set_policy=args.set_policy, public=args.publish)
table.apply()
except DerivaConfigError as e:
print(e.msg)
except HTTPError as e:
if e.response.status_code == requests.codes.unauthorized:
msg = 'Authentication required for {}'.format(args.host)
elif e.response.status_code == requests.codes.forbidden:
msg = 'Permission denied'
else:
msg = e
logging.debug(format_exception(e))
eprint(msg)
except RuntimeError as e:
sys.stderr.write(str(e))
return 1
except:
traceback.print_exc()
return 1
finally:
sys.stderr.write("\n\n")
return
def logout(self, delete_cookies=False):
if not (self.auth_url and (self.auth_url.host() and self.auth_url.scheme())):
return
if self.authenticated():
try:
logging.info("Logging out of host: %s" % self.auth_url.toString())
if delete_cookies and self.cookie_persistence:
self.authn_session_page.profile().cookieStore().deleteAllCookies()
self._session.delete(self.auth_url.toString() + "/authn/session")
if self.credential_file:
creds = read_credential(self.credential_file, create_default=True)
host = self.auth_url.host()
if creds.get(host):
del creds[host]
write_credential(self.credential_file, creds)
except Exception as e:
logging.warning("Logout error: %s" % format_exception(e))
self._cleanup()
def _onSessionContent(self, content):
try:
self.setHtml(SUCCESS_HTML)
self.authn_session = json.loads(content)
seconds_remaining = self.authn_session['seconds_remaining']
if not self._timer.isActive():
interval = seconds_remaining // 2
logging.info(
"Authentication successful for [%s]: credential refresh in %d seconds."
% (self.auth_url.toString(), interval))
self._timer.start(interval * 1000)
self.authn_expires = time.time() + seconds_remaining + 1
logging.trace("webauthn session:\n%s\n",
json.dumps(self.authn_session, indent=2))
qApp.restoreOverrideCursor()
QTimer.singleShot(100, self._execSuccessCallback)
except (ValueError, Exception) as e:
logging.error(format_exception(e))
self.setHtml(ERROR_HTML % content)
def purge_output_dirs(threshold=0, count=1):
if threshold < 1:
return
basedir = get_staging_path()
if not os.path.isdir(basedir):
return
paths = [
os.fspath(path) for path in sorted(
Path(basedir).iterdir(), key=os.path.getctime, reverse=True)
]
if not paths or (len(paths) < threshold):
return
for i in range(count):
try:
path = paths.pop()
if os.path.isdir(path):
shutil.rmtree(path)
except Exception as e:
logging.warning(format_exception(e))
def _onSessionContent(self, content):
try:
qApp.restoreOverrideCursor()
self.set_current_html(SUCCESS_HTML)
try:
self.authn_session = json.loads(content)
except json.JSONDecodeError:
raise RuntimeError("Unable to parse response from server: %s" % content)
seconds_remaining = self.authn_session['seconds_remaining']
if not self._timer.isActive():
interval = seconds_remaining // 2
logging.info("Authentication successful for [%s]: credential refresh in %d seconds." %
(self.auth_url.toString(), interval))
self._timer.start(interval * 1000)
self.authn_expires = time.time() + seconds_remaining + 1
logging.trace("webauthn session:\n%s\n", json.dumps(self.authn_session, indent=2))
QTimer.singleShot(100, self._execSuccessCallback)
except (ValueError, Exception) as e:
error = format_exception(e)
logging.error(error)
self.set_current_html(ERROR_HTML % content)
self._execFailureCallback(error)
def _catalogRecordUpdate(self, catalog_table, old_row, new_row):
"""
:param catalog_table:
:param new_row:
:param old_row:
:return:
"""
if self.cancelled:
return None
try:
keys = sorted(list(new_row.keys()))
old_keys = sorted(list(old_row.keys()))
if keys != old_keys:
raise RuntimeError(
"Cannot update catalog - "
"new row column list and old row column list do not match: New: %s != Old: %s"
% (keys, old_keys))
combined_row = {
'o%d' % i: old_row[keys[i]]
for i in range(len(keys))
}
combined_row.update(
{'n%d' % i: new_row[keys[i]]
for i in range(len(keys))})
update_uri = '/attributegroup/%s/%s;%s' % (catalog_table, ','.join(
["o%d:=%s" % (i, urlquote(keys[i]))
for i in range(len(keys))]), ','.join([
"n%d:=%s" % (i, urlquote(keys[i]))
for i in range(len(keys))
]))
logging.debug(
"Attempting catalog record update [%s] with data: %s" %
(update_uri, json.dumps(combined_row)))
return self.catalog.put(update_uri, json=[combined_row]).json()
except:
(etype, value, traceback) = sys.exc_info()
raise CatalogUpdateError(format_exception(value))
def cleanup_restored_catalog(self):
# cleanup restore state markers
logging.info("Cleaning up restore state...")
dst_model = self.dst_catalog.getCatalogModel()
for sname, schema in dst_model.schemas.items():
for tname, table in schema.tables.items():
annotation_uri = "/schema/%s/table/%s/annotation/%s" % (
urlquote(sname), urlquote(tname),
urlquote(self.RESTORE_STATE_URL))
try:
self.dst_catalog.delete(annotation_uri)
except Exception as e:
logging.warning(
"Unable to cleanup restore state marker annotation %s: %s"
% (annotation_uri, format_exception(e)))
continue
# truncate restore history
if self.truncate_after:
logging.info("Truncating restore history...")
snaptime = self.dst_catalog.get("/").json()["snaptime"]
self.dst_catalog.delete("/history/,%s" % urlquote(snaptime))
def find_processor(processor_name,
processor_type=None,
defaults=DEFAULT_PROCESSORS.copy(),
**kwargs):
if not processor_type:
if processor_name in defaults:
return defaults[processor_name]
else:
raise DerivaUploadConfigurationError(
"Unsupported processor type: %s" % processor_name)
if not _is_processor_whitelisted(processor_type, **kwargs):
raise DerivaUploadConfigurationError(
"Unknown external processor type [%s]: this processor must be added to the whitelist."
% processor_type)
clazz = None
try:
module_name, class_name = processor_type.rsplit(".", 1)
try:
module = sys.modules[module_name]
except KeyError:
module = import_module(module_name)
clazz = getattr(module, class_name) if module else None
except (ImportError, AttributeError):
pass
if not clazz:
raise DerivaUploadConfigurationError(
"Unable to import specified processor class: %s" % processor_type)
if not issubclass(clazz, BaseProcessor):
raise DerivaUploadConfigurationError(
format_exception(
NotImplementedError(
"The imported class %s is not a subclass of %s" %
(processor_type, BaseProcessor.__module__ + "." +
BaseProcessor.__name__))))
return clazz
