def __init__(self, **kwargs): self.client = None self.client_id = kwargs.get("client_id") or NATIVE_APP_CLIENT_ID self.hosts = kwargs.get("hosts") self.config_file = kwargs.get("config_file") self.default_scopes = DEFAULT_SCOPES.copy() if self.client_id == NATIVE_APP_CLIENT_ID: self.default_scopes.append(GROUPS_SCOPE_NAME) try: global GLOBUS_SDK, NATIVE_LOGIN GLOBUS_SDK = importlib.import_module("globus_sdk") NATIVE_LOGIN = importlib.import_module("fair_research_login") except Exception as e: raise DependencyError("Unable to load a required module: %s" % format_exception(e)) try: storage_file = 'globus-credential%s.json' % \ (("-" + self.client_id) if self.client_id != NATIVE_APP_CLIENT_ID else "") storage = DerivaJSONTokenStorage( filename=os.path.join(DEFAULT_CONFIG_PATH, storage_file)) self.client = NATIVE_LOGIN.NativeClient( client_id=self.client_id, token_storage=storage, app_name="Login from deriva-client on %s [%s]%s" % (platform.uname()[1], platform.platform(aliased=True), " to hosts [%s]" % ", ".join(self.hosts) if self.hosts else ""), default_scopes=self.default_scopes) except Exception as e: logging.error("Unable to instantiate a required class: %s" % format_exception(e))
def main(self): args = self.parse_cli() def _cmd_error_message(emsg): return "{prog} {subcmd}: {msg}".format(prog=self.parser.prog, subcmd=args.subcmd, msg=emsg) try: if not hasattr(args, 'func'): self.parser.print_usage() return 2 if args.subcmd == "login" or args.subcmd == "logout" or args.subcmd == "user-info": self.gnl = GlobusNativeLogin(**vars(args)) else: self.gau = GlobusAuthUtil(**vars(args)) response = args.func(args) if args.pretty: if isinstance(response, dict) or isinstance(response, list): try: print(json.dumps(response, indent=2)) return except: pprint(response) return elif not isinstance(response, str): pprint(response) return print(response) return except UsageException as e: eprint("{prog} {subcmd}: {msg}".format(prog=self.parser.prog, subcmd=args.subcmd, msg=e)) except ConnectionError as e: eprint("{prog}: Connection error occurred".format( prog=self.parser.prog)) except GlobusAuthAPIError as e: if 401 == e.http_status: msg = 'Authentication required: %s' % e.message elif 403 == e.http_status: msg = 'Permission denied: %s' % e.message else: msg = e logging.debug(format_exception(e)) eprint(_cmd_error_message(msg)) except DependencyError as e: eprint(_cmd_error_message(e)) except RuntimeError as e: logging.debug(format_exception(e)) eprint('An unexpected runtime error occurred') except: eprint('An unexpected error occurred') traceback.print_exc() return 1
def _getFileRecord(self, asset_mapping): """ Helper function that queries the catalog to get a record linked to the asset, or create it if it doesn't exist. :return: the file record """ column_map = asset_mapping.get("column_map", {}) rqt = asset_mapping['record_query_template'] try: path = rqt.format(**self.metadata) except KeyError as e: raise ConfigurationError( "Record query template substitution error: %s" % format_exception(e)) result = self.catalog.get(path).json() if result: self._updateFileMetadata(result[0]) return self.pruneDict(result[0], column_map) else: row = self.interpolateDict(self.metadata, column_map) result = self._catalogRecordCreate(self.metadata['target_table'], row) if result: self._updateFileMetadata(result[0]) return self.interpolateDict(self.metadata, column_map, allowNone=True)
def main(self): args = self.parse_cli() if args.path is None: sys.stderr.write("\nError: Input directory not specified.\n") self.parser.print_usage() return 2 if not args.quiet: sys.stderr.write("\n") try: DerivaUploadCLI.upload(self.uploader, os.path.abspath(args.path), args.host, args.catalog, args.token, args.config_file, args.credential_file, args.no_config_update, args.purge_state) except (RuntimeError, DerivaUploadError, DerivaUploadConfigurationError, DerivaUploadCatalogCreateError, DerivaUploadCatalogUpdateError) as e: sys.stderr.write(("\n" if not args.quiet else "") + format_exception(e)) if args.debug: traceback.print_exc() return 1 except: traceback.print_exc() return 1 finally: if not args.quiet: sys.stderr.write("\n\n") return 0
def __init__(self, **kwargs): client_id = kwargs.get("client_id") client_secret = kwargs.get("client_secret") if not (client_id and client_secret): cred_file = kwargs.get("credential_file", CLIENT_CRED_FILE) creds = read_config(cred_file) if creds: client = creds.get("web") if client: client_id = client.get('client_id') client_secret = client.get('client_secret') try: global GLOBUS_SDK, GlobusAuthAPIError GLOBUS_SDK = importlib.import_module("globus_sdk") GlobusAuthAPIError = GLOBUS_SDK.AuthAPIError except Exception as e: raise DependencyError("Unable to load a required module: %s" % format_exception(e)) if not (client_id and client_secret): logging.warning( "Client ID and secret not specified and/or could not be determined." ) self.client = GLOBUS_SDK.ConfidentialAppAuthClient( client_id, client_secret) self.client_id = client_id
def main(self): args = self.parse_cli() self.dumpdir = args.dir self.host = args.host self.catalog_id = args.catalog self.graph_format = args.graph_format if self.host is None: eprint('Host name must be provided') return 1 self.catalog = ErmrestCatalog('https', self.host, self.catalog_id, credentials=self._get_credential(self.host)) self.model = self.catalog.getCatalogModel() self.schemas = [s for s in (args.schemas if args.schemas else self.model.schemas) if s not in args.skip_schemas ] try: os.makedirs(self.dumpdir, exist_ok=True) except OSError as e: sys.stderr.write(str(e)) return 1 logger.info('Catalog has {} schema and {} tables'.format(len(self.model.schemas), sum([len(v.tables) for k, v in self.model.schemas.items()]))) logger.info('\n'.join([' {} has {} tables'.format(k, len(s.tables)) for k, s in self.model.schemas.items()])) try: if args.table: if ':' not in args.table: raise DerivaDumpCatalogException('Table name must be in form of schema:table') [schema_name, table_name] = args.table.split(":") self._dump_table(schema_name, table_name) elif args.graph: self._graph_catalog() else: self._dump_catalog() except DerivaDumpCatalogException as e: print(e) except HTTPError as e: if e.response.status_code == requests.codes.unauthorized: msg = 'Authentication required for {}'.format(args.server) elif e.response.status_code == requests.codes.forbidden: msg = 'Permission denied' else: msg = e logging.debug(format_exception(e)) eprint(msg) except RuntimeError as e: sys.stderr.write(str(e)) return 1 except: traceback.print_exc() return 1 finally: sys.stderr.write("\n\n") return
def main(): cli = AclCLI() args = cli.parse_cli() table_name = cli.get_table_arg(args) schema_names = cli.get_schema_arg_list(args) credentials = get_credential(args.host, args.credential_file) save_groups = not (args.dryrun or args.omit_groups) for schema in schema_names: acl_config = AclConfig(args.host, args.catalog, args.config_file, credentials, schema_name=schema, table_name=table_name, verbose=args.verbose or args.debug) try: if save_groups: acl_config.save_groups() save_groups = False if not args.groups_only: acl_config.set_acls() if not args.dryrun: acl_config.apply_acls() except HTTPError as e: print(format_exception(e)) raise if args.dryrun: print(acl_config.dumps())
def _catalogRecordCreate(self, catalog_table, row, default_columns=None): """ :param catalog_table: :param row: :param default_columns: :return: """ if self.cancelled: return None try: missing = self.catalog.validateRowColumns(row, catalog_table) if missing: raise CatalogCreateError( "Unable to update catalog entry because one or more specified columns do not exist in the " "target table: [%s]" % ','.join(missing)) if not default_columns: default_columns = self.catalog.getDefaultColumns( row, catalog_table) default_param = ( '?defaults=%s' % ','.join(default_columns)) if len(default_columns) > 0 else '' # for default in default_columns: # row[default] = None create_uri = '/entity/%s%s' % (catalog_table, default_param) logging.debug( "Attempting catalog record create [%s] with data: %s" % (create_uri, json.dumps(row))) return self.catalog.post(create_uri, json=[row]).json() except: (etype, value, traceback) = sys.exc_info() raise CatalogCreateError(format_exception(value))
def createManifestEntry(self, entry): manifest_entry = dict() url = entry.get("url") if not url: logging.warning( "Skipping a record due to missing required attribute \"url\" in fetch manifest entry %s" % json.dumps(entry)) return ext_url = self.getExternalUrl(url) length = entry.get("length") md5 = entry.get("md5") sha256 = entry.get("sha256") filename = entry.get("filename") content_type = entry.get("content_type") content_disposition = None # if any required fields are missing from the query result, attempt to get them from the remote server by # issuing a HEAD request against the supplied URL if not (length and (md5 or sha256)): try: headers = self.headForHeaders(url, raise_for_status=True) except requests.HTTPError as e: raise DerivaDownloadError("Exception during HEAD request: %s" % format_exception(e)) length = headers.get("Content-Length") content_type = headers.get("Content-Type") content_disposition = headers.get("Content-Disposition") if not md5: md5 = headers.get("Content-MD5") if md5: md5 = decodeBase64toHex(md5) if not sha256: sha256 = headers.get("Content-SHA256") if sha256: sha256 = decodeBase64toHex(sha256) # if content length or both hash values are missing, it is a fatal error if not length: raise DerivaDownloadError("Could not determine Content-Length for %s" % ext_url) if not (md5 or sha256): raise DerivaDownloadError("Could not locate an MD5 or SHA256 hash for %s" % ext_url) envvars = self.envars.copy() envvars.update(entry) subdir = self.sub_path.format(**envvars) # if a local filename is not provided, try to construct one using content_disposition, if available if not filename: filename = os.path.basename(url).split(":")[0] if not content_disposition else \ parse_content_disposition(content_disposition) output_path = ''.join([subdir, "/", filename]) if subdir else filename manifest_entry['url'] = ext_url manifest_entry['length'] = int(length) manifest_entry['filename'] = output_path if md5: manifest_entry['md5'] = md5 if sha256: manifest_entry['sha256'] = sha256 if content_type: manifest_entry["content_type"] = content_type return manifest_entry
def main(self): """Main routine of the CLI. """ args = self.parse_cli() def _resource_error_message(emsg): return "{prog} {subcmd}: {resource}: {msg}".format( prog=self.parser.prog, subcmd=args.subcmd, resource=args.resource, msg=emsg) try: if not hasattr(args, 'func'): self.parser.print_usage() return 1 self._post_parser_init(args) args.func(args) return 0 except UsageException as e: eprint("{prog} {subcmd}: {msg}".format(prog=self.parser.prog, subcmd=args.subcmd, msg=e)) except ConnectionError as e: eprint("{prog}: Connection error occurred".format(prog=self.parser.prog)) except DerivaPathError as e: eprint(e) except HTTPError as e: if e.response.status_code == requests.codes.unauthorized: msg = 'Authentication required' elif e.response.status_code == requests.codes.forbidden: msg = 'Permission denied' else: msg = e logging.debug(format_exception(e)) eprint(_resource_error_message(msg)) except ResourceException as e: logging.debug(format_exception(e.cause)) eprint(_resource_error_message(e)) except HatracHashMismatch as e: logging.debug(format_exception(e)) eprint(_resource_error_message('Checksum verification failed')) except RuntimeError as e: logging.debug(format_exception(e)) eprint('Unexpected runtime error occurred') except: eprint('Unexpected error occurred') traceback.print_exc() return 1
def _queryFileMetadata(self, file_path, asset_mapping, match_groupdict): """ Helper function that queries the catalog to get required metadata for a given file/asset """ file_name = self.getFileDisplayName(file_path) logging.info("Computing metadata for file: [%s]." % file_name) self._initFileMetadata(file_path, asset_mapping, match_groupdict) logging.info("Computing checksums for file: [%s]. Please wait..." % file_name) hashes = self.getFileHashes( file_path, asset_mapping.get('checksum_types', ['md5', 'sha256'])) for alg, checksum in hashes.items(): alg = alg.lower() self.metadata[alg] = checksum[0] self.metadata[alg + "_base64"] = checksum[1] for uri in asset_mapping.get("metadata_query_templates", []): try: path = uri.format(**self.metadata) except KeyError as e: raise RuntimeError( "Metadata query template substitution error: %s" % format_exception(e)) result = self.catalog.get(path).json() if result: self._updateFileMetadata(result[0], True) self._urlEncodeMetadata( asset_mapping.get("url_encoding_safe_overrides")) else: raise RuntimeError( "Metadata query did not return any results: %s" % path) self._getFileExtensionMetadata(self.metadata.get("file_ext")) for k, v in asset_mapping.get("column_value_templates", {}).items(): try: self.metadata[k] = v.format(**self.metadata) except KeyError as e: logging.warning( "Column value template substitution error: %s" % format_exception(e)) continue self._urlEncodeMetadata( asset_mapping.get("url_encoding_safe_overrides"))
def GET(self, catalog_id): warnings.warn( "The '{}' service is experimental and not intended for production usage." .format(__name__)) logger.debug("query: {}".format(web.ctx.query)) params = [urlunquote(param) for param in web.ctx.query[1:].split('&')] logger.debug("params: {}".format(params)) try: auth_token = web.cookies().get("webauthn") credentials = format_credential( token=auth_token) if auth_token else None return pattern_transformer(catalog_id, params, credentials) except requests.HTTPError as e: raise RestException.from_http_error(e) except ValueError as e: raise BadRequest(format_exception(e)) except KeyError as e: raise BadRequest(format_exception(e))
def cleanupTransferState(self): if self.transfer_state_fp and not self.transfer_state_fp.closed: try: self.transfer_state_fp.flush() self.transfer_state_fp.close() except Exception as e: logging.warning( "Unable to flush/close transfer state file: %s" % format_exception(e))
def excepthook(etype, value, tb): traceback.print_tb(tb) print(format_exception(value), file=sys.stderr) msg = QMessageBox() msg.setText(str(value)) msg.setStandardButtons(QMessageBox.Close) msg.setWindowTitle("Unhandled Exception: %s" % etype.__name__) msg.setIcon(QMessageBox.Critical) msg.setDetailedText('\n'.join(traceback.format_exception(etype, value, tb))) msg.exec_()
def main(self): try: args = self.parse_cli() except ValueError as e: sys.stderr.write(str(e)) return 2 if not args.quiet: sys.stderr.write("\n") try: try: downloaded = self.download(args) sys.stdout.write("\n%s\n" % (json.dumps(downloaded))) except ConnectionError as e: raise DerivaDownloadError("Connection error occurred. %s" % format_exception(e)) except HTTPError as e: if e.response.status_code == requests.codes.unauthorized: raise DerivaDownloadAuthenticationError( "The requested service requires authentication and a valid login session could " "not be found for the specified host. Server responded: %s" % e) elif e.response.status_code == requests.codes.forbidden: raise DerivaDownloadAuthorizationError( "A requested operation was forbidden. Server responded: %s" % e) except (DerivaDownloadError, DerivaDownloadConfigurationError, DerivaDownloadAuthenticationError, DerivaDownloadAuthorizationError) as e: sys.stderr.write(("\n" if not args.quiet else "") + format_exception(e)) if args.debug: traceback.print_exc() return 1 except: sys.stderr.write("An unexpected error occurred.") traceback.print_exc() return 1 finally: if not args.quiet: sys.stderr.write("\n\n") return 0
def writeTransferState(self): if not self.transfer_state_fp: return try: self.transfer_state_fp.seek(0, 0) self.transfer_state_fp.truncate() json.dump(self.transfer_state, self.transfer_state_fp, indent=2) self.transfer_state_fp.flush() except Exception as e: logging.warning("Unable to write transfer state file: %s" % format_exception(e))
def getCacheDir(self): cwd = os.getcwd() cache_dir = os.path.expanduser(self.config.get("cache_dir", cwd)) if not os.path.isdir(cache_dir): try: os.makedirs(cache_dir) except OSError as error: if error.errno != errno.EEXIST: logging.error(format_exception(error)) cache_dir = cwd return cache_dir
def _onPreAuthContent(self, content): try: if not content: logging.debug("no preauth content") return preauth = json.loads(content) logging.trace("webauthn preauth:\n%s\n", json.dumps(preauth, indent=2)) qApp.setOverrideCursor(Qt.WaitCursor) self.authn_session_page.setUrl(QUrl(preauth["redirect_url"])) except (ValueError, Exception) as e: logging.error(format_exception(e)) self.set_current_html(ERROR_HTML % content)
def onRetrieveInputFileResult(self, success, status, detail, file_path): if not success: try: os.remove(file_path) except Exception as e: logging.warning("Unable to remove file [%s]: %s" % (file_path, format_exception(e))) self.resetUI(status, detail) self.serverProblemMessageBox( "Unable to download required input file", "The image input file was not downloaded successfully.") return self.executeViewer(file_path)
def catalogQuery(self, headers=HEADERS, as_file=True): if as_file: output_dir = os.path.dirname(self.output_abspath) self.makeDirs(output_dir) try: if as_file: return self.catalog.getAsFile(self.query, self.output_abspath, headers=headers) else: return self.catalog.get(self.query, headers=headers).json() except requests.HTTPError as e: raise RuntimeError("Unable to execute catalog query: %s" % format_exception(e))
def logout(self, delete_cookies=False): if not (self.auth_url and (self.auth_url.host() and self.auth_url.scheme())): return if not self.authenticated(): return try: logging.info("Logging out of host: %s" % self.auth_url.toString()) if delete_cookies and self.cookie_persistence: self.authn_session_page.profile().cookieStore( ).deleteAllCookies() self._session.delete(self.auth_url.toString() + "/authn/session") except Exception as e: logging.warning("Logout error: %s" % format_exception(e)) self._cleanup()
def _getFileHatracMetadata(self, asset_mapping): try: hatrac_templates = asset_mapping["hatrac_templates"] # URI is required self.metadata["URI"] = hatrac_templates["hatrac_uri"].format( **self.metadata) # overridden content-disposition is optional content_disposition = hatrac_templates.get("content-disposition") self.metadata["content-disposition"] = \ None if not content_disposition else content_disposition.format(**self.metadata) self._urlEncodeMetadata( asset_mapping.get("url_encoding_safe_overrides")) except KeyError as e: raise ConfigurationError("Hatrac template substitution error: %s" % format_exception(e))
def main(self): args = self.parse_cli() catalog = DerivaCatalogConfigure(args.host, catalog_id=args.catalog_model) try: catalog = DerivaCatalog(args.host, args.catalog_model) [schema_name, table_name] = args.table_model.split(':') table = DerivaTable(catalog, schema_name, table_name) if args.asset_table: table.create_asset_table(args.asset_table) if args._visible_columns: table.create_default_visible_columns(really=args.replace) if args.configure == 'catalog': logging.info('Configuring catalog {}:{}'.format(args.host, args.catalog_model)) cfg = DerivaCatalogConfigure(args.host, catalog_id=args.catalog_model) cfg.configure_baseline_catalog(catalog_name=args.catalog_name, reader=args.reader, writer=args.writer, curator=args.curator, admin=args.admin, set_policy=args.set_policy, public=args.publish) cfg.apply() if args.table_model: [schema_name, table_name] = args.table_model.split(':') table = catalog.schema(schema_name).table_model(table_name) table.configure_table_defaults(set_policy=args.set_policy, public=args.publish) table.apply() except DerivaConfigError as e: print(e.msg) except HTTPError as e: if e.response.status_code == requests.codes.unauthorized: msg = 'Authentication required for {}'.format(args.host) elif e.response.status_code == requests.codes.forbidden: msg = 'Permission denied' else: msg = e logging.debug(format_exception(e)) eprint(msg) except RuntimeError as e: sys.stderr.write(str(e)) return 1 except: traceback.print_exc() return 1 finally: sys.stderr.write("\n\n") return
def logout(self, delete_cookies=False): if not (self.auth_url and (self.auth_url.host() and self.auth_url.scheme())): return if self.authenticated(): try: logging.info("Logging out of host: %s" % self.auth_url.toString()) if delete_cookies and self.cookie_persistence: self.authn_session_page.profile().cookieStore().deleteAllCookies() self._session.delete(self.auth_url.toString() + "/authn/session") if self.credential_file: creds = read_credential(self.credential_file, create_default=True) host = self.auth_url.host() if creds.get(host): del creds[host] write_credential(self.credential_file, creds) except Exception as e: logging.warning("Logout error: %s" % format_exception(e)) self._cleanup()
def _onSessionContent(self, content): try: self.setHtml(SUCCESS_HTML) self.authn_session = json.loads(content) seconds_remaining = self.authn_session['seconds_remaining'] if not self._timer.isActive(): interval = seconds_remaining // 2 logging.info( "Authentication successful for [%s]: credential refresh in %d seconds." % (self.auth_url.toString(), interval)) self._timer.start(interval * 1000) self.authn_expires = time.time() + seconds_remaining + 1 logging.trace("webauthn session:\n%s\n", json.dumps(self.authn_session, indent=2)) qApp.restoreOverrideCursor() QTimer.singleShot(100, self._execSuccessCallback) except (ValueError, Exception) as e: logging.error(format_exception(e)) self.setHtml(ERROR_HTML % content)
def purge_output_dirs(threshold=0, count=1): if threshold < 1: return basedir = get_staging_path() if not os.path.isdir(basedir): return paths = [ os.fspath(path) for path in sorted( Path(basedir).iterdir(), key=os.path.getctime, reverse=True) ] if not paths or (len(paths) < threshold): return for i in range(count): try: path = paths.pop() if os.path.isdir(path): shutil.rmtree(path) except Exception as e: logging.warning(format_exception(e))
def _onSessionContent(self, content): try: qApp.restoreOverrideCursor() self.set_current_html(SUCCESS_HTML) try: self.authn_session = json.loads(content) except json.JSONDecodeError: raise RuntimeError("Unable to parse response from server: %s" % content) seconds_remaining = self.authn_session['seconds_remaining'] if not self._timer.isActive(): interval = seconds_remaining // 2 logging.info("Authentication successful for [%s]: credential refresh in %d seconds." % (self.auth_url.toString(), interval)) self._timer.start(interval * 1000) self.authn_expires = time.time() + seconds_remaining + 1 logging.trace("webauthn session:\n%s\n", json.dumps(self.authn_session, indent=2)) QTimer.singleShot(100, self._execSuccessCallback) except (ValueError, Exception) as e: error = format_exception(e) logging.error(error) self.set_current_html(ERROR_HTML % content) self._execFailureCallback(error)
def _catalogRecordUpdate(self, catalog_table, old_row, new_row): """ :param catalog_table: :param new_row: :param old_row: :return: """ if self.cancelled: return None try: keys = sorted(list(new_row.keys())) old_keys = sorted(list(old_row.keys())) if keys != old_keys: raise RuntimeError( "Cannot update catalog - " "new row column list and old row column list do not match: New: %s != Old: %s" % (keys, old_keys)) combined_row = { 'o%d' % i: old_row[keys[i]] for i in range(len(keys)) } combined_row.update( {'n%d' % i: new_row[keys[i]] for i in range(len(keys))}) update_uri = '/attributegroup/%s/%s;%s' % (catalog_table, ','.join( ["o%d:=%s" % (i, urlquote(keys[i])) for i in range(len(keys))]), ','.join([ "n%d:=%s" % (i, urlquote(keys[i])) for i in range(len(keys)) ])) logging.debug( "Attempting catalog record update [%s] with data: %s" % (update_uri, json.dumps(combined_row))) return self.catalog.put(update_uri, json=[combined_row]).json() except: (etype, value, traceback) = sys.exc_info() raise CatalogUpdateError(format_exception(value))
def cleanup_restored_catalog(self): # cleanup restore state markers logging.info("Cleaning up restore state...") dst_model = self.dst_catalog.getCatalogModel() for sname, schema in dst_model.schemas.items(): for tname, table in schema.tables.items(): annotation_uri = "/schema/%s/table/%s/annotation/%s" % ( urlquote(sname), urlquote(tname), urlquote(self.RESTORE_STATE_URL)) try: self.dst_catalog.delete(annotation_uri) except Exception as e: logging.warning( "Unable to cleanup restore state marker annotation %s: %s" % (annotation_uri, format_exception(e))) continue # truncate restore history if self.truncate_after: logging.info("Truncating restore history...") snaptime = self.dst_catalog.get("/").json()["snaptime"] self.dst_catalog.delete("/history/,%s" % urlquote(snaptime))
def find_processor(processor_name, processor_type=None, defaults=DEFAULT_PROCESSORS.copy(), **kwargs): if not processor_type: if processor_name in defaults: return defaults[processor_name] else: raise DerivaUploadConfigurationError( "Unsupported processor type: %s" % processor_name) if not _is_processor_whitelisted(processor_type, **kwargs): raise DerivaUploadConfigurationError( "Unknown external processor type [%s]: this processor must be added to the whitelist." % processor_type) clazz = None try: module_name, class_name = processor_type.rsplit(".", 1) try: module = sys.modules[module_name] except KeyError: module = import_module(module_name) clazz = getattr(module, class_name) if module else None except (ImportError, AttributeError): pass if not clazz: raise DerivaUploadConfigurationError( "Unable to import specified processor class: %s" % processor_type) if not issubclass(clazz, BaseProcessor): raise DerivaUploadConfigurationError( format_exception( NotImplementedError( "The imported class %s is not a subclass of %s" % (processor_type, BaseProcessor.__module__ + "." + BaseProcessor.__name__)))) return clazz