async def put(self, schemaspace, resource): schemaspace = url_unescape(schemaspace) resource = url_unescape(resource) parent = self.settings.get("elyra") try: payload = self.get_json_body() # Get the current resource to ensure its pre-existence metadata_manager = MetadataManager(schemaspace=schemaspace, parent=parent) metadata_manager.get(resource) # Check if name is in the payload and varies from resource, if so, raise 400 if "name" in payload and payload["name"] != resource: raise NotImplementedError( f"The attempt to rename instance '{resource}' to '{payload['name']}' is not supported." ) instance = Metadata.from_dict(schemaspace, {**payload}) self.log.debug( f"MetadataHandler: Updating metadata instance '{resource}' in schemaspace '{schemaspace}'..." ) metadata = metadata_manager.update(resource, instance) except (ValidationError, ValueError, NotImplementedError) as err: raise web.HTTPError(400, str(err)) from err except MetadataNotFoundError as err: raise web.HTTPError(404, str(err)) from err except Exception as err: raise web.HTTPError(500, repr(err)) from err self.set_status(200) self.set_header("Content-Type", "application/json") self.finish(metadata.to_dict(trim=True))
async def get(self, namespace, resource): namespace = url_unescape(namespace) resource = url_unescape(resource) schema_manager = SchemaManager() try: schema = schema_manager.get_schema(namespace, resource) except (ValidationError, ValueError, SchemaNotFoundError) as err: raise web.HTTPError(404, str(err)) from err except Exception as err: raise web.HTTPError(500, repr(err)) from err self.set_header("Content-Type", 'application/json') self.finish(schema)
async def get(self, kernel_name: str) -> None: ksm = self.kernel_spec_cache kernel_name = url_unescape(kernel_name) kernel_user_filter = self.request.query_arguments.get('user') kernel_user = None if kernel_user_filter: kernel_user = kernel_user_filter[0].decode("utf-8") try: spec = await ensure_async(ksm.get_kernel_spec(kernel_name)) except KeyError: raise web.HTTPError(404, u'Kernel spec %s not found' % kernel_name) if is_kernelspec_model(spec): model = spec else: model = kernelspec_model(self, kernel_name, spec.to_dict(), spec.resource_dir) d = apply_user_filter(model, self.settings['eg_authorized_users'], self.settings['eg_unauthorized_users'], kernel_user) if d is None: raise web.HTTPError( 403, u'User %s is not authorized to use kernel spec %s' % (kernel_user, kernel_name)) self.set_header("Content-Type", 'application/json') self.finish(json.dumps(model))
async def get(self, path=''): path = path or '' path = url_unescape(path) self.log.debug(f"Parsing file: {path}") try: root_dir = self.settings['server_root_dir'] absolute_path = get_absolute_path(get_expanded_path(root_dir), path) properties = self.content_parser.parse(absolute_path) # TODO: Validation of model self.finish(json.dumps(properties)) except FileNotFoundError as fnfe: raise web.HTTPError(404, str(fnfe)) from fnfe except IsADirectoryError as iade: raise web.HTTPError(400, str(iade)) from iade except Exception as e: # Parser could not parse the given file, but this does not necessarily indicate an error with the file. # Log the issue and return an empty model so that other user processes are not disrupted. self.log.debug(f"Could not parse '{path}': {str(e)}") empty_properties = {"env_vars": {}, "inputs": [], "outputs": []} self.finish(json.dumps(empty_properties))
def test_url_escaping(unescaped, escaped): # Test escaping. path = url_escape(unescaped) assert path == escaped # Test unescaping. path = url_unescape(escaped) assert path == unescaped
def test_url_unescape(): # decodes a url string to a plain string # these tests decode paths with spaces path = url_unescape('/this%20is%20a%20test/for%20spaces/') nt.assert_equal(path, '/this is a test/for spaces/') path = url_unescape('notebook%20with%20space.ipynb') nt.assert_equal(path, 'notebook with space.ipynb') path = url_unescape('/path%20with%20a/notebook%20and%20space.ipynb') nt.assert_equal(path, '/path with a/notebook and space.ipynb') path = url_unescape( '/%20%21%40%24%23%25%5E%26%2A%20/%20test%20%25%5E%20notebook%20%40%23%24%20name.ipynb') nt.assert_equal(path, '/ !@$#%^&* / test %^ notebook @#$ name.ipynb')
async def get(self, namespace, resource): namespace = url_unescape(namespace) resource = url_unescape(resource) try: metadata_manager = MetadataManager(namespace=namespace) metadata = metadata_manager.get(resource) except (ValidationError, ValueError, NotImplementedError) as err: raise web.HTTPError(400, str(err)) from err except MetadataNotFoundError as err: raise web.HTTPError(404, str(err)) from err except Exception as err: raise web.HTTPError(500, repr(err)) from err self.set_header("Content-Type", 'application/json') self.finish(metadata.to_dict(trim=True))
async def post(self, schemaspace): schemaspace = url_unescape(schemaspace) parent = self.settings.get("elyra") try: instance = self._validate_body(schemaspace) self.log.debug( f"MetadataHandler: Creating metadata instance '{instance.name}' in schemaspace '{schemaspace}'..." ) metadata_manager = MetadataManager(schemaspace=schemaspace, parent=parent) metadata = metadata_manager.create(instance.name, instance) except (ValidationError, ValueError, SyntaxError) as err: raise web.HTTPError(400, str(err)) from err except (MetadataNotFoundError, SchemaNotFoundError) as err: raise web.HTTPError(404, str(err)) from err except MetadataExistsError as err: raise web.HTTPError(409, str(err)) from err except Exception as err: raise web.HTTPError(500, repr(err)) from err self.set_status(201) self.set_header("Content-Type", "application/json") location = url_path_join(self.base_url, "elyra", "metadata", schemaspace, metadata.name) self.set_header("Location", location) self.finish(metadata.to_dict(trim=True))
async def post(self, namespace): namespace = url_unescape(namespace) try: instance = self._validate_body(namespace) self.log.debug( "MetadataHandler: Creating metadata instance '{}' in namespace '{}'..." .format(instance.name, namespace)) metadata_manager = MetadataManager(namespace=namespace) metadata = metadata_manager.create(instance.name, instance) except (ValidationError, ValueError, SyntaxError) as err: raise web.HTTPError(400, str(err)) from err except (MetadataNotFoundError, SchemaNotFoundError) as err: raise web.HTTPError(404, str(err)) from err except MetadataExistsError as err: raise web.HTTPError(409, str(err)) from err except Exception as err: raise web.HTTPError(500, repr(err)) from err self.set_status(201) self.set_header("Content-Type", 'application/json') location = url_path_join(self.base_url, 'elyra', 'metadata', namespace, metadata.name) self.set_header('Location', location) self.finish(metadata.to_dict(trim=True))
async def get(self, schemaspace, resource): schemaspace = url_unescape(schemaspace) resource = url_unescape(resource) parent = self.settings.get("elyra") try: metadata_manager = MetadataManager(schemaspace=schemaspace, parent=parent) metadata = metadata_manager.get(resource) except (ValidationError, ValueError, NotImplementedError) as err: raise web.HTTPError(400, str(err)) from err except MetadataNotFoundError as err: raise web.HTTPError(404, str(err)) from err except Exception as err: raise web.HTTPError(500, repr(err)) from err self.set_header("Content-Type", "application/json") self.finish(metadata.to_dict(trim=True))
async def delete(self, namespace, resource): namespace = url_unescape(namespace) resource = url_unescape(resource) try: self.log.debug( "MetadataHandler: Deleting metadata instance '{}' in namespace '{}'..." .format(resource, namespace)) metadata_manager = MetadataManager(namespace=namespace) metadata_manager.remove(resource) except (ValidationError, ValueError) as err: raise web.HTTPError(400, str(err)) from err except PermissionError as err: raise web.HTTPError(403, str(err)) from err except MetadataNotFoundError as err: raise web.HTTPError(404, str(err)) from err except Exception as err: raise web.HTTPError(500, repr(err)) from err self.set_status(204) self.finish()
async def delete(self, schemaspace, resource): schemaspace = url_unescape(schemaspace) resource = url_unescape(resource) parent = self.settings.get("elyra") try: self.log.debug( f"MetadataHandler: Deleting metadata instance '{resource}' in schemaspace '{schemaspace}'..." ) metadata_manager = MetadataManager(schemaspace=schemaspace, parent=parent) metadata_manager.remove(resource) except (ValidationError, ValueError) as err: raise web.HTTPError(400, str(err)) from err except PermissionError as err: raise web.HTTPError(403, str(err)) from err except MetadataNotFoundError as err: raise web.HTTPError(404, str(err)) from err except Exception as err: raise web.HTTPError(500, repr(err)) from err self.set_status(204) self.finish()
async def get(self, schemaspace): schemaspace = url_unescape(schemaspace) try: schema_manager = SchemaManager.instance() schemas = schema_manager.get_schemaspace_schemas(schemaspace) except (ValidationError, ValueError, SchemaNotFoundError) as err: raise web.HTTPError(404, str(err)) from err except Exception as err: raise web.HTTPError(500, repr(err)) from err schemas_model = {schemaspace: list(schemas.values())} self.set_header("Content-Type", "application/json") self.finish(schemas_model)
async def get(self, namespace): namespace = url_unescape(namespace) schema_manager = SchemaManager() try: schemas = schema_manager.get_namespace_schemas(namespace) except (ValidationError, ValueError, SchemaNotFoundError) as err: raise web.HTTPError(404, str(err)) from err except Exception as err: raise web.HTTPError(500, repr(err)) from err schemas_model = dict() schemas_model[namespace] = list(schemas.values()) self.set_header("Content-Type", 'application/json') self.finish(schemas_model)
async def get(self, namespace): namespace = url_unescape(namespace) try: metadata_manager = MetadataManager(namespace=namespace) metadata = metadata_manager.get_all() except (ValidationError, ValueError) as err: raise web.HTTPError(400, str(err)) from err except MetadataNotFoundError as err: raise web.HTTPError(404, str(err)) from err except Exception as err: raise web.HTTPError(500, repr(err)) from err metadata_model = dict() metadata_model[namespace] = [r.to_dict(trim=True) for r in metadata] self.set_header("Content-Type", 'application/json') self.finish(metadata_model)
def _performConductorJWTLogonAndRetrieval(self, jwt_token: str, env_dict: dict[str, Any]): """ Authenticate to Conductor with a JWT Token and setup the kernel environment variables. :param jwt_token: JWT Token to authenticate with to Conductor :param env_dict: Environment Dictionary of this Kernel launch :return: None """ response = None if not jwt_token: return response # Assemble JWT Auth logon REST call env = self.env if env["KERNEL_IG_UUID"] is None: reasonErr = ("Instance group specified is None. Check environment " "specified instance group is available.") self.log_and_raise(http_status_code=500, reason=reasonErr) # Determine hostname of ascd_endpoint and setup the HA List portcolon = self.ascd_endpoint.rfind(":") slash = self.ascd_endpoint.find("://") host = self.ascd_endpoint[slash + 3:portcolon] HA_LIST = env["KERNEL_CONDUCTOR_HA_ENDPOINTS"].split(",") HA_LIST.insert(0, host) header = "Accept: application/json" authorization = "Authorization: Bearer %s" % jwt_token cookie_jar = pjoin(env["KERNEL_NOTEBOOK_DATA_DIR"], env["KERNEL_NOTEBOOK_COOKIE_JAR"]) sslconf = env["KERNEL_CURL_SECURITY_OPT"].split() url = "{}/auth/logon/jwt?topology={}".format(self.ascd_endpoint, env["KERNEL_TOPOLOGY"]) cmd = [ "curl", "-v", "-b", cookie_jar, "-X", "GET", "-H", header, "-H", authorization, url ] cmd[2:2] = sslconf output, stderr = self._performRestCall(cmd, url, HA_LIST) if "Error" in output: reasonErr = "Failed to perform JWT Auth Logon. " + output.splitlines( )[0] self.log.warning(cmd) self.log_and_raise(http_status_code=500, reason=reasonErr) self.rest_credential = url_unescape(output)[1:-1] # Assemble EGO Token Logon REST call authorization = "Authorization: PlatformToken token=" + output.strip( '"') url = "%s/auth/logon" % self.ascd_endpoint cmd = [ "curl", "-v", "-c", cookie_jar, "-X", "GET", "-H", header, "-H", authorization, url ] cmd[2:2] = sslconf output, stderr = self._performRestCall(cmd, url, HA_LIST) if "Error" in output: reasonErr = "Failed to perform EGO Auth Logon. " + output.splitlines( )[0] self.log.warning(cmd) self.log_and_raise(http_status_code=500, reason=reasonErr) # Get the Python path to use to make sure the right conda environment is used url = "{}/anaconda/instances/{}".format( self.ascd_endpoint, env["KERNEL_ANACONDA_INST_UUID"]) cmd = [ "curl", "-v", "-b", cookie_jar, "-X", "GET", "-H", header, "-H", authorization, url ] cmd[2:2] = sslconf output, stderr = self._performRestCall(cmd, url, HA_LIST) response = json.loads(output) if output else None if response is None or not response["parameters"]["deploy_home"][ "value"]: reasonErr = "Could not retrieve anaconda instance. Verify anaconda instance with id " reasonErr = reasonErr + env["KERNEL_ANACONDA_INST_UUID"] + " exists" self.log.warning(cmd) self.log_and_raise(http_status_code=500, reason=reasonErr) else: env_dict["KERNEL_PYSPARK_PYTHON"] = ( response["parameters"]["deploy_home"]["value"] + "/anaconda/envs/" + env["KERNEL_ANACONDA_ENV"] + "/bin/python") # Get instance group information we need url = "{}/instances?id={}&fields=sparkinstancegroup,outputs".format( self.ascd_endpoint, env["KERNEL_IG_UUID"], ) cmd = [ "curl", "-v", "-b", cookie_jar, "-X", "GET", "-H", header, "-H", authorization, url ] cmd[2:2] = sslconf output, stderr = self._performRestCall(cmd, url, HA_LIST) response = json.loads(output) if output else None if response is None or len(response) == 0 or response[0] is None: reasonErr = ( "Could not retrieve instance group. Verify instance group with id " + env["KERNEL_IG_UUID"] + " exists.") self.log.warning(cmd) self.log_and_raise(http_status_code=500, reason=reasonErr) elif (response is None or response[0] is None or "value" not in response[0]["outputs"]["batch_master_rest_urls"]): reasonErr = ( "Could not retrieve outputs for instance group. Verify instance group with id " + env["KERNEL_IG_UUID"] + " is started") self.log.warning(cmd) self.log_and_raise(http_status_code=500, reason=reasonErr) else: env_dict["KERNEL_SPARK_HOME"] = response[0]["sparkinstancegroup"][ "sparkhomedir"] env_dict["KERNEL_NOTEBOOK_MASTER_REST"] = response[0]["outputs"][ "batch_master_rest_urls"]["value"] self.conductor_endpoint = response[0]["outputs"][ "one_batch_master_web_submission_url"]["value"] return response
def _performConductorJWTLogonAndRetrieval(self, jwt_token, env_dict): """ Authenticate to Conductor with a JWT Token and setup the kernel environment variables. :param jwt_token: JWT Token to authenticate with to Conductor :param env_dict: Environment Dictionary of this Kernel launch :return: None """ response = None if not jwt_token: return response # Assemble JWT Auth logon REST call env = self.env if env['KERNEL_IG_UUID'] is None: reasonErr = 'Instance group specified is None. Check environment ' \ 'specified instance group is available.' self.log_and_raise(http_status_code=500, reason=reasonErr) # Determine hostname of ascd_endpoint and setup the HA List portcolon = self.ascd_endpoint.rfind(':') slash = self.ascd_endpoint.find('://') host = self.ascd_endpoint[slash + 3:portcolon] HA_LIST = env['KERNEL_CONDUCTOR_HA_ENDPOINTS'].split(',') HA_LIST.insert(0, host) header = 'Accept: application/json' authorization = 'Authorization: Bearer %s' % jwt_token cookie_jar = pjoin(env['KERNEL_NOTEBOOK_DATA_DIR'], env['KERNEL_NOTEBOOK_COOKIE_JAR']) sslconf = env['KERNEL_CURL_SECURITY_OPT'].split() url = '%s/auth/logon/jwt?topology=%s' % (self.ascd_endpoint, env['KERNEL_TOPOLOGY']) cmd = [ 'curl', '-v', '-b', cookie_jar, '-X', 'GET', '-H', header, '-H', authorization, url ] cmd[2:2] = sslconf output, stderr = self._performRestCall(cmd, url, HA_LIST) if 'Error' in output: reasonErr = 'Failed to perform JWT Auth Logon. ' + output.splitlines( )[0] self.log.warning(cmd) self.log_and_raise(http_status_code=500, reason=reasonErr) self.rest_credential = url_unescape(output)[1:-1] # Assemble EGO Token Logon REST call authorization = 'Authorization: PlatformToken token=' + output.strip( '"') url = '%s/auth/logon' % self.ascd_endpoint cmd = [ 'curl', '-v', '-c', cookie_jar, '-X', 'GET', '-H', header, '-H', authorization, url ] cmd[2:2] = sslconf output, stderr = self._performRestCall(cmd, url, HA_LIST) if 'Error' in output: reasonErr = 'Failed to perform EGO Auth Logon. ' + output.splitlines( )[0] self.log.warning(cmd) self.log_and_raise(http_status_code=500, reason=reasonErr) # Get the Python path to use to make sure the right conda environment is used url = '%s/anaconda/instances/%s' % (self.ascd_endpoint, env['KERNEL_ANACONDA_INST_UUID']) cmd = [ 'curl', '-v', '-b', cookie_jar, '-X', 'GET', '-H', header, '-H', authorization, url ] cmd[2:2] = sslconf output, stderr = self._performRestCall(cmd, url, HA_LIST) response = json.loads(output) if output else None if response is None or not response['parameters']['deploy_home'][ 'value']: reasonErr = 'Could not retrieve anaconda instance. Verify anaconda instance with id ' reasonErr = reasonErr + env['KERNEL_ANACONDA_INST_UUID'] + ' exists' self.log.warning(cmd) self.log_and_raise(http_status_code=500, reason=reasonErr) else: env_dict['KERNEL_PYSPARK_PYTHON'] = response['parameters']['deploy_home']['value'] \ + '/anaconda/envs/' + env['KERNEL_ANACONDA_ENV'] + '/bin/python' # Get instance group information we need url = '%s/instances?id=%s&fields=sparkinstancegroup,outputs' % ( self.ascd_endpoint, env['KERNEL_IG_UUID']) cmd = [ 'curl', '-v', '-b', cookie_jar, '-X', 'GET', '-H', header, '-H', authorization, url ] cmd[2:2] = sslconf output, stderr = self._performRestCall(cmd, url, HA_LIST) response = json.loads(output) if output else None if response is None or len(response) == 0 or response[0] is None: reasonErr = 'Could not retrieve instance group. Verify instance group with id ' \ + env['KERNEL_IG_UUID'] + ' exists.' self.log.warning(cmd) self.log_and_raise(http_status_code=500, reason=reasonErr) elif response is None or response[0] is None or 'value' not in response[ 0]['outputs']['batch_master_rest_urls']: reasonErr = 'Could not retrieve outputs for instance group. Verify instance group with id ' \ + env['KERNEL_IG_UUID'] + ' is started' self.log.warning(cmd) self.log_and_raise(http_status_code=500, reason=reasonErr) else: env_dict['KERNEL_SPARK_HOME'] = response[0]['sparkinstancegroup'][ 'sparkhomedir'] env_dict['KERNEL_NOTEBOOK_MASTER_REST'] = response[0]['outputs'][ 'batch_master_rest_urls']['value'] self.conductor_endpoint = response[0]['outputs'][ 'one_batch_master_web_submission_url']['value'] return response