async def put(self, schemaspace, resource):
schemaspace = url_unescape(schemaspace)
resource = url_unescape(resource)
parent = self.settings.get("elyra")
try:
payload = self.get_json_body()
# Get the current resource to ensure its pre-existence
metadata_manager = MetadataManager(schemaspace=schemaspace, parent=parent)
metadata_manager.get(resource)
# Check if name is in the payload and varies from resource, if so, raise 400
if "name" in payload and payload["name"] != resource:
raise NotImplementedError(
f"The attempt to rename instance '{resource}' to '{payload['name']}' is not supported."
)
instance = Metadata.from_dict(schemaspace, {**payload})
self.log.debug(
f"MetadataHandler: Updating metadata instance '{resource}' in schemaspace '{schemaspace}'..."
)
metadata = metadata_manager.update(resource, instance)
except (ValidationError, ValueError, NotImplementedError) as err:
raise web.HTTPError(400, str(err)) from err
except MetadataNotFoundError as err:
raise web.HTTPError(404, str(err)) from err
except Exception as err:
raise web.HTTPError(500, repr(err)) from err
self.set_status(200)
self.set_header("Content-Type", "application/json")
self.finish(metadata.to_dict(trim=True))
async def get(self, namespace, resource):
namespace = url_unescape(namespace)
resource = url_unescape(resource)
schema_manager = SchemaManager()
try:
schema = schema_manager.get_schema(namespace, resource)
except (ValidationError, ValueError, SchemaNotFoundError) as err:
raise web.HTTPError(404, str(err)) from err
except Exception as err:
raise web.HTTPError(500, repr(err)) from err
self.set_header("Content-Type", 'application/json')
self.finish(schema)
async def get(self, kernel_name: str) -> None:
ksm = self.kernel_spec_cache
kernel_name = url_unescape(kernel_name)
kernel_user_filter = self.request.query_arguments.get('user')
kernel_user = None
if kernel_user_filter:
kernel_user = kernel_user_filter[0].decode("utf-8")
try:
spec = await ensure_async(ksm.get_kernel_spec(kernel_name))
except KeyError:
raise web.HTTPError(404, u'Kernel spec %s not found' % kernel_name)
if is_kernelspec_model(spec):
model = spec
else:
model = kernelspec_model(self, kernel_name, spec.to_dict(),
spec.resource_dir)
d = apply_user_filter(model, self.settings['eg_authorized_users'],
self.settings['eg_unauthorized_users'],
kernel_user)
if d is None:
raise web.HTTPError(
403, u'User %s is not authorized to use kernel spec %s' %
(kernel_user, kernel_name))
self.set_header("Content-Type", 'application/json')
self.finish(json.dumps(model))
async def get(self, path=''):
path = path or ''
path = url_unescape(path)
self.log.debug(f"Parsing file: {path}")
try:
root_dir = self.settings['server_root_dir']
absolute_path = get_absolute_path(get_expanded_path(root_dir),
path)
properties = self.content_parser.parse(absolute_path)
# TODO: Validation of model
self.finish(json.dumps(properties))
except FileNotFoundError as fnfe:
raise web.HTTPError(404, str(fnfe)) from fnfe
except IsADirectoryError as iade:
raise web.HTTPError(400, str(iade)) from iade
except Exception as e:
# Parser could not parse the given file, but this does not necessarily indicate an error with the file.
# Log the issue and return an empty model so that other user processes are not disrupted.
self.log.debug(f"Could not parse '{path}': {str(e)}")
empty_properties = {"env_vars": {}, "inputs": [], "outputs": []}
self.finish(json.dumps(empty_properties))
def test_url_escaping(unescaped, escaped):
# Test escaping.
path = url_escape(unescaped)
assert path == escaped
# Test unescaping.
path = url_unescape(escaped)
assert path == unescaped
def test_url_unescape():
# decodes a url string to a plain string
# these tests decode paths with spaces
path = url_unescape('/this%20is%20a%20test/for%20spaces/')
nt.assert_equal(path, '/this is a test/for spaces/')
path = url_unescape('notebook%20with%20space.ipynb')
nt.assert_equal(path, 'notebook with space.ipynb')
path = url_unescape('/path%20with%20a/notebook%20and%20space.ipynb')
nt.assert_equal(path, '/path with a/notebook and space.ipynb')
path = url_unescape(
'/%20%21%40%24%23%25%5E%26%2A%20/%20test%20%25%5E%20notebook%20%40%23%24%20name.ipynb')
nt.assert_equal(path, '/ !@$#%^&* / test %^ notebook @#$ name.ipynb')
async def get(self, namespace, resource):
namespace = url_unescape(namespace)
resource = url_unescape(resource)
try:
metadata_manager = MetadataManager(namespace=namespace)
metadata = metadata_manager.get(resource)
except (ValidationError, ValueError, NotImplementedError) as err:
raise web.HTTPError(400, str(err)) from err
except MetadataNotFoundError as err:
raise web.HTTPError(404, str(err)) from err
except Exception as err:
raise web.HTTPError(500, repr(err)) from err
self.set_header("Content-Type", 'application/json')
self.finish(metadata.to_dict(trim=True))
async def post(self, schemaspace):
schemaspace = url_unescape(schemaspace)
parent = self.settings.get("elyra")
try:
instance = self._validate_body(schemaspace)
self.log.debug(
f"MetadataHandler: Creating metadata instance '{instance.name}' in schemaspace '{schemaspace}'..."
)
metadata_manager = MetadataManager(schemaspace=schemaspace, parent=parent)
metadata = metadata_manager.create(instance.name, instance)
except (ValidationError, ValueError, SyntaxError) as err:
raise web.HTTPError(400, str(err)) from err
except (MetadataNotFoundError, SchemaNotFoundError) as err:
raise web.HTTPError(404, str(err)) from err
except MetadataExistsError as err:
raise web.HTTPError(409, str(err)) from err
except Exception as err:
raise web.HTTPError(500, repr(err)) from err
self.set_status(201)
self.set_header("Content-Type", "application/json")
location = url_path_join(self.base_url, "elyra", "metadata", schemaspace, metadata.name)
self.set_header("Location", location)
self.finish(metadata.to_dict(trim=True))
async def post(self, namespace):
namespace = url_unescape(namespace)
try:
instance = self._validate_body(namespace)
self.log.debug(
"MetadataHandler: Creating metadata instance '{}' in namespace '{}'..."
.format(instance.name, namespace))
metadata_manager = MetadataManager(namespace=namespace)
metadata = metadata_manager.create(instance.name, instance)
except (ValidationError, ValueError, SyntaxError) as err:
raise web.HTTPError(400, str(err)) from err
except (MetadataNotFoundError, SchemaNotFoundError) as err:
raise web.HTTPError(404, str(err)) from err
except MetadataExistsError as err:
raise web.HTTPError(409, str(err)) from err
except Exception as err:
raise web.HTTPError(500, repr(err)) from err
self.set_status(201)
self.set_header("Content-Type", 'application/json')
location = url_path_join(self.base_url, 'elyra', 'metadata', namespace,
metadata.name)
self.set_header('Location', location)
self.finish(metadata.to_dict(trim=True))
async def get(self, schemaspace, resource):
schemaspace = url_unescape(schemaspace)
resource = url_unescape(resource)
parent = self.settings.get("elyra")
try:
metadata_manager = MetadataManager(schemaspace=schemaspace, parent=parent)
metadata = metadata_manager.get(resource)
except (ValidationError, ValueError, NotImplementedError) as err:
raise web.HTTPError(400, str(err)) from err
except MetadataNotFoundError as err:
raise web.HTTPError(404, str(err)) from err
except Exception as err:
raise web.HTTPError(500, repr(err)) from err
self.set_header("Content-Type", "application/json")
self.finish(metadata.to_dict(trim=True))
async def delete(self, namespace, resource):
namespace = url_unescape(namespace)
resource = url_unescape(resource)
try:
self.log.debug(
"MetadataHandler: Deleting metadata instance '{}' in namespace '{}'..."
.format(resource, namespace))
metadata_manager = MetadataManager(namespace=namespace)
metadata_manager.remove(resource)
except (ValidationError, ValueError) as err:
raise web.HTTPError(400, str(err)) from err
except PermissionError as err:
raise web.HTTPError(403, str(err)) from err
except MetadataNotFoundError as err:
raise web.HTTPError(404, str(err)) from err
except Exception as err:
raise web.HTTPError(500, repr(err)) from err
self.set_status(204)
self.finish()
async def delete(self, schemaspace, resource):
schemaspace = url_unescape(schemaspace)
resource = url_unescape(resource)
parent = self.settings.get("elyra")
try:
self.log.debug(
f"MetadataHandler: Deleting metadata instance '{resource}' in schemaspace '{schemaspace}'..."
)
metadata_manager = MetadataManager(schemaspace=schemaspace, parent=parent)
metadata_manager.remove(resource)
except (ValidationError, ValueError) as err:
raise web.HTTPError(400, str(err)) from err
except PermissionError as err:
raise web.HTTPError(403, str(err)) from err
except MetadataNotFoundError as err:
raise web.HTTPError(404, str(err)) from err
except Exception as err:
raise web.HTTPError(500, repr(err)) from err
self.set_status(204)
self.finish()
async def get(self, schemaspace):
schemaspace = url_unescape(schemaspace)
try:
schema_manager = SchemaManager.instance()
schemas = schema_manager.get_schemaspace_schemas(schemaspace)
except (ValidationError, ValueError, SchemaNotFoundError) as err:
raise web.HTTPError(404, str(err)) from err
except Exception as err:
raise web.HTTPError(500, repr(err)) from err
schemas_model = {schemaspace: list(schemas.values())}
self.set_header("Content-Type", "application/json")
self.finish(schemas_model)
async def get(self, namespace):
namespace = url_unescape(namespace)
schema_manager = SchemaManager()
try:
schemas = schema_manager.get_namespace_schemas(namespace)
except (ValidationError, ValueError, SchemaNotFoundError) as err:
raise web.HTTPError(404, str(err)) from err
except Exception as err:
raise web.HTTPError(500, repr(err)) from err
schemas_model = dict()
schemas_model[namespace] = list(schemas.values())
self.set_header("Content-Type", 'application/json')
self.finish(schemas_model)
async def get(self, namespace):
namespace = url_unescape(namespace)
try:
metadata_manager = MetadataManager(namespace=namespace)
metadata = metadata_manager.get_all()
except (ValidationError, ValueError) as err:
raise web.HTTPError(400, str(err)) from err
except MetadataNotFoundError as err:
raise web.HTTPError(404, str(err)) from err
except Exception as err:
raise web.HTTPError(500, repr(err)) from err
metadata_model = dict()
metadata_model[namespace] = [r.to_dict(trim=True) for r in metadata]
self.set_header("Content-Type", 'application/json')
self.finish(metadata_model)
def _performConductorJWTLogonAndRetrieval(self, jwt_token: str,
env_dict: dict[str, Any]):
"""
Authenticate to Conductor with a JWT Token and setup the kernel environment variables.
:param jwt_token: JWT Token to authenticate with to Conductor
:param env_dict: Environment Dictionary of this Kernel launch
:return: None
"""
response = None
if not jwt_token:
return response
# Assemble JWT Auth logon REST call
env = self.env
if env["KERNEL_IG_UUID"] is None:
reasonErr = ("Instance group specified is None. Check environment "
"specified instance group is available.")
self.log_and_raise(http_status_code=500, reason=reasonErr)
# Determine hostname of ascd_endpoint and setup the HA List
portcolon = self.ascd_endpoint.rfind(":")
slash = self.ascd_endpoint.find("://")
host = self.ascd_endpoint[slash + 3:portcolon]
HA_LIST = env["KERNEL_CONDUCTOR_HA_ENDPOINTS"].split(",")
HA_LIST.insert(0, host)
header = "Accept: application/json"
authorization = "Authorization: Bearer %s" % jwt_token
cookie_jar = pjoin(env["KERNEL_NOTEBOOK_DATA_DIR"],
env["KERNEL_NOTEBOOK_COOKIE_JAR"])
sslconf = env["KERNEL_CURL_SECURITY_OPT"].split()
url = "{}/auth/logon/jwt?topology={}".format(self.ascd_endpoint,
env["KERNEL_TOPOLOGY"])
cmd = [
"curl", "-v", "-b", cookie_jar, "-X", "GET", "-H", header, "-H",
authorization, url
]
cmd[2:2] = sslconf
output, stderr = self._performRestCall(cmd, url, HA_LIST)
if "Error" in output:
reasonErr = "Failed to perform JWT Auth Logon. " + output.splitlines(
)[0]
self.log.warning(cmd)
self.log_and_raise(http_status_code=500, reason=reasonErr)
self.rest_credential = url_unescape(output)[1:-1]
# Assemble EGO Token Logon REST call
authorization = "Authorization: PlatformToken token=" + output.strip(
'"')
url = "%s/auth/logon" % self.ascd_endpoint
cmd = [
"curl", "-v", "-c", cookie_jar, "-X", "GET", "-H", header, "-H",
authorization, url
]
cmd[2:2] = sslconf
output, stderr = self._performRestCall(cmd, url, HA_LIST)
if "Error" in output:
reasonErr = "Failed to perform EGO Auth Logon. " + output.splitlines(
)[0]
self.log.warning(cmd)
self.log_and_raise(http_status_code=500, reason=reasonErr)
# Get the Python path to use to make sure the right conda environment is used
url = "{}/anaconda/instances/{}".format(
self.ascd_endpoint, env["KERNEL_ANACONDA_INST_UUID"])
cmd = [
"curl", "-v", "-b", cookie_jar, "-X", "GET", "-H", header, "-H",
authorization, url
]
cmd[2:2] = sslconf
output, stderr = self._performRestCall(cmd, url, HA_LIST)
response = json.loads(output) if output else None
if response is None or not response["parameters"]["deploy_home"][
"value"]:
reasonErr = "Could not retrieve anaconda instance. Verify anaconda instance with id "
reasonErr = reasonErr + env["KERNEL_ANACONDA_INST_UUID"] + " exists"
self.log.warning(cmd)
self.log_and_raise(http_status_code=500, reason=reasonErr)
else:
env_dict["KERNEL_PYSPARK_PYTHON"] = (
response["parameters"]["deploy_home"]["value"] +
"/anaconda/envs/" + env["KERNEL_ANACONDA_ENV"] + "/bin/python")
# Get instance group information we need
url = "{}/instances?id={}&fields=sparkinstancegroup,outputs".format(
self.ascd_endpoint,
env["KERNEL_IG_UUID"],
)
cmd = [
"curl", "-v", "-b", cookie_jar, "-X", "GET", "-H", header, "-H",
authorization, url
]
cmd[2:2] = sslconf
output, stderr = self._performRestCall(cmd, url, HA_LIST)
response = json.loads(output) if output else None
if response is None or len(response) == 0 or response[0] is None:
reasonErr = (
"Could not retrieve instance group. Verify instance group with id "
+ env["KERNEL_IG_UUID"] + " exists.")
self.log.warning(cmd)
self.log_and_raise(http_status_code=500, reason=reasonErr)
elif (response is None or response[0] is None or "value"
not in response[0]["outputs"]["batch_master_rest_urls"]):
reasonErr = (
"Could not retrieve outputs for instance group. Verify instance group with id "
+ env["KERNEL_IG_UUID"] + " is started")
self.log.warning(cmd)
self.log_and_raise(http_status_code=500, reason=reasonErr)
else:
env_dict["KERNEL_SPARK_HOME"] = response[0]["sparkinstancegroup"][
"sparkhomedir"]
env_dict["KERNEL_NOTEBOOK_MASTER_REST"] = response[0]["outputs"][
"batch_master_rest_urls"]["value"]
self.conductor_endpoint = response[0]["outputs"][
"one_batch_master_web_submission_url"]["value"]
return response
def _performConductorJWTLogonAndRetrieval(self, jwt_token, env_dict):
"""
Authenticate to Conductor with a JWT Token and setup the kernel environment variables.
:param jwt_token: JWT Token to authenticate with to Conductor
:param env_dict: Environment Dictionary of this Kernel launch
:return: None
"""
response = None
if not jwt_token:
return response
# Assemble JWT Auth logon REST call
env = self.env
if env['KERNEL_IG_UUID'] is None:
reasonErr = 'Instance group specified is None. Check environment ' \
'specified instance group is available.'
self.log_and_raise(http_status_code=500, reason=reasonErr)
# Determine hostname of ascd_endpoint and setup the HA List
portcolon = self.ascd_endpoint.rfind(':')
slash = self.ascd_endpoint.find('://')
host = self.ascd_endpoint[slash + 3:portcolon]
HA_LIST = env['KERNEL_CONDUCTOR_HA_ENDPOINTS'].split(',')
HA_LIST.insert(0, host)
header = 'Accept: application/json'
authorization = 'Authorization: Bearer %s' % jwt_token
cookie_jar = pjoin(env['KERNEL_NOTEBOOK_DATA_DIR'],
env['KERNEL_NOTEBOOK_COOKIE_JAR'])
sslconf = env['KERNEL_CURL_SECURITY_OPT'].split()
url = '%s/auth/logon/jwt?topology=%s' % (self.ascd_endpoint,
env['KERNEL_TOPOLOGY'])
cmd = [
'curl', '-v', '-b', cookie_jar, '-X', 'GET', '-H', header, '-H',
authorization, url
]
cmd[2:2] = sslconf
output, stderr = self._performRestCall(cmd, url, HA_LIST)
if 'Error' in output:
reasonErr = 'Failed to perform JWT Auth Logon. ' + output.splitlines(
)[0]
self.log.warning(cmd)
self.log_and_raise(http_status_code=500, reason=reasonErr)
self.rest_credential = url_unescape(output)[1:-1]
# Assemble EGO Token Logon REST call
authorization = 'Authorization: PlatformToken token=' + output.strip(
'"')
url = '%s/auth/logon' % self.ascd_endpoint
cmd = [
'curl', '-v', '-c', cookie_jar, '-X', 'GET', '-H', header, '-H',
authorization, url
]
cmd[2:2] = sslconf
output, stderr = self._performRestCall(cmd, url, HA_LIST)
if 'Error' in output:
reasonErr = 'Failed to perform EGO Auth Logon. ' + output.splitlines(
)[0]
self.log.warning(cmd)
self.log_and_raise(http_status_code=500, reason=reasonErr)
# Get the Python path to use to make sure the right conda environment is used
url = '%s/anaconda/instances/%s' % (self.ascd_endpoint,
env['KERNEL_ANACONDA_INST_UUID'])
cmd = [
'curl', '-v', '-b', cookie_jar, '-X', 'GET', '-H', header, '-H',
authorization, url
]
cmd[2:2] = sslconf
output, stderr = self._performRestCall(cmd, url, HA_LIST)
response = json.loads(output) if output else None
if response is None or not response['parameters']['deploy_home'][
'value']:
reasonErr = 'Could not retrieve anaconda instance. Verify anaconda instance with id '
reasonErr = reasonErr + env['KERNEL_ANACONDA_INST_UUID'] + ' exists'
self.log.warning(cmd)
self.log_and_raise(http_status_code=500, reason=reasonErr)
else:
env_dict['KERNEL_PYSPARK_PYTHON'] = response['parameters']['deploy_home']['value'] \
+ '/anaconda/envs/' + env['KERNEL_ANACONDA_ENV'] + '/bin/python'
# Get instance group information we need
url = '%s/instances?id=%s&fields=sparkinstancegroup,outputs' % (
self.ascd_endpoint, env['KERNEL_IG_UUID'])
cmd = [
'curl', '-v', '-b', cookie_jar, '-X', 'GET', '-H', header, '-H',
authorization, url
]
cmd[2:2] = sslconf
output, stderr = self._performRestCall(cmd, url, HA_LIST)
response = json.loads(output) if output else None
if response is None or len(response) == 0 or response[0] is None:
reasonErr = 'Could not retrieve instance group. Verify instance group with id ' \
+ env['KERNEL_IG_UUID'] + ' exists.'
self.log.warning(cmd)
self.log_and_raise(http_status_code=500, reason=reasonErr)
elif response is None or response[0] is None or 'value' not in response[
0]['outputs']['batch_master_rest_urls']:
reasonErr = 'Could not retrieve outputs for instance group. Verify instance group with id ' \
+ env['KERNEL_IG_UUID'] + ' is started'
self.log.warning(cmd)
self.log_and_raise(http_status_code=500, reason=reasonErr)
else:
env_dict['KERNEL_SPARK_HOME'] = response[0]['sparkinstancegroup'][
'sparkhomedir']
env_dict['KERNEL_NOTEBOOK_MASTER_REST'] = response[0]['outputs'][
'batch_master_rest_urls']['value']
self.conductor_endpoint = response[0]['outputs'][
'one_batch_master_web_submission_url']['value']
return response
