def put(self, namespace, resource):
namespace = url_unescape(namespace)
resource = url_unescape(resource)
try:
payload = self.get_json_body()
# get the current resource...
metadata_manager = MetadataManager(namespace=namespace)
current = metadata_manager.get(resource)
# convert to dictionary and update current with desired changes
updates = current.to_dict()
updates.update(payload)
# Check if name is in the payload and varies from resource, if so, raise 400
if 'name' in payload and payload['name'] != resource:
raise NotImplementedError("The attempt to rename instance '{}' to '{}' is not supported.".
format(resource, payload['name']))
instance = Metadata(**updates)
self.log.debug("MetadataHandler: Updating metadata instance '{}' in namespace '{}'...".
format(resource, namespace))
metadata = metadata_manager.add(resource, instance, replace=True)
except (NotImplementedError) as err:
raise web.HTTPError(400, str(err))
except (ValidationError, ValueError, FileNotFoundError) as err:
raise web.HTTPError(404, str(err))
except Exception as ex:
raise web.HTTPError(500, repr(ex))
self.set_status(200)
self.set_header("Content-Type", 'application/json')
self.finish(metadata.to_dict(trim=True))
async def put(self, namespace, resource):
namespace = url_unescape(namespace)
resource = url_unescape(resource)
try:
payload = self.get_json_body()
# Get the current resource to ensure its pre-existence
metadata_manager = MetadataManager(namespace=namespace)
metadata_manager.get(resource)
# Check if name is in the payload and varies from resource, if so, raise 400
if 'name' in payload and payload['name'] != resource:
raise NotImplementedError(
"The attempt to rename instance '{}' to '{}' is not supported."
.format(resource, payload['name']))
instance = Metadata.from_dict(namespace, {**payload})
self.log.debug(
"MetadataHandler: Updating metadata instance '{}' in namespace '{}'..."
.format(resource, namespace))
metadata = metadata_manager.update(resource, instance)
except (ValidationError, ValueError, NotImplementedError) as err:
raise web.HTTPError(400, str(err)) from err
except MetadataNotFoundError as err:
raise web.HTTPError(404, str(err)) from err
except Exception as err:
raise web.HTTPError(500, repr(err)) from err
self.set_status(200)
self.set_header("Content-Type", 'application/json')
self.finish(metadata.to_dict(trim=True))
async def get(self, namespace, resource):
namespace = url_unescape(namespace)
resource = url_unescape(resource)
schema_manager = SchemaManager()
try:
schema = schema_manager.get_schema(namespace, resource)
except (ValidationError, ValueError, SchemaNotFoundError) as err:
raise web.HTTPError(404, str(err)) from err
except Exception as err:
raise web.HTTPError(500, repr(err)) from err
self.set_header("Content-Type", 'application/json')
self.finish(schema)
def get(self, namespace, resource):
namespace = url_unescape(namespace)
resource = url_unescape(resource)
metadata_manager = MetadataManager(namespace=namespace)
try:
self.log.debug("MetadataNamespaceHandler: Fetching metadata resource '{}' from namespace '{}'...".
format(resource, namespace))
metadata = yield maybe_future(metadata_manager.get(resource))
except (ValidationError, KeyError) as err:
raise web.HTTPError(404, str(err))
except Exception as ex:
raise web.HTTPError(500, repr(ex))
self.set_header("Content-Type", 'application/json')
self.finish(metadata.to_dict())
def get(self, namespace, resource):
namespace = url_unescape(namespace)
resource = url_unescape(resource)
schema_manager = SchemaManager()
try:
self.log.debug("SchemaResourceHandler: Fetching schema '{}' for namespace '{}'...".
format(resource, namespace))
schema = yield maybe_future(schema_manager.get_schema(namespace, resource))
except (ValidationError, ValueError, FileNotFoundError) as err:
raise web.HTTPError(404, str(err))
except Exception as ex:
raise web.HTTPError(500, repr(ex))
self.set_header("Content-Type", 'application/json')
self.finish(schema)
def test_url_unescape():
# decodes a url string to a plain string
# these tests decode paths with spaces
path = url_unescape('/this%20is%20a%20test/for%20spaces/')
nt.assert_equal(path, '/this is a test/for spaces/')
path = url_unescape('notebook%20with%20space.ipynb')
nt.assert_equal(path, 'notebook with space.ipynb')
path = url_unescape('/path%20with%20a/notebook%20and%20space.ipynb')
nt.assert_equal(path, '/path with a/notebook and space.ipynb')
path = url_unescape(
'/%20%21%40%24%23%25%5E%26%2A%20/%20test%20%25%5E%20notebook%20%40%23%24%20name.ipynb')
nt.assert_equal(path, '/ !@$#%^&* / test %^ notebook @#$ name.ipynb')
async def get(self, namespace, resource):
namespace = url_unescape(namespace)
resource = url_unescape(resource)
try:
metadata_manager = MetadataManager(namespace=namespace)
metadata = metadata_manager.get(resource)
except (ValidationError, ValueError, NotImplementedError) as err:
raise web.HTTPError(400, str(err)) from err
except MetadataNotFoundError as err:
raise web.HTTPError(404, str(err)) from err
except Exception as err:
raise web.HTTPError(500, repr(err)) from err
self.set_header("Content-Type", 'application/json')
self.finish(metadata.to_dict(trim=True))
async def get(self, kernel_name):
ksm = self.kernel_spec_cache
kernel_name = url_unescape(kernel_name)
kernel_user_filter = self.request.query_arguments.get('user')
kernel_user = None
if kernel_user_filter:
kernel_user = kernel_user_filter[0].decode("utf-8")
try:
spec = await maybe_future(ksm.get_kernel_spec(kernel_name))
except KeyError:
raise web.HTTPError(404, u'Kernel spec %s not found' % kernel_name)
if is_kernelspec_model(spec):
model = spec
else:
model = kernelspec_model(self, kernel_name, spec.to_dict(),
spec.resource_dir)
d = apply_user_filter(model, self.settings['eg_authorized_users'],
self.settings['eg_unauthorized_users'],
kernel_user)
if d is None:
raise web.HTTPError(
403, u'User %s is not authorized to use kernel spec %s' %
(kernel_user, kernel_name))
self.set_header("Content-Type", 'application/json')
self.finish(json.dumps(model))
def post(self, namespace):
namespace = url_unescape(namespace)
try:
instance = self._validate_body()
self.log.debug(
"MetadataHandler: Creating metadata instance '{}' in namespace '{}'..."
.format(instance.name, namespace))
metadata_manager = MetadataManager(namespace=namespace)
metadata = metadata_manager.add(instance.name,
instance,
replace=False)
except (ValidationError, ValueError, SyntaxError) as se:
raise web.HTTPError(400, str(se))
except FileNotFoundError as err:
raise web.HTTPError(404, str(err))
except FileExistsError as err:
raise web.HTTPError(409, str(err))
except Exception as ex:
raise web.HTTPError(500, repr(ex))
self.set_status(201)
self.set_header("Content-Type", 'application/json')
location = url_path_join(self.base_url, 'elyra', 'metadata', namespace,
metadata.name)
self.set_header('Location', location)
self.finish(metadata.to_dict(trim=True))
def test_url_unescape():
# decodes a url string to a plain string
# these tests decode paths with spaces
path = url_unescape('/this%20is%20a%20test/for%20spaces/')
nt.assert_equal(path, '/this is a test/for spaces/')
path = url_unescape('notebook%20with%20space.ipynb')
nt.assert_equal(path, 'notebook with space.ipynb')
path = url_unescape('/path%20with%20a/notebook%20and%20space.ipynb')
nt.assert_equal(path, '/path with a/notebook and space.ipynb')
path = url_unescape(
'/%20%21%40%24%23%25%5E%26%2A%20/%20test%20%25%5E%20notebook%20%40%23%24%20name.ipynb'
)
nt.assert_equal(path, '/ !@$#%^&* / test %^ notebook @#$ name.ipynb')
def post(self, path):
'''
Write uploaded files to disk.
:param path:
'''
files = self.request.files
if not len(files):
raise web.HTTPError(400, 'missing files to upload')
root_path = os.path.join(self.work_dir, path.strip('/'))
written_paths = []
for filename, metas in files.items():
path = url_unescape(os.path.join(root_path, filename))
with open(path, 'wb') as fh:
fh.write(metas[0].body)
written_paths.append(path)
self.finish({'files': written_paths, 'path': url_unescape(root_path)})
def delete(self, namespace, resource):
namespace = url_unescape(namespace)
resource = url_unescape(resource)
try:
self.log.debug("MetadataHandler: Deleting metadata instance '{}' in namespace '{}'...".
format(resource, namespace))
metadata_manager = MetadataManager(namespace=namespace)
metadata_manager.remove(resource)
except PermissionError as err:
raise web.HTTPError(403, str(err))
except (ValidationError, ValueError, FileNotFoundError) as err:
raise web.HTTPError(404, str(err))
except Exception as ex:
raise web.HTTPError(500, repr(ex))
self.set_status(204)
self.finish()
async def get(self, namespace, resource):
namespace = url_unescape(namespace)
resource = url_unescape(resource)
try:
metadata_manager = MetadataManager(namespace=namespace)
self.log.debug(
"MetadataResourceHandler: Fetching metadata resource '{}' from namespace '{}'..."
.format(resource, namespace))
metadata = metadata_manager.get(resource)
except (ValidationError, ValueError, NotImplementedError) as err:
raise web.HTTPError(400, str(err))
except FileNotFoundError as err:
raise web.HTTPError(404, str(err))
except Exception as ex:
raise web.HTTPError(500, repr(ex))
self.set_header("Content-Type", 'application/json')
self.finish(metadata.to_dict(trim=True))
def post(self, path):
'''
Write uploaded files to disk.
:param path:
'''
files = self.request.files
if not len(files):
raise web.HTTPError(400, 'missing files to upload')
root_path = os.path.join(self.work_dir, path.strip('/'))
written_paths = []
for filename, metas in files.items():
path = url_unescape(os.path.join(root_path, filename))
with open(path, 'wb') as fh:
fh.write(metas[0].body)
written_paths.append(path)
self.finish({
'files' : written_paths,
'path' : url_unescape(root_path)
})
async def get(self, namespace):
namespace = url_unescape(namespace)
schema_manager = SchemaManager()
try:
schemas = schema_manager.get_namespace_schemas(namespace)
except (ValidationError, ValueError, SchemaNotFoundError) as err:
raise web.HTTPError(404, str(err)) from err
except Exception as err:
raise web.HTTPError(500, repr(err)) from err
schemas_model = dict()
schemas_model[namespace] = list(schemas.values())
self.set_header("Content-Type", 'application/json')
self.finish(schemas_model)
def get(self, namespace):
namespace = url_unescape(namespace)
try:
metadata_manager = MetadataManager(namespace=namespace)
self.log.debug("MetadataHandler: Fetching all metadata resources from namespace '{}'...".format(namespace))
metadata = yield maybe_future(metadata_manager.get_all())
except (ValidationError, ValueError, FileNotFoundError) as err:
raise web.HTTPError(404, str(err))
except Exception as ex:
raise web.HTTPError(500, repr(ex))
metadata_model = dict()
metadata_model[namespace] = [r.to_dict(trim=True) for r in metadata]
self.set_header("Content-Type", 'application/json')
self.finish(metadata_model)
def get(self, namespace):
namespace = url_unescape(namespace)
schema_manager = SchemaManager()
try:
self.log.debug("SchemaHandler: Fetching all schemas for namespace '{}'...".format(namespace))
schemas = yield maybe_future(schema_manager.get_namespace_schemas(namespace))
except (ValidationError, ValueError, FileNotFoundError) as err:
raise web.HTTPError(404, str(err))
except Exception as ex:
raise web.HTTPError(500, repr(ex))
schemas_model = dict()
schemas_model[namespace] = list(schemas.values())
self.set_header("Content-Type", 'application/json')
self.finish(schemas_model)
async def get(self, namespace):
namespace = url_unescape(namespace)
try:
metadata_manager = MetadataManager(namespace=namespace)
metadata = metadata_manager.get_all()
except (ValidationError, ValueError) as err:
raise web.HTTPError(400, str(err)) from err
except MetadataNotFoundError as err:
raise web.HTTPError(404, str(err)) from err
except Exception as err:
raise web.HTTPError(500, repr(err)) from err
metadata_model = dict()
metadata_model[namespace] = [r.to_dict(trim=True) for r in metadata]
self.set_header("Content-Type", 'application/json')
self.finish(metadata_model)
def wrap(*args, **kwargs):
# print("auth_wrapper:")
# print(" cookies=", flask.request.cookies)
if 'auth' in flask.request.args:
auth = flask.request.args['auth']
if auth == self.token:
# print("SETTING COOKIE 2")
response = flask.Response(f(*args, **kwargs))
self.set_cookie(response, self.cookie_name, auth)
return f(*args, **kwargs)
if self.cookie_name not in flask.request.cookies:
# print("%s not in cookies" % self.cookie_name)
return flask.Response('Access Forbidden', status=403)
cookie = flask.request.cookies[self.cookie_name]
cookie = url_unescape(cookie)
for item in cookie.split(','):
# print(" compare: '%s' ****** '%s'" % (item, self.token))
if item == self.token:
return f(*args, **kwargs)
return flask.Response('Access Forbidden', status=403)
def test_url_unescape():
#test for ~
path = url_unescape('hostname/%7Esomething/')
assert path == 'hostname/~something/'
def test_url_unescape():
#test for < > + { } [] | ~ ' ; ? :
path = url_unescape(
'hostname/%3C%3E%2B%7B%7C%7D%5B%5D%7E%27%3B%3F%3Asomething/')
assert path == 'hostname/<>+{|}[]~\';?:something/'
def test_url_unescape_1():
path = url_unescape('/some%20test%20case%20for%20spaces')
assert path == '/some test case for spaces'
def _performConductorJWTLogonAndRetrieval(self, jwt_token, env_dict):
"""
Authenticate to Conductor with a JWT Token and setup the kernel environment variables.
:param jwt_token: JWT Token to authenticate with to Conductor
:param env_dict: Environment Dictionary of this Kernel launch
:return: None
"""
response = None
if not jwt_token:
return response
# Assemble JWT Auth logon REST call
env = self.env
if env['KERNEL_IG_UUID'] is None:
reasonErr = 'Instance group specified is None. Check environment ' \
'specified instance group is available.'
self.log_and_raise(http_status_code=500, reason=reasonErr)
# Determine hostname of ascd_endpoint and setup the HA List
portcolon = self.ascd_endpoint.rfind(':')
slash = self.ascd_endpoint.find('://')
host = self.ascd_endpoint[slash + 3:portcolon]
HA_LIST = env['KERNEL_CONDUCTOR_HA_ENDPOINTS'].split(',')
HA_LIST.insert(0, host)
header = 'Accept: application/json'
authorization = 'Authorization: Bearer %s' % jwt_token
cookie_jar = pjoin(env['KERNEL_NOTEBOOK_DATA_DIR'],
env['KERNEL_NOTEBOOK_COOKIE_JAR'])
sslconf = env['KERNEL_CURL_SECURITY_OPT'].split()
url = '%s/auth/logon/jwt?topology=%s' % (self.ascd_endpoint,
env['KERNEL_TOPOLOGY'])
cmd = [
'curl', '-v', '-b', cookie_jar, '-X', 'GET', '-H', header, '-H',
authorization, url
]
cmd[2:2] = sslconf
output, stderr = self._performRestCall(cmd, url, HA_LIST)
if 'Error' in output:
reasonErr = 'Failed to perform JWT Auth Logon. ' + output.splitlines(
)[0]
self.log.warning(cmd)
self.log_and_raise(http_status_code=500, reason=reasonErr)
self.rest_credential = url_unescape(output)[1:-1]
# Assemble EGO Token Logon REST call
authorization = 'Authorization: PlatformToken token=' + output.strip(
'"')
url = '%s/auth/logon' % self.ascd_endpoint
cmd = [
'curl', '-v', '-c', cookie_jar, '-X', 'GET', '-H', header, '-H',
authorization, url
]
cmd[2:2] = sslconf
output, stderr = self._performRestCall(cmd, url, HA_LIST)
if 'Error' in output:
reasonErr = 'Failed to perform EGO Auth Logon. ' + output.splitlines(
)[0]
self.log.warning(cmd)
self.log_and_raise(http_status_code=500, reason=reasonErr)
# Get the Python path to use to make sure the right conda environment is used
url = '%s/anaconda/instances/%s' % (self.ascd_endpoint,
env['KERNEL_ANACONDA_INST_UUID'])
cmd = [
'curl', '-v', '-b', cookie_jar, '-X', 'GET', '-H', header, '-H',
authorization, url
]
cmd[2:2] = sslconf
output, stderr = self._performRestCall(cmd, url, HA_LIST)
response = json.loads(output) if output else None
if response is None or not response['parameters']['deploy_home'][
'value']:
reasonErr = 'Could not retrieve anaconda instance. Verify anaconda instance with id '
reasonErr = reasonErr + env['KERNEL_ANACONDA_INST_UUID'] + ' exists'
self.log.warning(cmd)
self.log_and_raise(http_status_code=500, reason=reasonErr)
else:
env_dict['KERNEL_PYSPARK_PYTHON'] = response['parameters']['deploy_home']['value'] \
+ '/anaconda/envs/' + env['KERNEL_ANACONDA_ENV'] + '/bin/python'
# Get instance group information we need
url = '%s/instances?id=%s&fields=sparkinstancegroup,outputs' % (
self.ascd_endpoint, env['KERNEL_IG_UUID'])
cmd = [
'curl', '-v', '-b', cookie_jar, '-X', 'GET', '-H', header, '-H',
authorization, url
]
cmd[2:2] = sslconf
output, stderr = self._performRestCall(cmd, url, HA_LIST)
response = json.loads(output) if output else None
if response is None or len(response) == 0 or response[0] is None:
reasonErr = 'Could not retrieve instance group. Verify instance group with id ' \
+ env['KERNEL_IG_UUID'] + ' exists.'
self.log.warning(cmd)
self.log_and_raise(http_status_code=500, reason=reasonErr)
elif response is None or response[0] is None or 'value' not in response[
0]['outputs']['batch_master_rest_urls']:
reasonErr = 'Could not retrieve outputs for instance group. Verify instance group with id ' \
+ env['KERNEL_IG_UUID'] + ' is started'
self.log.warning(cmd)
self.log_and_raise(http_status_code=500, reason=reasonErr)
else:
env_dict['KERNEL_SPARK_HOME'] = response[0]['sparkinstancegroup'][
'sparkhomedir']
env_dict['KERNEL_NOTEBOOK_MASTER_REST'] = response[0]['outputs'][
'batch_master_rest_urls']['value']
self.conductor_endpoint = response[0]['outputs'][
'one_batch_master_web_submission_url']['value']
return response
