def put(self, namespace, resource): namespace = url_unescape(namespace) resource = url_unescape(resource) try: payload = self.get_json_body() # get the current resource... metadata_manager = MetadataManager(namespace=namespace) current = metadata_manager.get(resource) # convert to dictionary and update current with desired changes updates = current.to_dict() updates.update(payload) # Check if name is in the payload and varies from resource, if so, raise 400 if 'name' in payload and payload['name'] != resource: raise NotImplementedError("The attempt to rename instance '{}' to '{}' is not supported.". format(resource, payload['name'])) instance = Metadata(**updates) self.log.debug("MetadataHandler: Updating metadata instance '{}' in namespace '{}'...". format(resource, namespace)) metadata = metadata_manager.add(resource, instance, replace=True) except (NotImplementedError) as err: raise web.HTTPError(400, str(err)) except (ValidationError, ValueError, FileNotFoundError) as err: raise web.HTTPError(404, str(err)) except Exception as ex: raise web.HTTPError(500, repr(ex)) self.set_status(200) self.set_header("Content-Type", 'application/json') self.finish(metadata.to_dict(trim=True))
async def put(self, namespace, resource): namespace = url_unescape(namespace) resource = url_unescape(resource) try: payload = self.get_json_body() # Get the current resource to ensure its pre-existence metadata_manager = MetadataManager(namespace=namespace) metadata_manager.get(resource) # Check if name is in the payload and varies from resource, if so, raise 400 if 'name' in payload and payload['name'] != resource: raise NotImplementedError( "The attempt to rename instance '{}' to '{}' is not supported." .format(resource, payload['name'])) instance = Metadata.from_dict(namespace, {**payload}) self.log.debug( "MetadataHandler: Updating metadata instance '{}' in namespace '{}'..." .format(resource, namespace)) metadata = metadata_manager.update(resource, instance) except (ValidationError, ValueError, NotImplementedError) as err: raise web.HTTPError(400, str(err)) from err except MetadataNotFoundError as err: raise web.HTTPError(404, str(err)) from err except Exception as err: raise web.HTTPError(500, repr(err)) from err self.set_status(200) self.set_header("Content-Type", 'application/json') self.finish(metadata.to_dict(trim=True))
async def get(self, namespace, resource): namespace = url_unescape(namespace) resource = url_unescape(resource) schema_manager = SchemaManager() try: schema = schema_manager.get_schema(namespace, resource) except (ValidationError, ValueError, SchemaNotFoundError) as err: raise web.HTTPError(404, str(err)) from err except Exception as err: raise web.HTTPError(500, repr(err)) from err self.set_header("Content-Type", 'application/json') self.finish(schema)
def get(self, namespace, resource): namespace = url_unescape(namespace) resource = url_unescape(resource) metadata_manager = MetadataManager(namespace=namespace) try: self.log.debug("MetadataNamespaceHandler: Fetching metadata resource '{}' from namespace '{}'...". format(resource, namespace)) metadata = yield maybe_future(metadata_manager.get(resource)) except (ValidationError, KeyError) as err: raise web.HTTPError(404, str(err)) except Exception as ex: raise web.HTTPError(500, repr(ex)) self.set_header("Content-Type", 'application/json') self.finish(metadata.to_dict())
def get(self, namespace, resource): namespace = url_unescape(namespace) resource = url_unescape(resource) schema_manager = SchemaManager() try: self.log.debug("SchemaResourceHandler: Fetching schema '{}' for namespace '{}'...". format(resource, namespace)) schema = yield maybe_future(schema_manager.get_schema(namespace, resource)) except (ValidationError, ValueError, FileNotFoundError) as err: raise web.HTTPError(404, str(err)) except Exception as ex: raise web.HTTPError(500, repr(ex)) self.set_header("Content-Type", 'application/json') self.finish(schema)
def test_url_unescape(): # decodes a url string to a plain string # these tests decode paths with spaces path = url_unescape('/this%20is%20a%20test/for%20spaces/') nt.assert_equal(path, '/this is a test/for spaces/') path = url_unescape('notebook%20with%20space.ipynb') nt.assert_equal(path, 'notebook with space.ipynb') path = url_unescape('/path%20with%20a/notebook%20and%20space.ipynb') nt.assert_equal(path, '/path with a/notebook and space.ipynb') path = url_unescape( '/%20%21%40%24%23%25%5E%26%2A%20/%20test%20%25%5E%20notebook%20%40%23%24%20name.ipynb') nt.assert_equal(path, '/ !@$#%^&* / test %^ notebook @#$ name.ipynb')
async def get(self, namespace, resource): namespace = url_unescape(namespace) resource = url_unescape(resource) try: metadata_manager = MetadataManager(namespace=namespace) metadata = metadata_manager.get(resource) except (ValidationError, ValueError, NotImplementedError) as err: raise web.HTTPError(400, str(err)) from err except MetadataNotFoundError as err: raise web.HTTPError(404, str(err)) from err except Exception as err: raise web.HTTPError(500, repr(err)) from err self.set_header("Content-Type", 'application/json') self.finish(metadata.to_dict(trim=True))
async def get(self, kernel_name): ksm = self.kernel_spec_cache kernel_name = url_unescape(kernel_name) kernel_user_filter = self.request.query_arguments.get('user') kernel_user = None if kernel_user_filter: kernel_user = kernel_user_filter[0].decode("utf-8") try: spec = await maybe_future(ksm.get_kernel_spec(kernel_name)) except KeyError: raise web.HTTPError(404, u'Kernel spec %s not found' % kernel_name) if is_kernelspec_model(spec): model = spec else: model = kernelspec_model(self, kernel_name, spec.to_dict(), spec.resource_dir) d = apply_user_filter(model, self.settings['eg_authorized_users'], self.settings['eg_unauthorized_users'], kernel_user) if d is None: raise web.HTTPError( 403, u'User %s is not authorized to use kernel spec %s' % (kernel_user, kernel_name)) self.set_header("Content-Type", 'application/json') self.finish(json.dumps(model))
def post(self, namespace): namespace = url_unescape(namespace) try: instance = self._validate_body() self.log.debug( "MetadataHandler: Creating metadata instance '{}' in namespace '{}'..." .format(instance.name, namespace)) metadata_manager = MetadataManager(namespace=namespace) metadata = metadata_manager.add(instance.name, instance, replace=False) except (ValidationError, ValueError, SyntaxError) as se: raise web.HTTPError(400, str(se)) except FileNotFoundError as err: raise web.HTTPError(404, str(err)) except FileExistsError as err: raise web.HTTPError(409, str(err)) except Exception as ex: raise web.HTTPError(500, repr(ex)) self.set_status(201) self.set_header("Content-Type", 'application/json') location = url_path_join(self.base_url, 'elyra', 'metadata', namespace, metadata.name) self.set_header('Location', location) self.finish(metadata.to_dict(trim=True))
def test_url_unescape(): # decodes a url string to a plain string # these tests decode paths with spaces path = url_unescape('/this%20is%20a%20test/for%20spaces/') nt.assert_equal(path, '/this is a test/for spaces/') path = url_unescape('notebook%20with%20space.ipynb') nt.assert_equal(path, 'notebook with space.ipynb') path = url_unescape('/path%20with%20a/notebook%20and%20space.ipynb') nt.assert_equal(path, '/path with a/notebook and space.ipynb') path = url_unescape( '/%20%21%40%24%23%25%5E%26%2A%20/%20test%20%25%5E%20notebook%20%40%23%24%20name.ipynb' ) nt.assert_equal(path, '/ !@$#%^&* / test %^ notebook @#$ name.ipynb')
def post(self, path): ''' Write uploaded files to disk. :param path: ''' files = self.request.files if not len(files): raise web.HTTPError(400, 'missing files to upload') root_path = os.path.join(self.work_dir, path.strip('/')) written_paths = [] for filename, metas in files.items(): path = url_unescape(os.path.join(root_path, filename)) with open(path, 'wb') as fh: fh.write(metas[0].body) written_paths.append(path) self.finish({'files': written_paths, 'path': url_unescape(root_path)})
def delete(self, namespace, resource): namespace = url_unescape(namespace) resource = url_unescape(resource) try: self.log.debug("MetadataHandler: Deleting metadata instance '{}' in namespace '{}'...". format(resource, namespace)) metadata_manager = MetadataManager(namespace=namespace) metadata_manager.remove(resource) except PermissionError as err: raise web.HTTPError(403, str(err)) except (ValidationError, ValueError, FileNotFoundError) as err: raise web.HTTPError(404, str(err)) except Exception as ex: raise web.HTTPError(500, repr(ex)) self.set_status(204) self.finish()
async def get(self, namespace, resource): namespace = url_unescape(namespace) resource = url_unescape(resource) try: metadata_manager = MetadataManager(namespace=namespace) self.log.debug( "MetadataResourceHandler: Fetching metadata resource '{}' from namespace '{}'..." .format(resource, namespace)) metadata = metadata_manager.get(resource) except (ValidationError, ValueError, NotImplementedError) as err: raise web.HTTPError(400, str(err)) except FileNotFoundError as err: raise web.HTTPError(404, str(err)) except Exception as ex: raise web.HTTPError(500, repr(ex)) self.set_header("Content-Type", 'application/json') self.finish(metadata.to_dict(trim=True))
def post(self, path): ''' Write uploaded files to disk. :param path: ''' files = self.request.files if not len(files): raise web.HTTPError(400, 'missing files to upload') root_path = os.path.join(self.work_dir, path.strip('/')) written_paths = [] for filename, metas in files.items(): path = url_unescape(os.path.join(root_path, filename)) with open(path, 'wb') as fh: fh.write(metas[0].body) written_paths.append(path) self.finish({ 'files' : written_paths, 'path' : url_unescape(root_path) })
async def get(self, namespace): namespace = url_unescape(namespace) schema_manager = SchemaManager() try: schemas = schema_manager.get_namespace_schemas(namespace) except (ValidationError, ValueError, SchemaNotFoundError) as err: raise web.HTTPError(404, str(err)) from err except Exception as err: raise web.HTTPError(500, repr(err)) from err schemas_model = dict() schemas_model[namespace] = list(schemas.values()) self.set_header("Content-Type", 'application/json') self.finish(schemas_model)
def get(self, namespace): namespace = url_unescape(namespace) try: metadata_manager = MetadataManager(namespace=namespace) self.log.debug("MetadataHandler: Fetching all metadata resources from namespace '{}'...".format(namespace)) metadata = yield maybe_future(metadata_manager.get_all()) except (ValidationError, ValueError, FileNotFoundError) as err: raise web.HTTPError(404, str(err)) except Exception as ex: raise web.HTTPError(500, repr(ex)) metadata_model = dict() metadata_model[namespace] = [r.to_dict(trim=True) for r in metadata] self.set_header("Content-Type", 'application/json') self.finish(metadata_model)
def get(self, namespace): namespace = url_unescape(namespace) schema_manager = SchemaManager() try: self.log.debug("SchemaHandler: Fetching all schemas for namespace '{}'...".format(namespace)) schemas = yield maybe_future(schema_manager.get_namespace_schemas(namespace)) except (ValidationError, ValueError, FileNotFoundError) as err: raise web.HTTPError(404, str(err)) except Exception as ex: raise web.HTTPError(500, repr(ex)) schemas_model = dict() schemas_model[namespace] = list(schemas.values()) self.set_header("Content-Type", 'application/json') self.finish(schemas_model)
async def get(self, namespace): namespace = url_unescape(namespace) try: metadata_manager = MetadataManager(namespace=namespace) metadata = metadata_manager.get_all() except (ValidationError, ValueError) as err: raise web.HTTPError(400, str(err)) from err except MetadataNotFoundError as err: raise web.HTTPError(404, str(err)) from err except Exception as err: raise web.HTTPError(500, repr(err)) from err metadata_model = dict() metadata_model[namespace] = [r.to_dict(trim=True) for r in metadata] self.set_header("Content-Type", 'application/json') self.finish(metadata_model)
def wrap(*args, **kwargs): # print("auth_wrapper:") # print(" cookies=", flask.request.cookies) if 'auth' in flask.request.args: auth = flask.request.args['auth'] if auth == self.token: # print("SETTING COOKIE 2") response = flask.Response(f(*args, **kwargs)) self.set_cookie(response, self.cookie_name, auth) return f(*args, **kwargs) if self.cookie_name not in flask.request.cookies: # print("%s not in cookies" % self.cookie_name) return flask.Response('Access Forbidden', status=403) cookie = flask.request.cookies[self.cookie_name] cookie = url_unescape(cookie) for item in cookie.split(','): # print(" compare: '%s' ****** '%s'" % (item, self.token)) if item == self.token: return f(*args, **kwargs) return flask.Response('Access Forbidden', status=403)
def test_url_unescape(): #test for ~ path = url_unescape('hostname/%7Esomething/') assert path == 'hostname/~something/'
def test_url_unescape(): #test for < > + { } [] | ~ ' ; ? : path = url_unescape( 'hostname/%3C%3E%2B%7B%7C%7D%5B%5D%7E%27%3B%3F%3Asomething/') assert path == 'hostname/<>+{|}[]~\';?:something/'
def test_url_unescape_1(): path = url_unescape('/some%20test%20case%20for%20spaces') assert path == '/some test case for spaces'
def _performConductorJWTLogonAndRetrieval(self, jwt_token, env_dict): """ Authenticate to Conductor with a JWT Token and setup the kernel environment variables. :param jwt_token: JWT Token to authenticate with to Conductor :param env_dict: Environment Dictionary of this Kernel launch :return: None """ response = None if not jwt_token: return response # Assemble JWT Auth logon REST call env = self.env if env['KERNEL_IG_UUID'] is None: reasonErr = 'Instance group specified is None. Check environment ' \ 'specified instance group is available.' self.log_and_raise(http_status_code=500, reason=reasonErr) # Determine hostname of ascd_endpoint and setup the HA List portcolon = self.ascd_endpoint.rfind(':') slash = self.ascd_endpoint.find('://') host = self.ascd_endpoint[slash + 3:portcolon] HA_LIST = env['KERNEL_CONDUCTOR_HA_ENDPOINTS'].split(',') HA_LIST.insert(0, host) header = 'Accept: application/json' authorization = 'Authorization: Bearer %s' % jwt_token cookie_jar = pjoin(env['KERNEL_NOTEBOOK_DATA_DIR'], env['KERNEL_NOTEBOOK_COOKIE_JAR']) sslconf = env['KERNEL_CURL_SECURITY_OPT'].split() url = '%s/auth/logon/jwt?topology=%s' % (self.ascd_endpoint, env['KERNEL_TOPOLOGY']) cmd = [ 'curl', '-v', '-b', cookie_jar, '-X', 'GET', '-H', header, '-H', authorization, url ] cmd[2:2] = sslconf output, stderr = self._performRestCall(cmd, url, HA_LIST) if 'Error' in output: reasonErr = 'Failed to perform JWT Auth Logon. ' + output.splitlines( )[0] self.log.warning(cmd) self.log_and_raise(http_status_code=500, reason=reasonErr) self.rest_credential = url_unescape(output)[1:-1] # Assemble EGO Token Logon REST call authorization = 'Authorization: PlatformToken token=' + output.strip( '"') url = '%s/auth/logon' % self.ascd_endpoint cmd = [ 'curl', '-v', '-c', cookie_jar, '-X', 'GET', '-H', header, '-H', authorization, url ] cmd[2:2] = sslconf output, stderr = self._performRestCall(cmd, url, HA_LIST) if 'Error' in output: reasonErr = 'Failed to perform EGO Auth Logon. ' + output.splitlines( )[0] self.log.warning(cmd) self.log_and_raise(http_status_code=500, reason=reasonErr) # Get the Python path to use to make sure the right conda environment is used url = '%s/anaconda/instances/%s' % (self.ascd_endpoint, env['KERNEL_ANACONDA_INST_UUID']) cmd = [ 'curl', '-v', '-b', cookie_jar, '-X', 'GET', '-H', header, '-H', authorization, url ] cmd[2:2] = sslconf output, stderr = self._performRestCall(cmd, url, HA_LIST) response = json.loads(output) if output else None if response is None or not response['parameters']['deploy_home'][ 'value']: reasonErr = 'Could not retrieve anaconda instance. Verify anaconda instance with id ' reasonErr = reasonErr + env['KERNEL_ANACONDA_INST_UUID'] + ' exists' self.log.warning(cmd) self.log_and_raise(http_status_code=500, reason=reasonErr) else: env_dict['KERNEL_PYSPARK_PYTHON'] = response['parameters']['deploy_home']['value'] \ + '/anaconda/envs/' + env['KERNEL_ANACONDA_ENV'] + '/bin/python' # Get instance group information we need url = '%s/instances?id=%s&fields=sparkinstancegroup,outputs' % ( self.ascd_endpoint, env['KERNEL_IG_UUID']) cmd = [ 'curl', '-v', '-b', cookie_jar, '-X', 'GET', '-H', header, '-H', authorization, url ] cmd[2:2] = sslconf output, stderr = self._performRestCall(cmd, url, HA_LIST) response = json.loads(output) if output else None if response is None or len(response) == 0 or response[0] is None: reasonErr = 'Could not retrieve instance group. Verify instance group with id ' \ + env['KERNEL_IG_UUID'] + ' exists.' self.log.warning(cmd) self.log_and_raise(http_status_code=500, reason=reasonErr) elif response is None or response[0] is None or 'value' not in response[ 0]['outputs']['batch_master_rest_urls']: reasonErr = 'Could not retrieve outputs for instance group. Verify instance group with id ' \ + env['KERNEL_IG_UUID'] + ' is started' self.log.warning(cmd) self.log_and_raise(http_status_code=500, reason=reasonErr) else: env_dict['KERNEL_SPARK_HOME'] = response[0]['sparkinstancegroup'][ 'sparkhomedir'] env_dict['KERNEL_NOTEBOOK_MASTER_REST'] = response[0]['outputs'][ 'batch_master_rest_urls']['value'] self.conductor_endpoint = response[0]['outputs'][ 'one_batch_master_web_submission_url']['value'] return response