def start_training(host: str, port: str, use_ssl: bool, request_param_generator, params): params_raw = json.dumps(params) if type(params) is dict else params res = call_and_get_response('post', 'ml/modeltraining', host, port, request_param_generator, use_ssl, params_raw, EXTRA_HEADERS) return res.json()
def cancel_load(host, port, use_ssl, request_param_generator, load_id): payload = { 'loadId': load_id } res = call_and_get_response('get', LOADER_ACTION, host, port, request_param_generator, use_ssl, payload) return res.status_code == 200
def do_sparql_explain(query: str, host: str, port: str, use_ssl: bool, request_param_generator, accept_type='text/html', path_prefix: str = ''): query_type = get_query_type(query) action = query_type_to_action(query_type) data = { 'explain': 'dynamic', } if action == 'sparql': data['query'] = query elif action == 'sparqlupdate': data['update'] = query extra_headers = {'Accept': accept_type} sparql_path = SPARQL_ACTION if path_prefix == '' else f'{path_prefix}/{SPARQL_ACTION}' res = call_and_get_response('post', sparql_path, host, port, request_param_generator, use_ssl, data, extra_headers) return res.content.decode('utf-8')
def do_load(host, port, load_format, use_ssl, source, region, arn, fail_on_error, request_param_generator, mode="AUTO", parallelism="HIGH", update_single_cardinality="FALSE", queue_request="FALSE", dependencies=[]): payload = { 'source': source, 'format': load_format, 'mode': mode, 'region': region, 'failOnError': fail_on_error, 'parallelism': parallelism, 'updateSingleCardinalityProperties': update_single_cardinality, 'queueRequest': queue_request } if arn != '': payload['iamRoleArn'] = arn if dependencies: payload['dependencies'] = json.dumps(dependencies) res = call_and_get_response('post', LOADER_ACTION, host, port, request_param_generator, use_ssl, payload) return res.json()
def do_sparql_query(query, host, port, use_ssl, request_param_generator, extra_headers=None, path_prefix: str = ''): if extra_headers is None: extra_headers = {} logger.debug(f'query={query}, endpoint={host}, port={port}') query_type = get_query_type(query) action = query_type_to_action(query_type) data = {} if action == 'sparql': data['query'] = query elif action == 'sparqlupdate': data['update'] = query sparql_path = SPARQL_ACTION if path_prefix == '' else f'{path_prefix}/{SPARQL_ACTION}' res = call_and_get_response('post', sparql_path, host, port, request_param_generator, use_ssl, data, extra_headers) try: content = res.json( ) # attempt to return json, otherwise we will return the content string. except Exception: content = res.content.decode('utf-8') return content
def get_status(host, port, use_ssl, request_param_generator=DefaultRequestGenerator()): res = call_and_get_response('get', 'status', host, port, request_param_generator, use_ssl) try: js = res.json() except JSONDecodeError: js = res.content return js
def start_processing_job(host: str, port: str, use_ssl: bool, request_param_generator, params: dict): params_raw = json.dumps(params) if type(params) is dict else params res = call_and_get_response('post', 'ml/dataprocessing', host, port, request_param_generator, use_ssl, params_raw, EXTRA_HEADERS) job = res.json() return job
def get_training_status(host: str, port: str, use_ssl: bool, request_param_generator, training_job_name: str): res = call_and_get_response('get', f'ml/modeltraining/{training_job_name}', host, port, request_param_generator, use_ssl, extra_headers=EXTRA_HEADERS) return res.json()
def get_processing_status(host: str, port: str, use_ssl: bool, request_param_generator, job_name: str): res = call_and_get_response('get', f'ml/dataprocessing/{job_name}', host, port, request_param_generator, use_ssl, extra_headers=EXTRA_HEADERS) status = res.json() return status
def do_gremlin_explain(query_str, host, port, use_ssl, request_param_generator=DefaultRequestGenerator()): data = { 'gremlin': query_str } action = 'gremlin/explain' res = call_and_get_response('get', action, host, port, request_param_generator, use_ssl, data) content = res.content.decode('utf-8') result = { 'explain': content } return result
def do_load(host, port, load_format, use_ssl, source, region, arn, fail_on_error, parallelism, update_single_cardinality, request_param_generator): payload = { 'source': source, 'format': load_format, 'region': region, 'failOnError': fail_on_error, 'parallelism': parallelism, 'updateSingleCardinalityProperties': update_single_cardinality } if arn != '': payload['iamRoleArn'] = arn res = call_and_get_response('post', LOADER_ACTION, host, port, request_param_generator, use_ssl, payload) return res.json()
def do_gremlin_status(host, port, use_ssl, mode, request_param_generator, query_id: str, include_waiting: bool): data = {'includeWaiting': include_waiting} if query_id != '': data['queryId'] = query_id headers = {} if mode == AuthModeEnum.DEFAULT: """Add correct content-type header for the request. This is needed because call_and_get_response requires custom headers to be set. """ headers['Content-Type'] = 'application/x-www-form-urlencoded' res = call_and_get_response('post', GREMLIN_STATUS_ACTION, host, port, request_param_generator, use_ssl, data, headers) content = res.json() return content
def do_gremlin_cancel(host, port, use_ssl, mode, request_param_generator, query_id): if type(query_id) != str or query_id == '': raise ValueError("query id must be a non-empty string") data = {'cancelQuery': True, 'queryId': query_id} headers = {} if mode == AuthModeEnum.DEFAULT: """Add correct content-type header for the request. This is needed because call_and_get_response requires custom headers to be set. """ headers['Content-Type'] = 'application/x-www-form-urlencoded' res = call_and_get_response('post', GREMLIN_STATUS_ACTION, host, port, request_param_generator, use_ssl, data, headers) content = res.json() return content
def get_load_status(host, port, use_ssl, request_param_generator, id, loader_details="FALSE", loader_errors="FALSE", loader_page=1, loader_epp=10): payload = { 'loadId': id, 'details': loader_details, 'errors': loader_errors, 'page': loader_page, 'errorsPerPage': loader_epp } res = call_and_get_response('get', LOADER_ACTION, host, port, request_param_generator, use_ssl, payload) return res.json()
def do_sparql_status(host, port, use_ssl, request_param_generator, query_id=None): data = {} if query_id != '' and query_id is not None: data['queryId'] = query_id headers = {'Content-Type': 'application/x-www-form-urlencoded'} res = call_and_get_response('post', SPARQL_STATUS_ACTION, host, port, request_param_generator, use_ssl, data, headers) try: content = res.json( ) # attempt to return json, otherwise we will return the content string. except Exception: """When a invalid UUID is supplied, status servlet returns an empty string. See https://sim.amazon.com/issues/NEPTUNE-16137 """ content = 'UUID is invalid.' return content
def do_sparql_cancel(host, port, use_ssl, request_param_generator, query_id, silent=False): if type(query_id) is not str or query_id == '': raise ValueError("query id must be a non-empty string") data = {'cancelQuery': True, 'queryId': query_id, 'silent': silent} headers = {'Content-Type': 'application/x-www-form-urlencoded'} res = call_and_get_response('post', SPARQL_STATUS_ACTION, host, port, request_param_generator, use_ssl, data, headers) try: content = res.json() except Exception: """When a invalid UUID is supplied, status servlet returns an empty string. See https://sim.amazon.com/issues/NEPTUNE-16137 """ content = 'UUID is invalid.' return content
def get_status(host, port, use_ssl, request_param_generator=DefaultRequestGenerator()): res = call_and_get_response('get', 'status', host, port, request_param_generator, use_ssl) return res.json()
def get_loader_jobs(host, port, use_ssl, request_param_generator): res = call_and_get_response('get', LOADER_ACTION, host, port, request_param_generator, use_ssl) return res.json()
def get_load_status(host, port, use_ssl, request_param_generator, id): payload = { 'loadId': id } res = call_and_get_response('get', LOADER_ACTION, host, port, request_param_generator, use_ssl, payload) return res.json()