def start_training(host: str, port: str, use_ssl: bool,
request_param_generator, params):
params_raw = json.dumps(params) if type(params) is dict else params
res = call_and_get_response('post', 'ml/modeltraining', host, port,
request_param_generator, use_ssl, params_raw,
EXTRA_HEADERS)
return res.json()
def cancel_load(host, port, use_ssl, request_param_generator, load_id):
payload = {
'loadId': load_id
}
res = call_and_get_response('get', LOADER_ACTION, host, port, request_param_generator, use_ssl, payload)
return res.status_code == 200
def do_sparql_explain(query: str,
host: str,
port: str,
use_ssl: bool,
request_param_generator,
accept_type='text/html',
path_prefix: str = ''):
query_type = get_query_type(query)
action = query_type_to_action(query_type)
data = {
'explain': 'dynamic',
}
if action == 'sparql':
data['query'] = query
elif action == 'sparqlupdate':
data['update'] = query
extra_headers = {'Accept': accept_type}
sparql_path = SPARQL_ACTION if path_prefix == '' else f'{path_prefix}/{SPARQL_ACTION}'
res = call_and_get_response('post', sparql_path, host, port,
request_param_generator, use_ssl, data,
extra_headers)
return res.content.decode('utf-8')
def do_load(host,
port,
load_format,
use_ssl,
source,
region,
arn,
fail_on_error,
request_param_generator,
mode="AUTO",
parallelism="HIGH",
update_single_cardinality="FALSE",
queue_request="FALSE",
dependencies=[]):
payload = {
'source': source,
'format': load_format,
'mode': mode,
'region': region,
'failOnError': fail_on_error,
'parallelism': parallelism,
'updateSingleCardinalityProperties': update_single_cardinality,
'queueRequest': queue_request
}
if arn != '':
payload['iamRoleArn'] = arn
if dependencies:
payload['dependencies'] = json.dumps(dependencies)
res = call_and_get_response('post', LOADER_ACTION, host, port,
request_param_generator, use_ssl, payload)
return res.json()
def do_sparql_query(query,
host,
port,
use_ssl,
request_param_generator,
extra_headers=None,
path_prefix: str = ''):
if extra_headers is None:
extra_headers = {}
logger.debug(f'query={query}, endpoint={host}, port={port}')
query_type = get_query_type(query)
action = query_type_to_action(query_type)
data = {}
if action == 'sparql':
data['query'] = query
elif action == 'sparqlupdate':
data['update'] = query
sparql_path = SPARQL_ACTION if path_prefix == '' else f'{path_prefix}/{SPARQL_ACTION}'
res = call_and_get_response('post', sparql_path, host, port,
request_param_generator, use_ssl, data,
extra_headers)
try:
content = res.json(
) # attempt to return json, otherwise we will return the content string.
except Exception:
content = res.content.decode('utf-8')
return content
def get_status(host, port, use_ssl, request_param_generator=DefaultRequestGenerator()):
res = call_and_get_response('get', 'status', host, port, request_param_generator, use_ssl)
try:
js = res.json()
except JSONDecodeError:
js = res.content
return js
def start_processing_job(host: str, port: str, use_ssl: bool,
request_param_generator, params: dict):
params_raw = json.dumps(params) if type(params) is dict else params
res = call_and_get_response('post', 'ml/dataprocessing', host, port,
request_param_generator, use_ssl, params_raw,
EXTRA_HEADERS)
job = res.json()
return job
def get_training_status(host: str, port: str, use_ssl: bool,
request_param_generator, training_job_name: str):
res = call_and_get_response('get',
f'ml/modeltraining/{training_job_name}',
host,
port,
request_param_generator,
use_ssl,
extra_headers=EXTRA_HEADERS)
return res.json()
def get_processing_status(host: str, port: str, use_ssl: bool,
request_param_generator, job_name: str):
res = call_and_get_response('get',
f'ml/dataprocessing/{job_name}',
host,
port,
request_param_generator,
use_ssl,
extra_headers=EXTRA_HEADERS)
status = res.json()
return status
def do_gremlin_explain(query_str, host, port, use_ssl, request_param_generator=DefaultRequestGenerator()):
data = {
'gremlin': query_str
}
action = 'gremlin/explain'
res = call_and_get_response('get', action, host, port, request_param_generator, use_ssl, data)
content = res.content.decode('utf-8')
result = {
'explain': content
}
return result
def do_load(host, port, load_format, use_ssl, source, region, arn, fail_on_error, parallelism, update_single_cardinality, request_param_generator):
payload = {
'source': source,
'format': load_format,
'region': region,
'failOnError': fail_on_error,
'parallelism': parallelism,
'updateSingleCardinalityProperties': update_single_cardinality
}
if arn != '':
payload['iamRoleArn'] = arn
res = call_and_get_response('post', LOADER_ACTION, host, port, request_param_generator, use_ssl, payload)
return res.json()
def do_gremlin_status(host, port, use_ssl, mode, request_param_generator,
query_id: str, include_waiting: bool):
data = {'includeWaiting': include_waiting}
if query_id != '':
data['queryId'] = query_id
headers = {}
if mode == AuthModeEnum.DEFAULT:
"""Add correct content-type header for the request.
This is needed because call_and_get_response requires custom headers to be set.
"""
headers['Content-Type'] = 'application/x-www-form-urlencoded'
res = call_and_get_response('post', GREMLIN_STATUS_ACTION, host, port,
request_param_generator, use_ssl, data,
headers)
content = res.json()
return content
def do_gremlin_cancel(host, port, use_ssl, mode, request_param_generator,
query_id):
if type(query_id) != str or query_id == '':
raise ValueError("query id must be a non-empty string")
data = {'cancelQuery': True, 'queryId': query_id}
headers = {}
if mode == AuthModeEnum.DEFAULT:
"""Add correct content-type header for the request.
This is needed because call_and_get_response requires custom headers to be set.
"""
headers['Content-Type'] = 'application/x-www-form-urlencoded'
res = call_and_get_response('post', GREMLIN_STATUS_ACTION, host, port,
request_param_generator, use_ssl, data,
headers)
content = res.json()
return content
def get_load_status(host,
port,
use_ssl,
request_param_generator,
id,
loader_details="FALSE",
loader_errors="FALSE",
loader_page=1,
loader_epp=10):
payload = {
'loadId': id,
'details': loader_details,
'errors': loader_errors,
'page': loader_page,
'errorsPerPage': loader_epp
}
res = call_and_get_response('get', LOADER_ACTION, host, port,
request_param_generator, use_ssl, payload)
return res.json()
def do_sparql_status(host,
port,
use_ssl,
request_param_generator,
query_id=None):
data = {}
if query_id != '' and query_id is not None:
data['queryId'] = query_id
headers = {'Content-Type': 'application/x-www-form-urlencoded'}
res = call_and_get_response('post', SPARQL_STATUS_ACTION, host, port,
request_param_generator, use_ssl, data,
headers)
try:
content = res.json(
) # attempt to return json, otherwise we will return the content string.
except Exception:
"""When a invalid UUID is supplied, status servlet returns an empty string.
See https://sim.amazon.com/issues/NEPTUNE-16137
"""
content = 'UUID is invalid.'
return content
def do_sparql_cancel(host,
port,
use_ssl,
request_param_generator,
query_id,
silent=False):
if type(query_id) is not str or query_id == '':
raise ValueError("query id must be a non-empty string")
data = {'cancelQuery': True, 'queryId': query_id, 'silent': silent}
headers = {'Content-Type': 'application/x-www-form-urlencoded'}
res = call_and_get_response('post', SPARQL_STATUS_ACTION, host, port,
request_param_generator, use_ssl, data,
headers)
try:
content = res.json()
except Exception:
"""When a invalid UUID is supplied, status servlet returns an empty string.
See https://sim.amazon.com/issues/NEPTUNE-16137
"""
content = 'UUID is invalid.'
return content
def get_status(host, port, use_ssl, request_param_generator=DefaultRequestGenerator()):
res = call_and_get_response('get', 'status', host, port, request_param_generator, use_ssl)
return res.json()
def get_loader_jobs(host, port, use_ssl, request_param_generator):
res = call_and_get_response('get', LOADER_ACTION, host, port,
request_param_generator, use_ssl)
return res.json()
def get_load_status(host, port, use_ssl, request_param_generator, id):
payload = {
'loadId': id
}
res = call_and_get_response('get', LOADER_ACTION, host, port, request_param_generator, use_ssl, payload)
return res.json()
