def _process_query(self, endpoint_name, start):
self.logger.log(logging.DEBUG,
f'Processing query {endpoint_name}...')
try:
self._add_CORS_header()
if not self.request.body:
self.request.body = {}
# extract request data explicitly for caching purpose
request_json = self.request.body.decode('utf-8')
# Sanitize input data
data = self._sanitize_request_data(json.loads(request_json))
except Exception as e:
err_msg = format_exception(e, "Invalid Input Data")
self.error_out(400, err_msg)
return
try:
(po_name, _) = self._get_actual_model(
endpoint_name)
# po_name is None if self.python_service.ps.query_objects.get(
# endpoint_name) is None
if not po_name:
self.error_out(
404,
'UnknownURI',
info=f'Endpoint "{endpoint_name}" does not exist')
return
po_obj = self.python_service.ps.query_objects.get(po_name)
if not po_obj:
self.error_out(404, 'UnknownURI',
info=f'Endpoint "{po_name}" does not exist')
return
if po_name != endpoint_name:
self.logger.log(
logging.INFO,
f'Querying actual model: po_name={po_name}')
uid = _get_uuid()
# record query w/ request ID in query log
qry = Query(po_name, request_json)
gls_time = 0
# send a query to PythonService and return
(gls_time, _) = self._handle_result(po_name, data, qry, uid)
# if error occurred, GLS time is None.
if not gls_time:
return
except Exception as e:
err_msg = format_exception(e, 'process query')
self.error_out(500, 'Error processing query', info=err_msg)
return
def _process_query(self, endpoint_name, start):
try:
self._add_CORS_header()
if not self.request.body:
self.request.body = {}
# extract request data explicitly for caching purpose
request_json = self.request.body.decode('utf-8')
# Sanitize input data
data = _sanitize_request_data(simplejson.loads(request_json))
except Exception as e:
err_msg = format_exception(e, "Invalid Input Data")
self.error_out(400, err_msg)
return
try:
(po_name,
all_endpoint_names) = self._get_actual_model(endpoint_name)
# po_name is None if self.py_handler.ps.query_objects.get(
# endpoint_name) is None
if not po_name:
log_error("UnknownURI", endpoint_name=endpoint_name)
self.error_out(404,
'UnknownURI',
info="Endpoint '%s' does not exist" %
endpoint_name)
return
po_obj = self.py_handler.ps.query_objects.get(po_name)
if not po_obj:
log_error("UnknownURI", endpoint_name=po_name)
self.error_out(404,
'UnknownURI',
info="Endpoint '%s' does not exist" % po_name)
return
if po_name != endpoint_name:
log_info("Querying actual model", po_name=po_name)
uid = _get_uuid()
# record query w/ request ID in query log
qry = Query(po_name, request_json)
gls_time = 0
# send a query to PythonService and return
(gls_time, result) = self._handle_result(po_name, data, qry, uid)
# if error occurred, GLS time is None.
if not gls_time:
return
except Exception as e:
err_msg = format_exception(e, 'process query')
self.error_out(500, 'Error processing query', info=err_msg)
return
def query(self, object_uri, params, uid):
"""Execute a QueryObject query"""
logger.debug(f'Querying Python service {object_uri}...')
try:
if not isinstance(params, dict) and not isinstance(params, list):
return QueryFailed(
uri=object_uri,
error=('Query parameter needs to be a dictionary or a list'
f'. Given value is of type {type(params)}'))
obj_info = self.query_objects.get(object_uri)
logger.debug(f'Found object {obj_info}')
if obj_info:
pred_obj = obj_info['endpoint_obj']
version = obj_info['version']
if not pred_obj:
return QueryFailed(
uri=object_uri,
error=("There is no query object associated to the "
f'endpoint: {object_uri}'))
logger.debug(f'Qurying endpoint with params ({params})...')
if isinstance(params, dict):
result = pred_obj.query(**params)
else:
result = pred_obj.query(*params)
return QuerySuccessful(object_uri, version, result)
else:
return UnknownURI(object_uri)
except Exception as e:
err_msg = format_exception(e, '/query')
logger.error(err_msg)
return QueryFailed(uri=object_uri, error=err_msg)
def query(self, object_uri, params, uid):
"""Execute a QueryObject query"""
try:
if not isinstance(params, dict) and not isinstance(params, list):
return QueryFailed(
uri=object_uri,
error=("Query parameter needs to be a dictionary or a list"
". Given value is of type %s." % type(params)))
obj_info = self.query_objects.get(object_uri)
if obj_info:
pred_obj = obj_info['endpoint_obj']
version = obj_info['version']
if not pred_obj:
return QueryFailed(
uri=object_uri,
error=("There is no query object associated to the "
"endpoint: %s" % object_uri))
if isinstance(params, dict):
result = pred_obj.query(**params)
else:
result = pred_obj.query(*params)
return QuerySuccessful(object_uri, version, result)
else:
return UnknownURI(object_uri)
except Exception as e:
err_msg = format_exception(e, '/query')
logger.error(err_msg)
return QueryFailed(uri=object_uri, error=err_msg)
def post(self):
self._add_CORS_header()
try:
body = simplejson.loads(self.request.body.decode('utf-8'))
if 'script' not in body:
self.error_out(400, 'Script is empty.')
return
# Transforming user script into a proper function.
user_code = body['script']
arguments = None
arguments_str = ''
if 'data' in body:
arguments = body['data']
if arguments is not None:
if not isinstance(arguments, dict):
self.error_out(400, 'Script parameters need to be '
'provided as a dictionary.')
return
else:
arguments_expected = []
for i in range(1, len(arguments.keys()) + 1):
arguments_expected.append('_arg' + str(i))
if sorted(arguments_expected) == sorted(arguments.keys()):
arguments_str = ', ' + ', '.join(arguments.keys())
else:
self.error_out(400, 'Variables names should follow '
'the format _arg1, _arg2, _argN')
return
function_to_evaluate = ('def _user_script(tabpy'
+ arguments_str + '):\n')
for u in user_code.splitlines():
function_to_evaluate += ' ' + u + '\n'
logger.info(
"function to evaluate=%s" % function_to_evaluate)
result = yield self.call_subprocess(function_to_evaluate,
arguments)
if result is None:
self.error_out(400, 'Error running script. No return value')
else:
self.write(simplejson.dumps(result))
self.finish()
except Exception as e:
err_msg = "%s : " % e.__class__.__name__
err_msg += "%s" % str(e)
if err_msg != "KeyError : 'response'":
err_msg = format_exception(e, 'POST /evaluate')
self.error_out(500, 'Error processing script', info=err_msg)
else:
self.error_out(
404, 'Error processing script', info="The endpoint you're "
"trying to query did not respond. Please make sure the "
"endpoint exists and the correct set of arguments are "
"provided.")
def on_state_change(settings,
tabpy_state,
python_service,
logger=logging.getLogger(__name__)):
try:
logger.log(logging.INFO, "Loading state from state file")
config = util._get_state_from_file(
settings[SettingsParameters.StateFilePath], logger=logger)
new_ps_state = TabPyState(config=config, settings=settings)
(has_changes,
changes) = _get_latest_service_state(settings, tabpy_state,
new_ps_state, python_service)
if not has_changes:
logger.info("Nothing changed, return.")
return
new_endpoints = new_ps_state.get_endpoints()
for object_name in changes['endpoints']:
(object_type, object_version,
object_path) = changes['endpoints'][object_name]
if not object_path and not object_version: # removal
logger.info(f'Removing object: URI={object_name}')
python_service.manage_request(DeleteObjects([object_name]))
cleanup_endpoint_files(object_name,
settings[SettingsParameters.UploadDir],
logger=logger)
else:
endpoint_info = new_endpoints[object_name]
is_update = object_version > 1
if object_type == 'alias':
msg = LoadObject(object_name, endpoint_info['target'],
object_version, is_update, 'alias')
else:
local_path = object_path
msg = LoadObject(object_name, local_path, object_version,
is_update, object_type)
python_service.manage_request(msg)
wait_for_endpoint_loaded(python_service, object_name)
# cleanup old version of endpoint files
if object_version > 2:
cleanup_endpoint_files(
object_name,
settings[SettingsParameters.UploadDir],
logger=logger,
retain_versions=[object_version, object_version - 1])
except Exception as e:
err_msg = format_exception(e, 'on_state_change')
logger.log(logging.ERROR,
f'Error submitting update model request: error={err_msg}')
def post(self):
if self.should_fail_with_not_authorized():
self.fail_with_not_authorized()
return
logger.debug(
self.append_request_context('Processing POST for /endpoints'))
try:
if not self.request.body:
self.error_out(400, "Input body cannot be empty")
self.finish()
return
try:
request_data = json.loads(self.request.body.decode('utf-8'))
except Exception as ex:
self.error_out(400, "Failed to decode input body", str(ex))
self.finish()
return
if 'name' not in request_data:
self.error_out(400, "name is required to add an endpoint.")
self.finish()
return
name = request_data['name']
# check if endpoint already exist
if name in self.tabpy_state.get_endpoints():
self.error_out(400, "endpoint %s already exists." % name)
self.finish()
return
logger.debug(
self.append_request_context(
"Adding endpoint '{}'".format(name)))
err_msg = yield self._add_or_update_endpoint(
'add', name, 1, request_data)
if err_msg:
self.error_out(400, err_msg)
else:
logger.debug(
self.append_request_context(
"Endpoint {} successfully added".format(name)))
self.set_status(201)
self.write(self.tabpy_state.get_endpoints(name))
self.finish()
return
except Exception as e:
err_msg = format_exception(e, '/add_endpoint')
self.error_out(500, "error adding endpoint", err_msg)
self.finish()
return
def on_state_change(settings):
try:
py_handler = settings['py_handler']
logger.info("Loading state from state file")
config = util._get_state_from_file(settings['state_file_path'])
new_ps_state = TabPyState(config=config, settings=settings)
(has_changes,
changes) = _get_latest_service_state(settings, new_ps_state)
if not has_changes:
logger.info("Nothing changed, return.")
return
new_endpoints = new_ps_state.get_endpoints()
for object_name in changes['endpoints']:
(object_type, object_version,
object_path) = changes['endpoints'][object_name]
if not object_path and not object_version: # removal
logger.info("Removing object: URI={}".format(object_name))
py_handler.manage_request(DeleteObjects([object_name]))
cleanup_endpoint_files(object_name, settings['upload_dir'])
else:
endpoint_info = new_endpoints[object_name]
is_update = object_version > 1
if object_type == 'alias':
msg = LoadObject(object_name, endpoint_info['target'],
object_version, is_update, 'alias')
else:
local_path = object_path
msg = LoadObject(object_name, local_path, object_version,
is_update, object_type)
py_handler.manage_request(msg)
wait_for_endpoint_loaded(py_handler, object_name)
# cleanup old version of endpoint files
if object_version > 2:
cleanup_endpoint_files(
object_name, settings['upload_dir'],
[object_version, object_version - 1])
except Exception as e:
err_msg = format_exception(e, 'on_state_change')
logger.error(
"Error submitting update model request: error={}".format(err_msg))
def post(self):
if self.should_fail_with_not_authorized():
self.fail_with_not_authorized()
return
try:
if not self.request.body:
self.error_out(400, "Input body cannot be empty")
self.finish()
return
try:
request_data = json.loads(self.request.body.decode('utf-8'))
except Exception as ex:
self.error_out(400, "Failed to decode input body", str(ex))
self.finish()
return
if 'name' not in request_data:
self.error_out(400, "name is required to add an endpoint.")
self.finish()
return
name = request_data['name']
# check if endpoint already exist
if name in self.tabpy_state.get_endpoints():
self.error_out(400, f'endpoint {name} already exists.')
self.finish()
return
self.logger.log(logging.DEBUG, f'Adding endpoint "{name}"')
err_msg = yield self._add_or_update_endpoint(
'add', name, 1, request_data)
if err_msg:
self.error_out(400, err_msg)
else:
self.logger.log(logging.DEBUG,
f'Endpoint {name} successfully added')
self.set_status(201)
self.write(self.tabpy_state.get_endpoints(name))
self.finish()
return
except Exception as e:
err_msg = format_exception(e, '/add_endpoint')
self.error_out(500, "error adding endpoint", err_msg)
self.finish()
return
def put(self, name):
if self.should_fail_with_not_authorized():
self.fail_with_not_authorized()
return
logger.debug(
self.append_request_context(
f'Processing PUT for /endpoints/{name}'))
try:
if not self.request.body:
self.error_out(400, "Input body cannot be empty")
self.finish()
return
try:
request_data = json.loads(self.request.body.decode('utf-8'))
except BaseException as ex:
self.error_out(400,
log_message="Failed to decode input body",
info=str(ex))
self.finish()
return
# check if endpoint exists
endpoints = self.tabpy_state.get_endpoints(name)
if len(endpoints) == 0:
self.error_out(404, "endpoint %s does not exist." % name)
self.finish()
return
new_version = int(endpoints[name]['version']) + 1
logger.info(
self.append_request_context('Endpoint info: %s' %
request_data))
err_msg = yield self._add_or_update_endpoint(
'update', name, new_version, request_data)
if err_msg:
self.error_out(400, err_msg)
self.finish()
else:
self.write(self.tabpy_state.get_endpoints(name))
self.finish()
except Exception as e:
err_msg = format_exception(e, 'update_endpoint')
self.error_out(500, err_msg)
self.finish()
def post(self):
try:
if not self.request.body:
self.error_out(400, "Input body cannot be empty")
self.finish()
return
try:
request_data = simplejson.loads(
self.request.body.decode('utf-8'))
except:
self.error_out(400, "Failed to decode input body")
self.finish()
return
if 'name' not in request_data:
self.error_out(400,
"name is required to add an endpoint.")
self.finish()
return
name = request_data['name']
# check if endpoint already exist
if name in self.tabpy.get_endpoints():
self.error_out(400, "endpoint %s already exists." % name)
self.finish()
return
logger.debug("Adding endpoint '{}'".format(name))
err_msg = yield self._add_or_update_endpoint('add', name, 1,
request_data)
if err_msg:
self.error_out(400, err_msg)
else:
logger.debug("Endopoint {} successfully added".format(name))
self.set_status(201)
self.write(self.tabpy.get_endpoints(name))
self.finish()
return
except Exception as e:
err_msg = format_exception(e, '/add_endpoint')
self.error_out(500, "error adding endpoint", err_msg)
self.finish()
return
def delete(self, name):
if self.should_fail_with_not_authorized():
self.fail_with_not_authorized()
return
self.logger.log(logging.DEBUG,
f'Processing DELETE for /endpoints/{name}')
try:
endpoints = self.tabpy_state.get_endpoints(name)
if len(endpoints) == 0:
self.error_out(404, f'endpoint {name} does not exist.')
self.finish()
return
# update state
try:
endpoint_info = self.tabpy_state.delete_endpoint(name)
except Exception as e:
self.error_out(400,
f'Error when removing endpoint: {e.message}')
self.finish()
return
# delete files
if endpoint_info['type'] != 'alias':
delete_path = get_query_object_path(
self.settings['state_file_path'], name, None)
try:
yield self._delete_po_future(delete_path)
except Exception as e:
self.error_out(400, f'Error while deleting: {e}')
self.finish()
return
self.set_status(204)
self.finish()
except Exception as e:
err_msg = format_exception(e, 'delete endpoint')
self.error_out(500, err_msg)
self.finish()
on_state_change(self.settings, self.tabpy_state, self.python_service,
self.logger)
def delete(self, name):
if self.should_fail_with_not_authorized():
self.fail_with_not_authorized()
return
logger.debug(
self.append_request_context(
'Processing DELETE for /endpoints/{}'.format(name)))
try:
endpoints = self.tabpy_state.get_endpoints(name)
if len(endpoints) == 0:
self.error_out(404, "endpoint %s does not exist." % name)
self.finish()
return
# update state
try:
endpoint_info = self.tabpy_state.delete_endpoint(name)
except Exception as e:
self.error_out(400,
"Error when removing endpoint: %s" % e.message)
self.finish()
return
# delete files
if endpoint_info['type'] != 'alias':
delete_path = get_query_object_path(
self.settings['state_file_path'], name, None)
try:
yield self._delete_po_future(delete_path)
except Exception as e:
self.error_out(400, "Error while deleting: %s" % e)
self.finish()
return
self.set_status(204)
self.finish()
except Exception as e:
err_msg = format_exception(e, 'delete endpoint')
self.error_out(500, err_msg)
self.finish()
on_state_change(self.settings, self.tabpy_state, self.python_service)
def delete(self, name):
try:
endpoints = self.tabpy.get_endpoints(name)
if len(endpoints) == 0:
self.error_out(404,
"endpoint %s does not exist." % name)
self.finish()
return
# update state
try:
endpoint_info = self.tabpy.delete_endpoint(name)
except Exception as e:
self.error_out(400,
"Error when removing endpoint: %s" % e.message)
self.finish()
return
# delete files
if endpoint_info['type'] != 'alias':
delete_path = get_query_object_path(
self.settings['state_file_path'], name, None)
try:
yield self._delete_po_future(delete_path)
except Exception as e:
self.error_out(400,
"Error while deleting: %s" % e)
self.finish()
return
self.set_status(204)
self.finish()
except Exception as e:
err_msg = format_exception(e, 'delete endpoint')
self.error_out(500, err_msg)
self.finish()
on_state_change(self.settings)
def put(self, name):
try:
if not self.request.body:
self.error_out(400, "Input body cannot be empty")
self.finish()
return
try:
request_data = simplejson.loads(
self.request.body.decode('utf-8'))
except:
self.error_out(400, "Failed to decode input body")
self.finish()
return
# check if endpoint exists
endpoints = self.tabpy.get_endpoints(name)
if len(endpoints) == 0:
self.error_out(404,
"endpoint %s does not exist." % name)
self.finish()
return
new_version = int(endpoints[name]['version']) + 1
logger.info('Endpoint info: %s' % request_data)
err_msg = yield self._add_or_update_endpoint(
'update', name, new_version, request_data)
if err_msg:
self.error_out(400, err_msg)
self.finish()
else:
self.write(self.tabpy.get_endpoints(name))
self.finish()
except Exception as e:
err_msg = format_exception(e, 'update_endpoint')
self.error_out(500, err_msg)
self.finish()
def post(self):
if self.should_fail_with_not_authorized():
self.fail_with_not_authorized()
return
self._add_CORS_header()
try:
body = json.loads(self.request.body.decode('utf-8'))
if 'script' not in body:
self.error_out(400, 'Script is empty.')
return
# Transforming user script into a proper function.
user_code = body['script']
arguments = None
arguments_str = ''
if 'data' in body:
arguments = body['data']
if arguments is not None:
if not isinstance(arguments, dict):
self.error_out(
400, 'Script parameters need to be '
'provided as a dictionary.')
return
else:
arguments_expected = []
for i in range(1, len(arguments.keys()) + 1):
arguments_expected.append('_arg' + str(i))
if sorted(arguments_expected) == sorted(arguments.keys()):
arguments_str = ', ' + ', '.join(arguments.keys())
else:
self.error_out(
400, 'Variables names should follow '
'the format _arg1, _arg2, _argN')
return
function_to_evaluate = f'def _user_script(tabpy{arguments_str}):\n'
for u in user_code.splitlines():
function_to_evaluate += ' ' + u + '\n'
self.logger.log(logging.INFO,
f'function to evaluate={function_to_evaluate}')
try:
result = yield self._call_subprocess(function_to_evaluate,
arguments)
except (gen.TimeoutError, requests.exceptions.ConnectTimeout,
requests.exceptions.ReadTimeout):
self.logger.log(logging.ERROR, self._error_message_timeout)
self.error_out(408, self._error_message_timeout)
return
if result is None:
self.error_out(400, 'Error running script. No return value')
else:
self.write(json.dumps(result))
self.finish()
except Exception as e:
err_msg = f'{e.__class__.__name__} : {str(e)}'
if err_msg != "KeyError : 'response'":
err_msg = format_exception(e, 'POST /evaluate')
self.error_out(500, 'Error processing script', info=err_msg)
else:
self.error_out(
404,
'Error processing script',
info="The endpoint you're "
"trying to query did not respond. Please make sure the "
"endpoint exists and the correct set of arguments are "
"provided.")