def init_ps_server(settings):
tabpy = settings['tabpy']
existing_pos = tabpy.get_endpoints()
for (object_name, obj_info) in (existing_pos.items() if sys.version_info >
(3, 0) else existing_pos.iteritems()):
try:
object_version = obj_info['version']
get_query_object_path(settings['state_file_path'], object_name,
object_version)
except Exception as e:
log_error('Exception encounted when downloading object: %s'
', error: %s' % (object_name, e))
def init_ps_server(settings, tabpy_state):
logger.info("Initializing TabPy Server...")
existing_pos = tabpy_state.get_endpoints()
for (object_name, obj_info) in (existing_pos.items()):
try:
object_version = obj_info['version']
get_query_object_path(settings[SettingsParameters.StateFilePath],
object_name, object_version)
except Exception as e:
logger.error(
f'Exception encounted when downloading object: {object_name}'
f', error: {e}')
def init_ps_server(settings, tabpy_state):
logger.info("Initializing TabPy Server...")
existing_pos = tabpy_state.get_endpoints()
for (object_name, obj_info) in (existing_pos.items() if sys.version_info >
(3, 0) else existing_pos.iteritems()):
try:
object_version = obj_info['version']
get_query_object_path(settings['state_file_path'], object_name,
object_version)
except Exception as e:
logger.error('Exception encounted when downloading object: %s'
', error: %s' % (object_name, e))
def init_model_evaluator(settings, tabpy_state, python_service):
'''
This will go through all models that the service currently have and
initialize them.
'''
logger.info("Initializing models...")
existing_pos = tabpy_state.get_endpoints()
for (object_name, obj_info) in (existing_pos.items()):
object_version = obj_info['version']
object_type = obj_info['type']
object_path = get_query_object_path(
settings[SettingsParameters.StateFilePath], object_name,
object_version)
logger.info(f'Load endpoint: {object_name}, '
f'version: {object_version}, '
f'type: {object_type}')
if object_type == 'alias':
msg = LoadObject(object_name, obj_info['target'], object_version,
False, 'alias')
else:
local_path = object_path
msg = LoadObject(object_name, local_path, object_version, False,
object_type)
python_service.manage_request(msg)
def init_model_evaluator(settings):
'''
This will go through all models that the service currently have and
initialize them.
'''
tabpy = settings['tabpy']
py_handler = settings['py_handler']
existing_pos = tabpy.get_endpoints()
for (object_name, obj_info) in (existing_pos.items() if sys.version_info >
(3, 0) else existing_pos.iteritems()):
object_version = obj_info['version']
object_type = obj_info['type']
object_path = get_query_object_path(settings['state_file_path'],
object_name, object_version)
log_info('Load endpoint: %s, version: %s, type: %s' %
(object_name, object_version, object_type))
if object_type == 'alias':
msg = LoadObject(object_name, obj_info['target'], object_version,
False, 'alias')
else:
local_path = object_path
msg = LoadObject(object_name, local_path, object_version, False,
object_type)
py_handler.manage_request(msg)
def delete(self, name):
if self.should_fail_with_not_authorized():
self.fail_with_not_authorized()
return
self.logger.log(logging.DEBUG,
f'Processing DELETE for /endpoints/{name}')
try:
endpoints = self.tabpy_state.get_endpoints(name)
if len(endpoints) == 0:
self.error_out(404, f'endpoint {name} does not exist.')
self.finish()
return
# update state
try:
endpoint_info = self.tabpy_state.delete_endpoint(name)
except Exception as e:
self.error_out(400,
f'Error when removing endpoint: {e.message}')
self.finish()
return
# delete files
if endpoint_info['type'] != 'alias':
delete_path = get_query_object_path(
self.settings['state_file_path'], name, None)
try:
yield self._delete_po_future(delete_path)
except Exception as e:
self.error_out(400, f'Error while deleting: {e}')
self.finish()
return
self.set_status(204)
self.finish()
except Exception as e:
err_msg = format_exception(e, 'delete endpoint')
self.error_out(500, err_msg)
self.finish()
on_state_change(self.settings, self.tabpy_state, self.python_service,
self.logger)
def delete(self, name):
if self.should_fail_with_not_authorized():
self.fail_with_not_authorized()
return
logger.debug(
self.append_request_context(
'Processing DELETE for /endpoints/{}'.format(name)))
try:
endpoints = self.tabpy_state.get_endpoints(name)
if len(endpoints) == 0:
self.error_out(404, "endpoint %s does not exist." % name)
self.finish()
return
# update state
try:
endpoint_info = self.tabpy_state.delete_endpoint(name)
except Exception as e:
self.error_out(400,
"Error when removing endpoint: %s" % e.message)
self.finish()
return
# delete files
if endpoint_info['type'] != 'alias':
delete_path = get_query_object_path(
self.settings['state_file_path'], name, None)
try:
yield self._delete_po_future(delete_path)
except Exception as e:
self.error_out(400, "Error while deleting: %s" % e)
self.finish()
return
self.set_status(204)
self.finish()
except Exception as e:
err_msg = format_exception(e, 'delete endpoint')
self.error_out(500, err_msg)
self.finish()
on_state_change(self.settings, self.tabpy_state, self.python_service)
def _get_latest_service_state(settings, tabpy_state, new_ps_state,
python_service):
'''
Update the endpoints from the latest remote state file.
Returns
--------
(has_changes, endpoint_diff):
has_changes: True or False
endpoint_diff: Summary of what has changed, one entry for each changes
'''
# Shortcut when nothing is changed
changes = {'endpoints': {}}
# update endpoints
new_endpoints = new_ps_state.get_endpoints()
diff = {}
current_endpoints = python_service.ps.query_objects
for (endpoint_name, endpoint_info) in new_endpoints.items():
existing_endpoint = current_endpoints.get(endpoint_name)
if (existing_endpoint is None) or \
endpoint_info['version'] != existing_endpoint['version']:
# Either a new endpoint or new endpoint version
path_to_new_version = get_query_object_path(
settings[SettingsParameters.StateFilePath], endpoint_name,
endpoint_info['version'])
endpoint_type = endpoint_info.get('type', 'model')
diff[endpoint_name] = (endpoint_type, endpoint_info['version'],
path_to_new_version)
# add removed models too
for (endpoint_name, endpoint_info) in current_endpoints.items():
if endpoint_name not in new_endpoints.keys():
endpoint_type = current_endpoints[endpoint_name].get(
'type', 'model')
diff[endpoint_name] = (endpoint_type, None, None)
if diff:
changes['endpoints'] = diff
tabpy_state = new_ps_state
return (True, changes)
def delete(self, name):
try:
endpoints = self.tabpy.get_endpoints(name)
if len(endpoints) == 0:
self.error_out(404,
"endpoint %s does not exist." % name)
self.finish()
return
# update state
try:
endpoint_info = self.tabpy.delete_endpoint(name)
except Exception as e:
self.error_out(400,
"Error when removing endpoint: %s" % e.message)
self.finish()
return
# delete files
if endpoint_info['type'] != 'alias':
delete_path = get_query_object_path(
self.settings['state_file_path'], name, None)
try:
yield self._delete_po_future(delete_path)
except Exception as e:
self.error_out(400,
"Error while deleting: %s" % e)
self.finish()
return
self.set_status(204)
self.finish()
except Exception as e:
err_msg = format_exception(e, 'delete endpoint')
self.error_out(500, err_msg)
self.finish()
on_state_change(self.settings)
def _add_or_update_endpoint(self, action, name, version, request_data):
'''
Add or update an endpoint
'''
logging.debug("Adding/updating model {}...".format(name))
_name_checker = _compile('^[a-zA-Z0-9-_\\s]+$')
if not isinstance(name, (str, unicode)):
log_and_raise("Endpoint name must be a string or unicode",
TypeError)
if not _name_checker.match(name):
raise gen.Return('endpoint name can only contain: a-z, A-Z, 0-9,'
' underscore, hyphens and spaces.')
if self.settings.get('add_or_updating_endpoint'):
log_and_raise(
"Another endpoint update is already in progress"
", please wait a while and try again", RuntimeError)
request_uuid = random_uuid()
self.settings['add_or_updating_endpoint'] = request_uuid
try:
description = (request_data['description']
if 'description' in request_data else None)
if 'docstring' in request_data:
if sys.version_info > (3, 0):
docstring = str(
bytes(request_data['docstring'],
"utf-8").decode('unicode_escape'))
else:
docstring = request_data['docstring'].decode(
'string_escape')
else:
docstring = None
endpoint_type = (request_data['type']
if 'type' in request_data else None)
methods = (request_data['methods']
if 'methods' in request_data else [])
dependencies = (request_data['dependencies']
if 'dependencies' in request_data else None)
target = (request_data['target']
if 'target' in request_data else None)
schema = (request_data['schema']
if 'schema' in request_data else None)
src_path = (request_data['src_path']
if 'src_path' in request_data else None)
target_path = get_query_object_path(
self.settings[SettingsParameters.StateFilePath], name, version)
_path_checker = _compile('^[\\a-zA-Z0-9-_\\s/]+$')
# copy from staging
if src_path:
if not isinstance(request_data['src_path'], (str, unicode)):
raise gen.Return("src_path must be a string.")
if not _path_checker.match(src_path):
raise gen.Return('Endpoint name can only contain: a-z, A-'
'Z, 0-9,underscore, hyphens and spaces.')
yield self._copy_po_future(src_path, target_path)
elif endpoint_type != 'alias':
raise gen.Return("src_path is required to add/update an "
"endpoint.")
# alias special logic:
if endpoint_type == 'alias':
if not target:
raise gen.Return('Target is required for alias endpoint.')
dependencies = [target]
# update local config
try:
if action == 'add':
self.tabpy_state.add_endpoint(name=name,
description=description,
docstring=docstring,
endpoint_type=endpoint_type,
methods=methods,
dependencies=dependencies,
target=target,
schema=schema)
else:
self.tabpy_state.update_endpoint(
name=name,
description=description,
docstring=docstring,
endpoint_type=endpoint_type,
methods=methods,
dependencies=dependencies,
target=target,
schema=schema,
version=version)
except Exception as e:
raise gen.Return("Error when changing TabPy state: %s" % e)
on_state_change(self.settings, self.tabpy_state,
self.python_service)
finally:
self.settings['add_or_updating_endpoint'] = None
def _add_or_update_endpoint(self, action, name, version, request_data):
'''
Add or update an endpoint
'''
self.logger.log(logging.DEBUG, f'Adding/updating model {name}...')
_name_checker = _compile(r'^[a-zA-Z0-9-_\s]+$')
if not isinstance(name, str):
msg = 'Endpoint name must be a string'
self.logger.log(logging.CRITICAL, msg)
raise TypeError(msg)
if not _name_checker.match(name):
raise gen.Return('endpoint name can only contain: a-z, A-Z, 0-9,'
' underscore, hyphens and spaces.')
if self.settings.get('add_or_updating_endpoint'):
msg = ('Another endpoint update is already in progress'
', please wait a while and try again')
self.logger.log(logging.CRITICAL, msg)
raise RuntimeError(msg)
request_uuid = random_uuid()
self.settings['add_or_updating_endpoint'] = request_uuid
try:
description = (request_data['description']
if 'description' in request_data else None)
if 'docstring' in request_data:
docstring = str(
bytes(request_data['docstring'],
"utf-8").decode('unicode_escape'))
else:
docstring = None
endpoint_type = (request_data['type']
if 'type' in request_data else None)
methods = (request_data['methods']
if 'methods' in request_data else [])
dependencies = (request_data['dependencies']
if 'dependencies' in request_data else None)
target = (request_data['target']
if 'target' in request_data else None)
schema = (request_data['schema']
if 'schema' in request_data else None)
src_path = (request_data['src_path']
if 'src_path' in request_data else None)
target_path = get_query_object_path(
self.settings[SettingsParameters.StateFilePath], name, version)
self.logger.log(logging.DEBUG,
f'Checking source path {src_path}...')
_path_checker = _compile(r'^[\\\:a-zA-Z0-9-_~\s/\.]+$')
# copy from staging
if src_path:
if not isinstance(request_data['src_path'], str):
raise gen.Return("src_path must be a string.")
if not _path_checker.match(src_path):
raise gen.Return(
'Endpoint source path name can only contain: '
'a-z, A-Z, 0-9, underscore, hyphens and spaces.')
yield self._copy_po_future(src_path, target_path)
elif endpoint_type != 'alias':
raise gen.Return("src_path is required to add/update an "
"endpoint.")
# alias special logic:
if endpoint_type == 'alias':
if not target:
raise gen.Return('Target is required for alias endpoint.')
dependencies = [target]
# update local config
try:
if action == 'add':
self.tabpy_state.add_endpoint(name=name,
description=description,
docstring=docstring,
endpoint_type=endpoint_type,
methods=methods,
dependencies=dependencies,
target=target,
schema=schema)
else:
self.tabpy_state.update_endpoint(
name=name,
description=description,
docstring=docstring,
endpoint_type=endpoint_type,
methods=methods,
dependencies=dependencies,
target=target,
schema=schema,
version=version)
except Exception as e:
raise gen.Return(f'Error when changing TabPy state: {e}')
on_state_change(self.settings, self.tabpy_state,
self.python_service, self.logger)
finally:
self.settings['add_or_updating_endpoint'] = None
