def query(self, object_uri, params, uid):
"""Execute a QueryObject query"""
try:
if not isinstance(params, dict) and not isinstance(params, list):
return QueryFailed(
uri=object_uri,
error=("Query parameter needs to be a dictionary or a list"
". Given value is of type %s." % type(params)))
obj_info = self.query_objects.get(object_uri)
if obj_info:
pred_obj = obj_info['endpoint_obj']
version = obj_info['version']
if not pred_obj:
return QueryFailed(
uri=object_uri,
error=("There is no query object associated to the "
"endpoint: %s" % object_uri))
if isinstance(params, dict):
result = pred_obj.query(**params)
else:
result = pred_obj.query(*params)
return QuerySuccessful(object_uri, version, result)
else:
return UnknownURI(object_uri)
except Exception as e:
err_msg = format_exception(e, '/query')
log_error(err_msg)
return QueryFailed(uri=object_uri, error=err_msg)
def _load_object(self, object_uri, object_url, object_version, is_update,
object_type):
try:
log_info(msg="Loading object",
uri=object_uri, url=object_url,
version=object_version, is_update=is_update)
if object_type == 'model':
po = QueryObject.load(object_url)
elif object_type == 'alias':
po = object_url
else:
raise RuntimeError('Unknown object type: %s' % object_type)
self.query_objects[object_uri] = {'version': object_version,
'type': object_type,
'endpoint_obj': po,
'status': 'LoadSuccessful',
'last_error': None}
except Exception as e:
log_error("Unable to load QueryObject", path=object_url,
error=str(e))
self.query_objects[object_uri] = {
'version': object_version,
'type': object_type,
'endpoint_obj': None,
'status': 'LoadFailed',
'last_error': 'Load failed: %s' % str(e)}
def name(self):
'''
Returns the name of the TabPy service.
'''
name = None
try:
name = self._get_config_value(_SERVICE_INFO_SECTION_NAME, 'Name')
except Exception as e:
log_error("Unable to get name: %s" % e)
return name
def get_endpoints(self, name=None):
'''
Return a dictionary of endpoints
Parameters
----------
name : str
The name of the endpoint.
If "name" is specified, only the information about that endpoint
will be returned.
Returns
-------
endpoints : dict
The dictionary containing information about each endpoint.
The keys are the endpoint names.
The values for each include:
- description
- doc string
- type
- target
'''
endpoints = {}
try:
endpoint_names = self._get_config_value(_DEPLOYMENT_SECTION_NAME,
name)
except Exception as e:
log_error("error in get_endpoints: %s" % str(e))
return {}
if name:
endpoint_info = simplejson.loads(endpoint_names)
docstring = self._get_config_value(_QUERY_OBJECT_DOCSTRING, name)
if sys.version_info > (3, 0):
endpoint_info['docstring'] = str(
bytes(docstring, "utf-8").decode('unicode_escape'))
else:
endpoint_info['docstring'] = docstring.decode('string_escape')
endpoints = {name: endpoint_info}
else:
for endpoint_name in endpoint_names:
endpoint_info = simplejson.loads(
self._get_config_value(_DEPLOYMENT_SECTION_NAME,
endpoint_name))
docstring = self._get_config_value(_QUERY_OBJECT_DOCSTRING,
endpoint_name, True, '')
if sys.version_info > (3, 0):
endpoint_info['docstring'] = str(
bytes(docstring, "utf-8").decode('unicode_escape'))
else:
endpoint_info['docstring'] = docstring.decode(
'string_escape')
endpoints[endpoint_name] = endpoint_info
return endpoints
def get_revision_number(self):
'''
Returns the revision number of this TabPy service.
'''
rev = -1
try:
rev = int(
self._get_config_value(_META_SECTION_NAME, 'Revision Number'))
except Exception as e:
log_error("Unable to get revision number: %s" % e)
return rev
def get_description(self):
'''
Returns the description of the TabPy service.
'''
description = None
try:
description = self._get_config_value(_SERVICE_INFO_SECTION_NAME,
'Description')
except Exception as e:
log_error("Unable to get description: %s" % e)
return description
def _set_revision_number(self, revision_number):
'''
Set the revision number of this TabPy service.
'''
if not isinstance(revision_number, int):
raise ValueError("revision number must be an int.")
try:
self._set_config_value(_META_SECTION_NAME, 'Revision Number',
revision_number)
except Exception as e:
log_error("Unable to set revision number: %s" % e)
def creation_time(self):
'''
Returns the creation time of the TabPy service.
'''
creation_time = 0
try:
creation_time = self._get_config_value(_SERVICE_INFO_SECTION_NAME,
'Creation Time')
except Exception as e:
log_error("Unable to get name: %s" % e)
return creation_time
def init_ps_server(settings):
tabpy = settings['tabpy']
existing_pos = tabpy.get_endpoints()
for (object_name, obj_info) in (existing_pos.items() if sys.version_info >
(3, 0) else existing_pos.iteritems()):
try:
object_version = obj_info['version']
get_query_object_path(settings['state_file_path'], object_name,
object_version)
except Exception as e:
log_error('Exception encounted when downloading object: %s'
', error: %s' % (object_name, e))
def delete_endpoint(self, name):
'''
Delete an existing endpoint on the TabPy
Parameters
----------
name : str
The name of the endpoint to be deleted.
Returns
-------
deleted endpoint object
Note:
Cannot delete this endpoint if other endpoints are currently
depending on this endpoint.
'''
if not name or name == '':
raise ValueError("Name of the endpoint must be a valid string.")
endpoints = self.get_endpoints()
if name not in endpoints:
raise ValueError("Endpoint %s does not exist." % name)
endpoint_to_delete = endpoints[name]
# get dependencies and target
deps = set()
for endpoint_name in endpoints:
if endpoint_name != name:
deps_list = endpoints[endpoint_name].get('dependencies', [])
if name in deps_list:
deps.add(endpoint_name)
# check if other endpoints are depending on this endpoint
if len(deps) > 0:
raise ValueError("Cannot remove endpoint %s, it is currently "
"used by %s endpoints." % (name, list(deps)))
del endpoints[name]
# delete the endpoint from state
try:
self._remove_config_option(_QUERY_OBJECT_DOCSTRING,
name,
_update_revision=False)
self._remove_config_option(_DEPLOYMENT_SECTION_NAME, name)
return endpoint_to_delete
except Exception as e:
log_error("Unable to delete endpoint %s" % e)
raise ValueError("Unable to delete endpoint: %s" % e)
def set_name(self, name):
'''
Set the name of this TabPy service.
Parameters
----------
name : str
Name of TabPy service.
'''
if not isinstance(name, (str, unicode)):
raise ValueError("name must be a string.")
try:
self._set_config_value(_SERVICE_INFO_SECTION_NAME, 'Name', name)
except Exception as e:
log_error("Unable to set name: %s" % e)
def save_state_to_str(config):
'''
Convert from ConfigParser to String
'''
if not config:
raise ValueError("Invalid config")
value = None
try:
string_f = StringIO()
config.write(string_f)
value = string_f.getvalue()
except:
log_error("Cannot convert config to string")
finally:
string_f.close()
return value
def set_description(self, description):
'''
Set the description of this TabPy service.
Parameters
----------
description : str
Description of TabPy service.
'''
if not isinstance(description, (str, unicode)):
raise ValueError("Description must be a string.")
try:
self._set_config_value(_SERVICE_INFO_SECTION_NAME, 'Description',
description)
except Exception as e:
log_error("Unable to set description: %s" % e)
def load_object(self, object_uri, object_url, object_version, is_update,
object_type):
try:
obj_info = self.query_objects.get(object_uri)
if obj_info and obj_info['endpoint_obj'] and (
obj_info['version'] >= object_version):
log_info("Received load message for object already loaded")
return DownloadSkipped(
object_uri, obj_info['version'], "Object with greater "
"or equal version already loaded")
else:
if object_uri not in self.query_objects:
self.query_objects[object_uri] = {
'version': object_version,
'type': object_type,
'endpoint_obj': None,
'status': 'LoadInProgress',
'last_error': None}
else:
self.query_objects[
object_uri]['status'] = 'LoadInProgress'
self.EXECUTOR.submit(
self._load_object, object_uri, object_url,
object_version, is_update, object_type)
return LoadInProgress(
object_uri, object_url, object_version, is_update,
object_type)
except Exception as e:
log_error("Unable to load QueryObject", path=object_url,
error=str(e))
self.query_objects[object_uri] = {
'version': object_version,
'type': object_type,
'endpoint_obj': None,
'status': 'LoadFailed',
'last_error': str(e)}
return LoadFailed(object_uri, object_version, str(e))
def manage_request(self, msg):
try:
log_info("Received request", request_type=type(msg).__name__)
if isinstance(msg, LoadObject):
response = self.ps.load_object(*msg)
elif isinstance(msg, DeleteObjects):
response = self.ps.delete_objects(msg.uris)
elif isinstance(msg, FlushObjects):
response = self.ps.flush_objects()
elif isinstance(msg, CountObjects):
response = self.ps.count_objects()
elif isinstance(msg, ListObjects):
response = self.ps.list_objects()
else:
response = UnknownMessage(msg)
return response
except Exception as e:
log_error("Error processing request", error=e.message)
return UnknownMessage(e.message)
def delete_objects(self, object_uris):
"""Delete one or more objects from the query_objects map"""
if isinstance(object_uris, list):
deleted = []
for uri in object_uris:
deleted.extend(self.delete_objects(uri).uris)
return ObjectsDeleted(deleted)
elif isinstance(object_uris, str) or isinstance(object_uris, unicode):
deleted_obj = self.query_objects.pop(object_uris, None)
if deleted_obj:
return ObjectsDeleted([object_uris])
else:
log_warning("Received message to delete query object "
"that doesn't exist", object_uris=object_uris)
return ObjectsDeleted([])
else:
log_error("Unexpected input to delete objects", input=object_uris,
info="Input should be list or str. Type: %s" % type(
object_uris))
return ObjectsDeleted([])
def wait_for_endpoint_loaded(py_handler, object_uri):
'''
This method waits for the object to be loaded.
'''
log_info('Waiting for object to be loaded...')
while True:
msg = ListObjects()
list_object_msg = py_handler.manage_request(msg)
if not isinstance(list_object_msg, ObjectList):
log_error("Error loading endpoint %s: %s" %
(object_uri, list_object_msg))
return
for (uri,
info) in (list_object_msg.objects.items() if sys.version_info >
(3, 0) else list_object_msg.objects.iteritems()):
if uri == object_uri:
if info['status'] != 'LoadInProgress':
log_info("Object load status: %s" % info['status'])
return
sleep(0.1)
def _add_update_endpoints_config(self, endpoints):
# save the endpoint info to config
dstring = ''
for endpoint_name in endpoints:
try:
info = endpoints[endpoint_name]
if sys.version_info > (3, 0):
dstring = str(
bytes(info['docstring'],
"utf-8").decode('unicode_escape'))
else:
dstring = info['docstring'].decode('string_escape')
self._set_config_value(_QUERY_OBJECT_DOCSTRING,
endpoint_name,
dstring,
_update_revision=False)
del info['docstring']
self._set_config_value(_DEPLOYMENT_SECTION_NAME, endpoint_name,
simplejson.dumps(info))
except Exception as e:
log_error("Unable to write endpoints config: %s" % e)
raise
def update_endpoint(self,
name,
description=None,
docstring=None,
endpoint_type=None,
version=None,
methods=None,
target=None,
dependencies=None,
schema=None):
'''
Update an existing endpoint on the TabPy.
Parameters
----------
name : str
Name of the endpoint
description : str, optional
Description of this endpoint
doc_string : str, optional
The doc string for this endpoint, if needed.
endpoint_type : str, optional
The endpoint type (model, alias)
version : str, optional
The version of this endpoint
dependencies=[]
List of dependent endpoints for this existing endpoint
target : str, optional
The target endpoint name for the alias.
Note:
For those parameters that are not specified, those values will not
get changed.
'''
try:
endpoints = self.get_endpoints()
if not name or not isinstance(name, (str, unicode)):
raise ValueError("name of the endpoint must be string.")
elif name not in endpoints:
raise ValueError("endpoint %s does not exist." % name)
endpoint_info = endpoints[name]
if description and not isinstance(description, (str, unicode)):
raise ValueError("description must be a string.")
elif not description:
description = endpoint_info['description']
if docstring and not isinstance(docstring, (str, unicode)):
raise ValueError("docstring must be a string.")
elif not docstring:
docstring = endpoint_info['docstring']
if endpoint_type and not isinstance(endpoint_type, (str, unicode)):
raise ValueError("endpoint type must be a string.")
elif not endpoint_type:
endpoint_type = endpoint_info['type']
if version and not isinstance(version, int):
raise ValueError("version must be an int.")
elif not version:
version = endpoint_info['version']
if dependencies and not isinstance(dependencies, list):
raise ValueError("dependencies must be a list.")
elif not dependencies:
if 'dependencies' in endpoint_info:
dependencies = endpoint_info['dependencies']
else:
dependencies = []
if target and not isinstance(target, (str, unicode)):
raise ValueError("target must be a string.")
elif target and target not in endpoints:
raise ValueError("target endpoint is not valid.")
elif not target:
target = endpoint_info['target']
endpoint_info = {
'description': description,
'docstring': docstring,
'type': endpoint_type,
'version': version,
'dependencies': dependencies,
'target': target,
'creation_time': endpoint_info['creation_time'],
'last_modified_time': int(time()),
'schema': schema
}
endpoints[name] = endpoint_info
self._add_update_endpoints_config(endpoints)
except Exception as e:
log_error("Error in update_endpoint: %s" % e)
raise
def add_endpoint(self,
name,
description=None,
docstring=None,
endpoint_type=None,
methods=None,
target=None,
dependencies=None,
schema=None):
'''
Add a new endpoint to the TabPy.
Parameters
----------
name : str
Name of the endpoint
description : str, optional
Description of this endpoint
doc_string : str, optional
The doc string for this endpoint, if needed.
endpoint_type : str
The endpoint type (model, alias)
target : str, optional
The target endpoint name for the alias to be added.
Note:
The version of this endpoint will be set to 1 since it is a new
endpoint.
'''
try:
endpoints = self.get_endpoints()
if name is None or not isinstance(
name, (str, unicode)) or len(name) == 0:
raise ValueError(
"name of the endpoint must be a valid string.")
elif name in endpoints:
raise ValueError("endpoint %s already exists." % name)
if description and not isinstance(description, (str, unicode)):
raise ValueError("description must be a string.")
elif not description:
description = ''
if docstring and not isinstance(docstring, (str, unicode)):
raise ValueError("docstring must be a string.")
elif not docstring:
docstring = '-- no docstring found in query function --'
if not endpoint_type or not isinstance(endpoint_type,
(str, unicode)):
raise ValueError("endpoint type must be a string.")
if dependencies and not isinstance(dependencies, list):
raise ValueError("dependencies must be a list.")
elif not dependencies:
dependencies = []
if target and not isinstance(target, (str, unicode)):
raise ValueError("target must be a string.")
elif target and target not in endpoints:
raise ValueError("target endpoint is not valid.")
endpoint_info = {
"description": description,
"docstring": docstring,
"type": endpoint_type,
"version": 1,
"dependencies": dependencies,
"target": target,
"creation_time": int(time()),
"last_modified_time": int(time()),
"schema": schema
}
endpoints[name] = endpoint_info
self._add_update_endpoints_config(endpoints)
except Exception as e:
log_error("Error in add_endpoint: %s" % e)
raise