def query(self, object_uri, params, uid): """Execute a QueryObject query""" try: if not isinstance(params, dict) and not isinstance(params, list): return QueryFailed( uri=object_uri, error=("Query parameter needs to be a dictionary or a list" ". Given value is of type %s." % type(params))) obj_info = self.query_objects.get(object_uri) if obj_info: pred_obj = obj_info['endpoint_obj'] version = obj_info['version'] if not pred_obj: return QueryFailed( uri=object_uri, error=("There is no query object associated to the " "endpoint: %s" % object_uri)) if isinstance(params, dict): result = pred_obj.query(**params) else: result = pred_obj.query(*params) return QuerySuccessful(object_uri, version, result) else: return UnknownURI(object_uri) except Exception as e: err_msg = format_exception(e, '/query') log_error(err_msg) return QueryFailed(uri=object_uri, error=err_msg)
def _load_object(self, object_uri, object_url, object_version, is_update, object_type): try: log_info(msg="Loading object", uri=object_uri, url=object_url, version=object_version, is_update=is_update) if object_type == 'model': po = QueryObject.load(object_url) elif object_type == 'alias': po = object_url else: raise RuntimeError('Unknown object type: %s' % object_type) self.query_objects[object_uri] = {'version': object_version, 'type': object_type, 'endpoint_obj': po, 'status': 'LoadSuccessful', 'last_error': None} except Exception as e: log_error("Unable to load QueryObject", path=object_url, error=str(e)) self.query_objects[object_uri] = { 'version': object_version, 'type': object_type, 'endpoint_obj': None, 'status': 'LoadFailed', 'last_error': 'Load failed: %s' % str(e)}
def name(self): ''' Returns the name of the TabPy service. ''' name = None try: name = self._get_config_value(_SERVICE_INFO_SECTION_NAME, 'Name') except Exception as e: log_error("Unable to get name: %s" % e) return name
def get_endpoints(self, name=None): ''' Return a dictionary of endpoints Parameters ---------- name : str The name of the endpoint. If "name" is specified, only the information about that endpoint will be returned. Returns ------- endpoints : dict The dictionary containing information about each endpoint. The keys are the endpoint names. The values for each include: - description - doc string - type - target ''' endpoints = {} try: endpoint_names = self._get_config_value(_DEPLOYMENT_SECTION_NAME, name) except Exception as e: log_error("error in get_endpoints: %s" % str(e)) return {} if name: endpoint_info = simplejson.loads(endpoint_names) docstring = self._get_config_value(_QUERY_OBJECT_DOCSTRING, name) if sys.version_info > (3, 0): endpoint_info['docstring'] = str( bytes(docstring, "utf-8").decode('unicode_escape')) else: endpoint_info['docstring'] = docstring.decode('string_escape') endpoints = {name: endpoint_info} else: for endpoint_name in endpoint_names: endpoint_info = simplejson.loads( self._get_config_value(_DEPLOYMENT_SECTION_NAME, endpoint_name)) docstring = self._get_config_value(_QUERY_OBJECT_DOCSTRING, endpoint_name, True, '') if sys.version_info > (3, 0): endpoint_info['docstring'] = str( bytes(docstring, "utf-8").decode('unicode_escape')) else: endpoint_info['docstring'] = docstring.decode( 'string_escape') endpoints[endpoint_name] = endpoint_info return endpoints
def get_revision_number(self): ''' Returns the revision number of this TabPy service. ''' rev = -1 try: rev = int( self._get_config_value(_META_SECTION_NAME, 'Revision Number')) except Exception as e: log_error("Unable to get revision number: %s" % e) return rev
def get_description(self): ''' Returns the description of the TabPy service. ''' description = None try: description = self._get_config_value(_SERVICE_INFO_SECTION_NAME, 'Description') except Exception as e: log_error("Unable to get description: %s" % e) return description
def _set_revision_number(self, revision_number): ''' Set the revision number of this TabPy service. ''' if not isinstance(revision_number, int): raise ValueError("revision number must be an int.") try: self._set_config_value(_META_SECTION_NAME, 'Revision Number', revision_number) except Exception as e: log_error("Unable to set revision number: %s" % e)
def creation_time(self): ''' Returns the creation time of the TabPy service. ''' creation_time = 0 try: creation_time = self._get_config_value(_SERVICE_INFO_SECTION_NAME, 'Creation Time') except Exception as e: log_error("Unable to get name: %s" % e) return creation_time
def init_ps_server(settings): tabpy = settings['tabpy'] existing_pos = tabpy.get_endpoints() for (object_name, obj_info) in (existing_pos.items() if sys.version_info > (3, 0) else existing_pos.iteritems()): try: object_version = obj_info['version'] get_query_object_path(settings['state_file_path'], object_name, object_version) except Exception as e: log_error('Exception encounted when downloading object: %s' ', error: %s' % (object_name, e))
def delete_endpoint(self, name): ''' Delete an existing endpoint on the TabPy Parameters ---------- name : str The name of the endpoint to be deleted. Returns ------- deleted endpoint object Note: Cannot delete this endpoint if other endpoints are currently depending on this endpoint. ''' if not name or name == '': raise ValueError("Name of the endpoint must be a valid string.") endpoints = self.get_endpoints() if name not in endpoints: raise ValueError("Endpoint %s does not exist." % name) endpoint_to_delete = endpoints[name] # get dependencies and target deps = set() for endpoint_name in endpoints: if endpoint_name != name: deps_list = endpoints[endpoint_name].get('dependencies', []) if name in deps_list: deps.add(endpoint_name) # check if other endpoints are depending on this endpoint if len(deps) > 0: raise ValueError("Cannot remove endpoint %s, it is currently " "used by %s endpoints." % (name, list(deps))) del endpoints[name] # delete the endpoint from state try: self._remove_config_option(_QUERY_OBJECT_DOCSTRING, name, _update_revision=False) self._remove_config_option(_DEPLOYMENT_SECTION_NAME, name) return endpoint_to_delete except Exception as e: log_error("Unable to delete endpoint %s" % e) raise ValueError("Unable to delete endpoint: %s" % e)
def set_name(self, name): ''' Set the name of this TabPy service. Parameters ---------- name : str Name of TabPy service. ''' if not isinstance(name, (str, unicode)): raise ValueError("name must be a string.") try: self._set_config_value(_SERVICE_INFO_SECTION_NAME, 'Name', name) except Exception as e: log_error("Unable to set name: %s" % e)
def save_state_to_str(config): ''' Convert from ConfigParser to String ''' if not config: raise ValueError("Invalid config") value = None try: string_f = StringIO() config.write(string_f) value = string_f.getvalue() except: log_error("Cannot convert config to string") finally: string_f.close() return value
def set_description(self, description): ''' Set the description of this TabPy service. Parameters ---------- description : str Description of TabPy service. ''' if not isinstance(description, (str, unicode)): raise ValueError("Description must be a string.") try: self._set_config_value(_SERVICE_INFO_SECTION_NAME, 'Description', description) except Exception as e: log_error("Unable to set description: %s" % e)
def load_object(self, object_uri, object_url, object_version, is_update, object_type): try: obj_info = self.query_objects.get(object_uri) if obj_info and obj_info['endpoint_obj'] and ( obj_info['version'] >= object_version): log_info("Received load message for object already loaded") return DownloadSkipped( object_uri, obj_info['version'], "Object with greater " "or equal version already loaded") else: if object_uri not in self.query_objects: self.query_objects[object_uri] = { 'version': object_version, 'type': object_type, 'endpoint_obj': None, 'status': 'LoadInProgress', 'last_error': None} else: self.query_objects[ object_uri]['status'] = 'LoadInProgress' self.EXECUTOR.submit( self._load_object, object_uri, object_url, object_version, is_update, object_type) return LoadInProgress( object_uri, object_url, object_version, is_update, object_type) except Exception as e: log_error("Unable to load QueryObject", path=object_url, error=str(e)) self.query_objects[object_uri] = { 'version': object_version, 'type': object_type, 'endpoint_obj': None, 'status': 'LoadFailed', 'last_error': str(e)} return LoadFailed(object_uri, object_version, str(e))
def manage_request(self, msg): try: log_info("Received request", request_type=type(msg).__name__) if isinstance(msg, LoadObject): response = self.ps.load_object(*msg) elif isinstance(msg, DeleteObjects): response = self.ps.delete_objects(msg.uris) elif isinstance(msg, FlushObjects): response = self.ps.flush_objects() elif isinstance(msg, CountObjects): response = self.ps.count_objects() elif isinstance(msg, ListObjects): response = self.ps.list_objects() else: response = UnknownMessage(msg) return response except Exception as e: log_error("Error processing request", error=e.message) return UnknownMessage(e.message)
def delete_objects(self, object_uris): """Delete one or more objects from the query_objects map""" if isinstance(object_uris, list): deleted = [] for uri in object_uris: deleted.extend(self.delete_objects(uri).uris) return ObjectsDeleted(deleted) elif isinstance(object_uris, str) or isinstance(object_uris, unicode): deleted_obj = self.query_objects.pop(object_uris, None) if deleted_obj: return ObjectsDeleted([object_uris]) else: log_warning("Received message to delete query object " "that doesn't exist", object_uris=object_uris) return ObjectsDeleted([]) else: log_error("Unexpected input to delete objects", input=object_uris, info="Input should be list or str. Type: %s" % type( object_uris)) return ObjectsDeleted([])
def wait_for_endpoint_loaded(py_handler, object_uri): ''' This method waits for the object to be loaded. ''' log_info('Waiting for object to be loaded...') while True: msg = ListObjects() list_object_msg = py_handler.manage_request(msg) if not isinstance(list_object_msg, ObjectList): log_error("Error loading endpoint %s: %s" % (object_uri, list_object_msg)) return for (uri, info) in (list_object_msg.objects.items() if sys.version_info > (3, 0) else list_object_msg.objects.iteritems()): if uri == object_uri: if info['status'] != 'LoadInProgress': log_info("Object load status: %s" % info['status']) return sleep(0.1)
def _add_update_endpoints_config(self, endpoints): # save the endpoint info to config dstring = '' for endpoint_name in endpoints: try: info = endpoints[endpoint_name] if sys.version_info > (3, 0): dstring = str( bytes(info['docstring'], "utf-8").decode('unicode_escape')) else: dstring = info['docstring'].decode('string_escape') self._set_config_value(_QUERY_OBJECT_DOCSTRING, endpoint_name, dstring, _update_revision=False) del info['docstring'] self._set_config_value(_DEPLOYMENT_SECTION_NAME, endpoint_name, simplejson.dumps(info)) except Exception as e: log_error("Unable to write endpoints config: %s" % e) raise
def update_endpoint(self, name, description=None, docstring=None, endpoint_type=None, version=None, methods=None, target=None, dependencies=None, schema=None): ''' Update an existing endpoint on the TabPy. Parameters ---------- name : str Name of the endpoint description : str, optional Description of this endpoint doc_string : str, optional The doc string for this endpoint, if needed. endpoint_type : str, optional The endpoint type (model, alias) version : str, optional The version of this endpoint dependencies=[] List of dependent endpoints for this existing endpoint target : str, optional The target endpoint name for the alias. Note: For those parameters that are not specified, those values will not get changed. ''' try: endpoints = self.get_endpoints() if not name or not isinstance(name, (str, unicode)): raise ValueError("name of the endpoint must be string.") elif name not in endpoints: raise ValueError("endpoint %s does not exist." % name) endpoint_info = endpoints[name] if description and not isinstance(description, (str, unicode)): raise ValueError("description must be a string.") elif not description: description = endpoint_info['description'] if docstring and not isinstance(docstring, (str, unicode)): raise ValueError("docstring must be a string.") elif not docstring: docstring = endpoint_info['docstring'] if endpoint_type and not isinstance(endpoint_type, (str, unicode)): raise ValueError("endpoint type must be a string.") elif not endpoint_type: endpoint_type = endpoint_info['type'] if version and not isinstance(version, int): raise ValueError("version must be an int.") elif not version: version = endpoint_info['version'] if dependencies and not isinstance(dependencies, list): raise ValueError("dependencies must be a list.") elif not dependencies: if 'dependencies' in endpoint_info: dependencies = endpoint_info['dependencies'] else: dependencies = [] if target and not isinstance(target, (str, unicode)): raise ValueError("target must be a string.") elif target and target not in endpoints: raise ValueError("target endpoint is not valid.") elif not target: target = endpoint_info['target'] endpoint_info = { 'description': description, 'docstring': docstring, 'type': endpoint_type, 'version': version, 'dependencies': dependencies, 'target': target, 'creation_time': endpoint_info['creation_time'], 'last_modified_time': int(time()), 'schema': schema } endpoints[name] = endpoint_info self._add_update_endpoints_config(endpoints) except Exception as e: log_error("Error in update_endpoint: %s" % e) raise
def add_endpoint(self, name, description=None, docstring=None, endpoint_type=None, methods=None, target=None, dependencies=None, schema=None): ''' Add a new endpoint to the TabPy. Parameters ---------- name : str Name of the endpoint description : str, optional Description of this endpoint doc_string : str, optional The doc string for this endpoint, if needed. endpoint_type : str The endpoint type (model, alias) target : str, optional The target endpoint name for the alias to be added. Note: The version of this endpoint will be set to 1 since it is a new endpoint. ''' try: endpoints = self.get_endpoints() if name is None or not isinstance( name, (str, unicode)) or len(name) == 0: raise ValueError( "name of the endpoint must be a valid string.") elif name in endpoints: raise ValueError("endpoint %s already exists." % name) if description and not isinstance(description, (str, unicode)): raise ValueError("description must be a string.") elif not description: description = '' if docstring and not isinstance(docstring, (str, unicode)): raise ValueError("docstring must be a string.") elif not docstring: docstring = '-- no docstring found in query function --' if not endpoint_type or not isinstance(endpoint_type, (str, unicode)): raise ValueError("endpoint type must be a string.") if dependencies and not isinstance(dependencies, list): raise ValueError("dependencies must be a list.") elif not dependencies: dependencies = [] if target and not isinstance(target, (str, unicode)): raise ValueError("target must be a string.") elif target and target not in endpoints: raise ValueError("target endpoint is not valid.") endpoint_info = { "description": description, "docstring": docstring, "type": endpoint_type, "version": 1, "dependencies": dependencies, "target": target, "creation_time": int(time()), "last_modified_time": int(time()), "schema": schema } endpoints[name] = endpoint_info self._add_update_endpoints_config(endpoints) except Exception as e: log_error("Error in add_endpoint: %s" % e) raise