def on_init(self): self.resource_interface = (Config( ["res/config/resource_management.yml"])).data['ResourceInterface'] self._augment_resource_interface_from_interfaces() # Keep a cache of known resource ids self.restype_cache = {} self.ds_discovery = DatastoreDiscovery(self)
def on_init(self): self.resource_interface = (Config(["res/config/resource_management.yml"])).data['ResourceInterface'] self._augment_resource_interface_from_interfaces() # Keep a cache of known resource ids self.restype_cache = {} self.ds_discovery = DatastoreDiscovery(self)
class ResourceManagementService(BaseResourceManagementService): """ Service that manages resource types and lifecycle workflows. It also provides generic operations that manage any kind of resource and their lifecycle. Also provides a resource discovery and query capability. """ MAX_SEARCH_RESULTS = CFG.get_safe( 'service.resource_management.max_search_results', 250) def on_init(self): self.resource_interface = (Config( ["res/config/resource_management.yml"])).data['ResourceInterface'] self._augment_resource_interface_from_interfaces() # Keep a cache of known resource ids self.restype_cache = {} self.ds_discovery = DatastoreDiscovery(self) # ------------------------------------------------------------------------- # Search and query def query(self, query=None, id_only=True, search_args=None): """Issue a query provided in structured dict format or internal datastore query format. Returns a list of resource or event objects or their IDs only. Search_args may contain parameterized values. See the query format definition: https://confluence.oceanobservatories.org/display/CIDev/Discovery+Service+Query+Format """ if not query: raise BadRequest("Invalid query") return self._discovery_request(query, id_only, search_args=search_args, query_params=search_args) def query_view(self, view_id='', view_name='', ext_query=None, id_only=True, search_args=None): """Execute an existing query as defined within a View resource, providing additional arguments for parameterized values. If ext_query is provided, it will be combined with the query defined by the View. Search_args may contain parameterized values. Returns a list of resource or event objects or their IDs only. """ if not view_id and not view_name: raise BadRequest("Must provide argument view_id or view_name") if view_id and view_name: raise BadRequest( "Cannot provide both arguments view_id and view_name") if view_id: view_obj = self.clients.resource_registry.read(view_id) else: view_obj = self.ds_discovery.get_builtin_view(view_name) if not view_obj: view_objs, _ = self.clients.resource_registry.find_resources( restype=RT.View, name=view_name) if not view_objs: raise NotFound("View with name '%s' not found" % view_name) view_obj = view_objs[0] if view_obj.type_ != RT.View: raise BadRequest("Argument view_id is not a View resource") view_query = view_obj.view_definition if not QUERY_EXP_KEY in view_query: raise BadRequest("Unknown View query format") # Get default query params and override them with provided args param_defaults = { param.name: param.default for param in view_obj.view_parameters } query_params = param_defaults if view_obj.param_values: query_params.update(view_obj.param_values) if search_args: query_params.update(search_args) # Merge ext_query into query if ext_query: if ext_query["where"] and view_query["where"]: view_query["where"] = [ DQ.EXP_AND, [view_query["where"], ext_query["where"]] ] else: view_query["where"] = view_query["where"] or ext_query["where"] if ext_query["order_by"]: # Override ordering if present view_query["where"] = ext_query["order_by"] # Other query settings view_qargs = view_query["query_args"] ext_qargs = ext_query["query_args"] view_qargs["id_only"] = ext_qargs.get("id_only", view_qargs["id_only"]) view_qargs["limit"] = ext_qargs.get("limit", view_qargs["limit"]) view_qargs["skip"] = ext_qargs.get("skip", view_qargs["skip"]) return self._discovery_request(view_query, id_only=id_only, search_args=search_args, query_params=query_params) def _discovery_request(self, query=None, id_only=True, search_args=None, query_params=None): search_args = search_args or {} if not query: raise BadRequest('No request query provided') if QUERY_EXP_KEY in query and self.ds_discovery: query.setdefault("query_args", {})["id_only"] = id_only # Query in datastore query format (dict) log.debug("Executing datastore query: %s", query) elif 'query' not in query: raise BadRequest('Unsupported request. %s' % query) # if count requested, run id_only query without limit/skip count = search_args.get("count", False) if count: # Only return the count of ID only search query.pop("limit", None) query.pop("skip", None) res = self.ds_discovery.execute_query(query, id_only=True, query_args=search_args, query_params=query_params) return [len(res)] # TODO: Not all queries are permissible by all users # Execute the query query_results = self.ds_discovery.execute_query( query, id_only=id_only, query_args=search_args, query_params=query_params) # Strip out unwanted object attributes for size filtered_res = self._strip_query_results(query_results, id_only=id_only, search_args=search_args) return filtered_res def _strip_query_results(self, query_results, id_only, search_args): # Filter the results for smaller result size attr_filter = search_args.get("attribute_filter", []) if type(attr_filter) not in (list, tuple): raise BadRequest("Illegal argument type: attribute_filter") if not id_only and attr_filter: filtered_res = [ dict(__noion__=True, **{ k: v for k, v in obj.__dict__.iteritems() if k in attr_filter or k in {"_id", "type_"} }) for obj in query_results ] return filtered_res return query_results # ------------------------------------------------------------------------- # View management (CRUD) def create_view(self, view=None): if view is None or not isinstance(view, View): raise BadRequest("Illegal argument: view") # view_objs, _ = self.clients.resource_registry.find_resources(restype=RT.View, name=view.name) # if view_objs: # raise BadRequest("View with name '%s' already exists" % view.name) view_id, _ = self.clients.resource_registry.create(view) return view_id def read_view(self, view_id=''): view_res = self.clients.resource_registry.read(view_id) if not isinstance(view_res, View): raise BadRequest("Resource %s is not a View" % view_id) return view_res def update_view(self, view=None): if view is None or not isinstance(view, View): raise BadRequest("Illegal argument: view") self.clients.resource_registry.update(view) return True def delete_view(self, view_id=''): self.clients.resource_registry.delete(view_id) return True # ------------------------------------------------------------------------- # Generic resource interface def create_resource(self, resource=None): """Creates an arbitrary resource object via its defined create function, so that it can successively can be accessed via the agent interface. """ if not isinstance(resource, Resource): raise BadRequest("Can only create resources, not type %s" % type(resource)) res_type = resource._get_type() res_interface = self._get_type_interface(res_type) if not 'create' in res_interface: raise BadRequest("Resource type %s does not support: CREATE" % res_type) res = self._call_crud(res_interface['create'], resource, None, res_type) if type(res) in (list, tuple): res = res[0] return res def update_resource(self, resource=None): """Updates an existing resource via the configured service operation. """ if not isinstance(resource, Resource): raise BadRequest("Can only update resources, not type %s" % type(resource)) res_type = resource._get_type() res_interface = self._get_type_interface(res_type) if not 'update' in res_interface: raise BadRequest("Resource type %s does not support: UPDATE" % res_type) self._call_crud(res_interface['update'], resource, None, res_type) def read_resource(self, resource_id=''): """Returns an existing resource via the configured service operation. """ res_type = self._get_resource_type(resource_id) res_interface = self._get_type_interface(res_type) if not 'read' in res_interface: raise BadRequest("Resource type %s does not support: READ" % res_type) res_obj = self._call_crud(res_interface['read'], None, resource_id, res_type) return res_obj def delete_resource(self, resource_id=''): """Deletes an existing resource via the configured service operation. """ res_type = self._get_resource_type(resource_id) res_interface = self._get_type_interface(res_type) if not 'delete' in res_interface: raise BadRequest("Resource type %s does not support: DELETE" % res_type) self._call_crud(res_interface['delete'], None, resource_id, res_type) CORE_ATTRIBUTES = { "_id", "name", "description", "ts_created", "ts_updated", "lcstate", "availability", "visibility", "alt_resource_type" } def get_org_resource_attributes(self, org_id='', order_by='', type_filter=None, limit=0, skip=0): """For a given org, return a list of dicts with core resource attributes (_id, type_, name, description, ts_created, ts_modified, lcstate, availability, visibility and alt_resource_type). The returned list is ordered by name unless otherwise specified. Supports pagination and white-list filtering if provided. """ if not org_id: raise BadRequest("Must provide org_id") res_list = [] res_objs, _ = self.clients.resource_registry.find_objects( org_id, PRED.hasResource, id_only=False) res_list.extend(res_objs) # TODO: The following is not correct - this should be all attachments in the Org res_objs, _ = self.clients.resource_registry.find_objects( org_id, PRED.hasAttachment, id_only=False) res_list.extend(res_objs) # TODO: The following should be shared in the Org res_objs, _ = self.clients.resource_registry.find_objects( org_id, PRED.hasRole, id_only=False) res_list.extend(res_objs) def get_core_attrs(resource): res_attr = { k: v for k, v in resource.__dict__.iteritems() if k in self.CORE_ATTRIBUTES } # HACK: Cannot use type_ because that would treat the dict as IonObject and add back all attributes res_attr["type__"] = resource.type_ return res_attr if type_filter: type_filter = set(type_filter) attr_list = [ get_core_attrs(res) for res in res_list if not type_filter or res.type_ in type_filter ] order_by = order_by or "name" attr_list = sorted(attr_list, key=lambda o: o.get(order_by, "")) if skip: attr_list = attr_list[skip:] if limit: attr_list = attr_list[:limit] # Need to return a similar type than RR.find_objects # Major bug in service gateway return attr_list, [] def get_distinct_values(self, restype='', attr_list=None, res_filter=None): """Returns a list of distinct values for given resource type and list of attribute names. Only supports simple types for the attribute values. Returns a sorted list of values or tuples of values. """ if not restype or type(restype) != str: raise BadRequest("Illegal value for argument restype") if not hasattr(interface.objects, restype): raise BadRequest("Given restype is not a resource type") if not attr_list or not type(attr_list) in (list, tuple): raise BadRequest("Illegal value for argument attr_list") type_cls = getattr(interface.objects, restype) try: if not all(type_cls._schema[an]["type"] in {"str", "int", "float"} for an in attr_list): raise BadRequest("Attribute in attr_list if invalid type") except KeyError: raise BadRequest("Attribute in attr_list unknown") if res_filter and type(res_filter) not in (list, tuple): raise BadRequest("Illegal value for argument res_filter") # NOTE: This can alternatively be implemented as a SELECT DISTINCT query, but this is not # supported by the underlying datastore interface. rq = ResourceQuery() if res_filter: rq.set_filter(rq.eq(rq.ATT_TYPE, restype), res_filter) else: rq.set_filter(rq.eq(rq.ATT_TYPE, restype)) res_list = self.clients.resource_registry.find_resources_ext( query=rq.get_query(), id_only=False) log.debug("Found %s resources of type %s", len(res_list), restype) att_values = sorted( {tuple(getattr(res, an) for an in attr_list) for res in res_list}) log.debug("Found %s distinct vales for attribute(s): %s", len(att_values), attr_list) return att_values def execute_lifecycle_transition(self, resource_id='', transition_event=''): """Alter object lifecycle according to given transition event. Throws exception if resource object does not exist or given transition_event is unknown/illegal. The new life cycle state after applying the transition is returned. """ res_type = self._get_resource_type(resource_id) res_interface = self._get_type_interface(res_type) if not 'execute_lifecycle_transition' in res_interface: raise BadRequest( "Resource type %s does not support: execute_lifecycle_transition" % res_type) res = self._call_crud(res_interface['execute_lifecycle_transition'], transition_event, resource_id, res_type) return res def get_lifecycle_events(self, resource_id=''): """For a given resource, return a list of possible lifecycle transition events. """ pass # ------------------------------------------------------------------------- # Agent interface def negotiate(self, resource_id='', sap_in=None): """Initiate a negotiation with this agent. The subject of this negotiation is the given ServiceAgreementProposal. The response is either a new ServiceAgreementProposal as counter-offer, or the same ServiceAgreementProposal indicating the offer has been accepted. """ pass def get_capabilities(self, resource_id='', current_state=True): """Introspect for agent capabilities. """ res_type = self._get_resource_type(resource_id) if self._has_agent(res_type): rac = ResourceAgentClient(resource_id=resource_id) return rac.get_capabilities(resource_id=resource_id, current_state=current_state) res_interface = self._get_type_interface(res_type) cap_list = [] for param in res_interface['params'].keys(): cap = AgentCapability(name=param, cap_type=CapabilityType.RES_PAR) cap_list.append(cap) for cmd in res_interface['commands'].keys(): cap = AgentCapability(name=cmd, cap_type=CapabilityType.RES_CMD) cap_list.append(cap) return cap_list def execute_resource(self, resource_id='', command=None): """Execute command on the resource represented by agent. """ res_type = self._get_resource_type(resource_id) if self._has_agent(res_type): rac = ResourceAgentClient(resource_id=resource_id) return rac.execute_resource(resource_id=resource_id, command=command) cmd_res = None res_interface = self._get_type_interface(res_type) target = get_safe(res_interface, "commands.%s.execute" % command.command, None) if target: res = self._call_execute(target, resource_id, res_type, command.args, command.kwargs) cmd_res = AgentCommandResult(command_id=command.command_id, command=command.command, ts_execute=get_ion_ts(), status=0) else: log.warn("execute_resource(): command %s not defined", command.command) return cmd_res def get_resource(self, resource_id='', params=None): """Return the value of the given resource parameter. """ res_type = self._get_resource_type(resource_id) if self._has_agent(res_type): rac = ResourceAgentClient(resource_id=resource_id) return rac.get_resource(resource_id=resource_id, params=params) res_interface = self._get_type_interface(res_type) get_result = {} for param in params: getter = get_safe(res_interface, "params.%s.get" % param, None) if getter: get_res = self._call_getter(getter, resource_id, res_type) get_result[param] = get_res else: get_result[param] = None return get_result def set_resource(self, resource_id='', params=None): """Set the value of the given resource parameters. """ res_type = self._get_resource_type(resource_id) if self._has_agent(res_type): rac = ResourceAgentClient(resource_id=resource_id) return rac.set_resource(resource_id=resource_id, params=params) res_interface = self._get_type_interface(res_type) for param in params: setter = get_safe(res_interface, "params.%s.set" % param, None) if setter: self._call_setter(setter, params[param], resource_id, res_type) else: log.warn("set_resource(): param %s not defined", param) def get_resource_state(self, resource_id=''): """Return the current resource specific state, if available. """ res_type = self._get_resource_type(resource_id) if self._has_agent(res_type): rac = ResourceAgentClient(resource_id=resource_id) return rac.get_resource_state(resource_id=resource_id) raise BadRequest("Not implemented for resource type %s", res_type) def ping_resource(self, resource_id=''): """Ping the resource. """ res_type = self._get_resource_type(resource_id) if self._has_agent(res_type): rac = ResourceAgentClient(resource_id=resource_id) return rac.ping_resource(resource_id=resource_id) raise BadRequest("Not implemented for resource type %s" % res_type) def execute_agent(self, resource_id='', command=None): """Execute command on the agent. """ res_type = self._get_resource_type(resource_id) if self._has_agent(res_type): rac = ResourceAgentClient(resource_id=resource_id) return rac.execute_agent(resource_id=resource_id, command=command) raise BadRequest("Not implemented for resource type %s" % res_type) def get_agent(self, resource_id='', params=None): """Return the value of the given agent parameters. """ res_type = self._get_resource_type(resource_id) if self._has_agent(res_type): rac = ResourceAgentClient(resource_id=resource_id) return rac.get_agent(resource_id=resource_id, params=params) raise BadRequest("Not implemented for resource type %s" % res_type) def set_agent(self, resource_id='', params=None): """Set the value of the given agent parameters. """ res_type = self._get_resource_type(resource_id) if self._has_agent(res_type): rac = ResourceAgentClient(resource_id=resource_id) return rac.set_agent(resource_id=resource_id, params=params) raise BadRequest("Not implemented for resource type %s" % res_type) def get_agent_state(self, resource_id=''): """Return the current resource agent common state. """ res_type = self._get_resource_type(resource_id) if self._has_agent(res_type): rac = ResourceAgentClient(resource_id=resource_id) return rac.get_agent_state(resource_id=resource_id) raise BadRequest("Not implemented for resource type %s" % res_type) def ping_agent(self, resource_id=''): """Ping the agent. """ res_type = self._get_resource_type(resource_id) if self._has_agent(res_type): rac = ResourceAgentClient(resource_id=resource_id) return rac.ping_agent(resource_id=resource_id) raise BadRequest("Not implemented for resource type %s" % res_type) # ----------------------------------------------------------------- def _augment_resource_interface_from_interfaces(self): """ Add resource type specific entries for CRUD, params and commands based on decorator annotations in service interfaces. This enables systematic definition and extension. @TODO Implement this so that static definitions are not needed anymore """ pass def _get_resource_type(self, resource_id): if resource_id in self.restype_cache: return self.restype_cache[resource_id] res = self.container.resource_registry.read(resource_id) res_type = res._get_type() self.restype_cache[resource_id] = res_type if len(self.restype_cache) > 10000: log.warn("Resource type cache exceeds size: %s", len(self.restype_cache)) return res_type def _has_agent(self, res_type): type_interface = self.resource_interface.get(res_type, None) return type_interface and type_interface.get('agent', False) def _get_type_interface(self, res_type): """ Creates a merge of params and commands up the type inheritance chain. Note: Entire param and command entries if subtypes replace their super types definition. """ res_interface = dict(params={}, commands={}) base_types = IonObject(res_type)._get_extends() base_types.insert(0, res_type) for rt in reversed(base_types): type_interface = self.resource_interface.get(rt, None) if not type_interface: continue for tpar, tval in type_interface.iteritems(): if tpar in res_interface: rval = res_interface[tpar] if isinstance(rval, dict): rval.update(tval) else: res_interface[tpar] = tval else: res_interface[tpar] = dict(tval) if isinstance( tval, dict) else tval return res_interface def _call_getter(self, func_sig, resource_id, res_type): return self._call_target(func_sig, resource_id=resource_id, res_type=res_type) def _call_setter(self, func_sig, value, resource_id, res_type): return self._call_target(func_sig, value=value, resource_id=resource_id, res_type=res_type) def _call_execute(self, func_sig, resource_id, res_type, cmd_args, cmd_kwargs): return self._call_target(func_sig, resource_id=resource_id, res_type=res_type, cmd_kwargs=cmd_kwargs) def _call_crud(self, func_sig, value, resource_id, res_type): return self._call_target(func_sig, value=value, resource_id=resource_id, res_type=res_type) def _call_target(self, target, value=None, resource_id=None, res_type=None, cmd_args=None, cmd_kwargs=None): """ Makes a call to a specified function. Function specification can be of varying type. """ try: if not target: return None match = re.match( "(func|serviceop):([\w.]+)\s*\(\s*([\w,$\s]*)\s*\)\s*(?:->\s*([\w\.]+))?\s*$", target) if match: func_type, func_name, func_args, res_path = match.groups() func = None if func_type == "func": if func_name.startswith("self."): func = getattr(self, func_name[5:]) else: func = named_any(func_name) elif func_type == "serviceop": svc_name, svc_op = func_name.split('.', 1) try: svc_client_cls = get_service_registry( ).get_service_by_name(svc_name).client except Exception as ex: log.error("No service client found for service: %s", svc_name) else: svc_client = svc_client_cls(process=self) func = getattr(svc_client, svc_op) if not func: return None args = self._get_call_args(func_args, resource_id, res_type, value, cmd_args) kwargs = {} if not cmd_kwargs else cmd_kwargs func_res = func(*args, **kwargs) log.info("Function %s result: %s", func, func_res) if res_path and isinstance(func_res, dict): func_res = get_safe(func_res, res_path, None) return func_res else: log.error("Unknown call target expression: %s", target) except Unauthorized as ex: # No need to log as this is not an application error, however, need to pass on the exception because # when called by the Service Gateway, the error message in the exception is required raise ex except Exception as ex: log.exception("_call_target exception") raise ex #Should to pass it back because when called by the Service Gateway, the error message in the exception is required def _get_call_args(self, func_arg_str, resource_id, res_type, value=None, cmd_args=None): args = [] func_args = func_arg_str.split(',') if func_args: for arg in func_args: arg = arg.strip() if arg == "$RESOURCE_ID": args.append(resource_id) elif arg == "$RESOURCE_TYPE": args.append(res_type) elif arg == "$VALUE" or arg == "$RESOURCE": args.append(value) elif arg == "$ARGS": if cmd_args is not None: args.extend(cmd_args) elif not arg: args.append(None) else: args.append(arg) return args # Callable functions def get_resource_size(self, resource_id): res_obj = self.container.resource_registry.rr_store.read_doc( resource_id) import json obj_str = json.dumps(res_obj) res_len = len(obj_str) log.info("Resource %s length: %s", resource_id, res_len) return res_len def set_resource_description(self, resource_id, value): res_obj = self.container.resource_registry.read(resource_id) res_obj.description = value self.container.resource_registry.update(res_obj) log.info("Resource %s description updated: %s", resource_id, value)
class ResourceManagementService(BaseResourceManagementService): """ Service that manages resource types and lifecycle workflows. It also provides generic operations that manage any kind of resource and their lifecycle. Also provides a resource discovery and query capability. """ MAX_SEARCH_RESULTS = CFG.get_safe('service.resource_management.max_search_results', 250) def on_init(self): self.resource_interface = (Config(["res/config/resource_management.yml"])).data['ResourceInterface'] self._augment_resource_interface_from_interfaces() # Keep a cache of known resource ids self.restype_cache = {} self.ds_discovery = DatastoreDiscovery(self) # ------------------------------------------------------------------------- # Search and query def query(self, query=None, id_only=True, search_args=None): """Issue a query provided in structured dict format or internal datastore query format. Returns a list of resource or event objects or their IDs only. Search_args may contain parameterized values. See the query format definition: https://confluence.oceanobservatories.org/display/CIDev/Discovery+Service+Query+Format """ if not query: raise BadRequest("Invalid query") return self._discovery_request(query, id_only, search_args=search_args, query_params=search_args) def query_view(self, view_id='', view_name='', ext_query=None, id_only=True, search_args=None): """Execute an existing query as defined within a View resource, providing additional arguments for parameterized values. If ext_query is provided, it will be combined with the query defined by the View. Search_args may contain parameterized values. Returns a list of resource or event objects or their IDs only. """ if not view_id and not view_name: raise BadRequest("Must provide argument view_id or view_name") if view_id and view_name: raise BadRequest("Cannot provide both arguments view_id and view_name") if view_id: view_obj = self.clients.resource_registry.read(view_id) else: view_obj = self.ds_discovery.get_builtin_view(view_name) if not view_obj: view_objs, _ = self.clients.resource_registry.find_resources(restype=RT.View, name=view_name) if not view_objs: raise NotFound("View with name '%s' not found" % view_name) view_obj = view_objs[0] if view_obj.type_ != RT.View: raise BadRequest("Argument view_id is not a View resource") view_query = view_obj.view_definition if not QUERY_EXP_KEY in view_query: raise BadRequest("Unknown View query format") # Get default query params and override them with provided args param_defaults = {param.name: param.default for param in view_obj.view_parameters} query_params = param_defaults if view_obj.param_values: query_params.update(view_obj.param_values) if search_args: query_params.update(search_args) # Merge ext_query into query if ext_query: if ext_query["where"] and view_query["where"]: view_query["where"] = [DQ.EXP_AND, [view_query["where"], ext_query["where"]]] else: view_query["where"] = view_query["where"] or ext_query["where"] if ext_query["order_by"]: # Override ordering if present view_query["where"] = ext_query["order_by"] # Other query settings view_qargs = view_query["query_args"] ext_qargs = ext_query["query_args"] view_qargs["id_only"] = ext_qargs.get("id_only", view_qargs["id_only"]) view_qargs["limit"] = ext_qargs.get("limit", view_qargs["limit"]) view_qargs["skip"] = ext_qargs.get("skip", view_qargs["skip"]) return self._discovery_request(view_query, id_only=id_only, search_args=search_args, query_params=query_params) def _discovery_request(self, query=None, id_only=True, search_args=None, query_params=None): search_args = search_args or {} if not query: raise BadRequest('No request query provided') if QUERY_EXP_KEY in query and self.ds_discovery: query.setdefault("query_args", {})["id_only"] = id_only # Query in datastore query format (dict) log.debug("Executing datastore query: %s", query) elif 'query' not in query: raise BadRequest('Unsupported request. %s' % query) # if count requested, run id_only query without limit/skip count = search_args.get("count", False) if count: # Only return the count of ID only search query.pop("limit", None) query.pop("skip", None) res = self.ds_discovery.execute_query(query, id_only=True, query_args=search_args, query_params=query_params) return [len(res)] # TODO: Not all queries are permissible by all users # Execute the query query_results = self.ds_discovery.execute_query(query, id_only=id_only, query_args=search_args, query_params=query_params) # Strip out unwanted object attributes for size filtered_res = self._strip_query_results(query_results, id_only=id_only, search_args=search_args) return filtered_res def _strip_query_results(self, query_results, id_only, search_args): # Filter the results for smaller result size attr_filter = search_args.get("attribute_filter", []) if type(attr_filter) not in (list, tuple): raise BadRequest("Illegal argument type: attribute_filter") if not id_only and attr_filter: filtered_res = [dict(__noion__=True, **{k: v for k, v in obj.__dict__.iteritems() if k in attr_filter or k in {"_id", "type_"}}) for obj in query_results] return filtered_res return query_results # ------------------------------------------------------------------------- # View management (CRUD) def create_view(self, view=None): if view is None or not isinstance(view, View): raise BadRequest("Illegal argument: view") # view_objs, _ = self.clients.resource_registry.find_resources(restype=RT.View, name=view.name) # if view_objs: # raise BadRequest("View with name '%s' already exists" % view.name) view_id, _ = self.clients.resource_registry.create(view) return view_id def read_view(self, view_id=''): view_res = self.clients.resource_registry.read(view_id) if not isinstance(view_res, View): raise BadRequest("Resource %s is not a View" % view_id) return view_res def update_view(self, view=None): if view is None or not isinstance(view, View): raise BadRequest("Illegal argument: view") self.clients.resource_registry.update(view) return True def delete_view(self, view_id=''): self.clients.resource_registry.delete(view_id) return True # ------------------------------------------------------------------------- # Generic resource interface def create_resource(self, resource=None): """Creates an arbitrary resource object via its defined create function, so that it can successively can be accessed via the agent interface. """ if not isinstance(resource, Resource): raise BadRequest("Can only create resources, not type %s" % type(resource)) res_type = resource._get_type() res_interface = self._get_type_interface(res_type) if not 'create' in res_interface: raise BadRequest("Resource type %s does not support: CREATE" % res_type) res = self._call_crud(res_interface['create'], resource, None, res_type) if type(res) in (list,tuple): res = res[0] return res def update_resource(self, resource=None): """Updates an existing resource via the configured service operation. """ if not isinstance(resource, Resource): raise BadRequest("Can only update resources, not type %s" % type(resource)) res_type = resource._get_type() res_interface = self._get_type_interface(res_type) if not 'update' in res_interface: raise BadRequest("Resource type %s does not support: UPDATE" % res_type) self._call_crud(res_interface['update'], resource, None, res_type) def read_resource(self, resource_id=''): """Returns an existing resource via the configured service operation. """ res_type = self._get_resource_type(resource_id) res_interface = self._get_type_interface(res_type) if not 'read' in res_interface: raise BadRequest("Resource type %s does not support: READ" % res_type) res_obj = self._call_crud(res_interface['read'], None, resource_id, res_type) return res_obj def delete_resource(self, resource_id=''): """Deletes an existing resource via the configured service operation. """ res_type = self._get_resource_type(resource_id) res_interface = self._get_type_interface(res_type) if not 'delete' in res_interface: raise BadRequest("Resource type %s does not support: DELETE" % res_type) self._call_crud(res_interface['delete'], None, resource_id, res_type) CORE_ATTRIBUTES = {"_id", "name", "description", "ts_created", "ts_updated", "lcstate", "availability", "visibility", "alt_resource_type"} def get_org_resource_attributes(self, org_id='', order_by='', type_filter=None, limit=0, skip=0): """For a given org, return a list of dicts with core resource attributes (_id, type_, name, description, ts_created, ts_modified, lcstate, availability, visibility and alt_resource_type). The returned list is ordered by name unless otherwise specified. Supports pagination and white-list filtering if provided. """ if not org_id: raise BadRequest("Must provide org_id") res_list = [] res_objs, _ = self.clients.resource_registry.find_objects(org_id, PRED.hasResource, id_only=False) res_list.extend(res_objs) # TODO: The following is not correct - this should be all attachments in the Org res_objs, _ = self.clients.resource_registry.find_objects(org_id, PRED.hasAttachment, id_only=False) res_list.extend(res_objs) # TODO: The following should be shared in the Org res_objs, _ = self.clients.resource_registry.find_objects(org_id, PRED.hasRole, id_only=False) res_list.extend(res_objs) def get_core_attrs(resource): res_attr = {k:v for k, v in resource.__dict__.iteritems() if k in self.CORE_ATTRIBUTES} # HACK: Cannot use type_ because that would treat the dict as IonObject and add back all attributes res_attr["type__"] = resource.type_ return res_attr if type_filter: type_filter = set(type_filter) attr_list = [get_core_attrs(res) for res in res_list if not type_filter or res.type_ in type_filter] order_by = order_by or "name" attr_list = sorted(attr_list, key=lambda o: o.get(order_by, "")) if skip: attr_list = attr_list[skip:] if limit: attr_list = attr_list[:limit] # Need to return a similar type than RR.find_objects # Major bug in service gateway return attr_list, [] def get_distinct_values(self, restype='', attr_list=None, res_filter=None): """Returns a list of distinct values for given resource type and list of attribute names. Only supports simple types for the attribute values. Returns a sorted list of values or tuples of values. """ if not restype or type(restype) != str: raise BadRequest("Illegal value for argument restype") if not hasattr(interface.objects, restype): raise BadRequest("Given restype is not a resource type") if not attr_list or not type(attr_list) in (list, tuple): raise BadRequest("Illegal value for argument attr_list") type_cls = getattr(interface.objects, restype) try: if not all(type_cls._schema[an]["type"] in {"str", "int", "float"} for an in attr_list): raise BadRequest("Attribute in attr_list if invalid type") except KeyError: raise BadRequest("Attribute in attr_list unknown") if res_filter and type(res_filter) not in (list, tuple): raise BadRequest("Illegal value for argument res_filter") # NOTE: This can alternatively be implemented as a SELECT DISTINCT query, but this is not # supported by the underlying datastore interface. rq = ResourceQuery() if res_filter: rq.set_filter(rq.eq(rq.ATT_TYPE, restype), res_filter) else: rq.set_filter(rq.eq(rq.ATT_TYPE, restype)) res_list = self.clients.resource_registry.find_resources_ext(query=rq.get_query(), id_only=False) log.debug("Found %s resources of type %s", len(res_list), restype) att_values = sorted({tuple(getattr(res, an) for an in attr_list) for res in res_list}) log.debug("Found %s distinct vales for attribute(s): %s", len(att_values), attr_list) return att_values def execute_lifecycle_transition(self, resource_id='', transition_event=''): """Alter object lifecycle according to given transition event. Throws exception if resource object does not exist or given transition_event is unknown/illegal. The new life cycle state after applying the transition is returned. """ res_type = self._get_resource_type(resource_id) res_interface = self._get_type_interface(res_type) if not 'execute_lifecycle_transition' in res_interface: raise BadRequest("Resource type %s does not support: execute_lifecycle_transition" % res_type) res = self._call_crud(res_interface['execute_lifecycle_transition'], transition_event, resource_id, res_type) return res def get_lifecycle_events(self, resource_id=''): """For a given resource, return a list of possible lifecycle transition events. """ pass # ------------------------------------------------------------------------- # Agent interface def negotiate(self, resource_id='', sap_in=None): """Initiate a negotiation with this agent. The subject of this negotiation is the given ServiceAgreementProposal. The response is either a new ServiceAgreementProposal as counter-offer, or the same ServiceAgreementProposal indicating the offer has been accepted. """ pass def get_capabilities(self, resource_id='', current_state=True): """Introspect for agent capabilities. """ res_type = self._get_resource_type(resource_id) if self._has_agent(res_type): rac = ResourceAgentClient(resource_id=resource_id) return rac.get_capabilities(resource_id=resource_id, current_state=current_state) res_interface = self._get_type_interface(res_type) cap_list = [] for param in res_interface['params'].keys(): cap = AgentCapability(name=param, cap_type=CapabilityType.RES_PAR) cap_list.append(cap) for cmd in res_interface['commands'].keys(): cap = AgentCapability(name=cmd, cap_type=CapabilityType.RES_CMD) cap_list.append(cap) return cap_list def execute_resource(self, resource_id='', command=None): """Execute command on the resource represented by agent. """ res_type = self._get_resource_type(resource_id) if self._has_agent(res_type): rac = ResourceAgentClient(resource_id=resource_id) return rac.execute_resource(resource_id=resource_id, command=command) cmd_res = None res_interface = self._get_type_interface(res_type) target = get_safe(res_interface, "commands.%s.execute" % command.command, None) if target: res = self._call_execute(target, resource_id, res_type, command.args, command.kwargs) cmd_res = AgentCommandResult(command_id=command.command_id, command=command.command, ts_execute=get_ion_ts(), status=0) else: log.warn("execute_resource(): command %s not defined", command.command) return cmd_res def get_resource(self, resource_id='', params=None): """Return the value of the given resource parameter. """ res_type = self._get_resource_type(resource_id) if self._has_agent(res_type): rac = ResourceAgentClient(resource_id=resource_id) return rac.get_resource(resource_id=resource_id, params=params) res_interface = self._get_type_interface(res_type) get_result = {} for param in params: getter = get_safe(res_interface, "params.%s.get" % param, None) if getter: get_res = self._call_getter(getter, resource_id, res_type) get_result[param] = get_res else: get_result[param] = None return get_result def set_resource(self, resource_id='', params=None): """Set the value of the given resource parameters. """ res_type = self._get_resource_type(resource_id) if self._has_agent(res_type): rac = ResourceAgentClient(resource_id=resource_id) return rac.set_resource(resource_id=resource_id, params=params) res_interface = self._get_type_interface(res_type) for param in params: setter = get_safe(res_interface, "params.%s.set" % param, None) if setter: self._call_setter(setter, params[param], resource_id, res_type) else: log.warn("set_resource(): param %s not defined", param) def get_resource_state(self, resource_id=''): """Return the current resource specific state, if available. """ res_type = self._get_resource_type(resource_id) if self._has_agent(res_type): rac = ResourceAgentClient(resource_id=resource_id) return rac.get_resource_state(resource_id=resource_id) raise BadRequest("Not implemented for resource type %s", res_type) def ping_resource(self, resource_id=''): """Ping the resource. """ res_type = self._get_resource_type(resource_id) if self._has_agent(res_type): rac = ResourceAgentClient(resource_id=resource_id) return rac.ping_resource(resource_id=resource_id) raise BadRequest("Not implemented for resource type %s" % res_type) def execute_agent(self, resource_id='', command=None): """Execute command on the agent. """ res_type = self._get_resource_type(resource_id) if self._has_agent(res_type): rac = ResourceAgentClient(resource_id=resource_id) return rac.execute_agent(resource_id=resource_id, command=command) raise BadRequest("Not implemented for resource type %s" % res_type) def get_agent(self, resource_id='', params=None): """Return the value of the given agent parameters. """ res_type = self._get_resource_type(resource_id) if self._has_agent(res_type): rac = ResourceAgentClient(resource_id=resource_id) return rac.get_agent(resource_id=resource_id, params=params) raise BadRequest("Not implemented for resource type %s" % res_type) def set_agent(self, resource_id='', params=None): """Set the value of the given agent parameters. """ res_type = self._get_resource_type(resource_id) if self._has_agent(res_type): rac = ResourceAgentClient(resource_id=resource_id) return rac.set_agent(resource_id=resource_id, params=params) raise BadRequest("Not implemented for resource type %s" % res_type) def get_agent_state(self, resource_id=''): """Return the current resource agent common state. """ res_type = self._get_resource_type(resource_id) if self._has_agent(res_type): rac = ResourceAgentClient(resource_id=resource_id) return rac.get_agent_state(resource_id=resource_id) raise BadRequest("Not implemented for resource type %s" % res_type) def ping_agent(self, resource_id=''): """Ping the agent. """ res_type = self._get_resource_type(resource_id) if self._has_agent(res_type): rac = ResourceAgentClient(resource_id=resource_id) return rac.ping_agent(resource_id=resource_id) raise BadRequest("Not implemented for resource type %s" % res_type) # ----------------------------------------------------------------- def _augment_resource_interface_from_interfaces(self): """ Add resource type specific entries for CRUD, params and commands based on decorator annotations in service interfaces. This enables systematic definition and extension. @TODO Implement this so that static definitions are not needed anymore """ pass def _get_resource_type(self, resource_id): if resource_id in self.restype_cache: return self.restype_cache[resource_id] res = self.container.resource_registry.read(resource_id) res_type = res._get_type() self.restype_cache[resource_id] = res_type if len(self.restype_cache) > 10000: log.warn("Resource type cache exceeds size: %s", len(self.restype_cache)) return res_type def _has_agent(self, res_type): type_interface = self.resource_interface.get(res_type, None) return type_interface and type_interface.get('agent', False) def _get_type_interface(self, res_type): """ Creates a merge of params and commands up the type inheritance chain. Note: Entire param and command entries if subtypes replace their super types definition. """ res_interface = dict(params={}, commands={}) base_types = IonObject(res_type)._get_extends() base_types.insert(0, res_type) for rt in reversed(base_types): type_interface = self.resource_interface.get(rt, None) if not type_interface: continue for tpar, tval in type_interface.iteritems(): if tpar in res_interface: rval = res_interface[tpar] if isinstance(rval, dict): rval.update(tval) else: res_interface[tpar] = tval else: res_interface[tpar] = dict(tval) if isinstance(tval, dict) else tval return res_interface def _call_getter(self, func_sig, resource_id, res_type): return self._call_target(func_sig, resource_id=resource_id, res_type=res_type) def _call_setter(self, func_sig, value, resource_id, res_type): return self._call_target(func_sig, value=value, resource_id=resource_id, res_type=res_type) def _call_execute(self, func_sig, resource_id, res_type, cmd_args, cmd_kwargs): return self._call_target(func_sig, resource_id=resource_id, res_type=res_type, cmd_kwargs=cmd_kwargs) def _call_crud(self, func_sig, value, resource_id, res_type): return self._call_target(func_sig, value=value, resource_id=resource_id, res_type=res_type) def _call_target(self, target, value=None, resource_id=None, res_type=None, cmd_args=None, cmd_kwargs=None): """ Makes a call to a specified function. Function specification can be of varying type. """ try: if not target: return None match = re.match("(func|serviceop):([\w.]+)\s*\(\s*([\w,$\s]*)\s*\)\s*(?:->\s*([\w\.]+))?\s*$", target) if match: func_type, func_name, func_args, res_path = match.groups() func = None if func_type == "func": if func_name.startswith("self."): func = getattr(self, func_name[5:]) else: func = named_any(func_name) elif func_type == "serviceop": svc_name, svc_op = func_name.split('.', 1) try: svc_client_cls = get_service_registry().get_service_by_name(svc_name).client except Exception as ex: log.error("No service client found for service: %s", svc_name) else: svc_client = svc_client_cls(process=self) func = getattr(svc_client, svc_op) if not func: return None args = self._get_call_args(func_args, resource_id, res_type, value, cmd_args) kwargs = {} if not cmd_kwargs else cmd_kwargs func_res = func(*args, **kwargs) log.info("Function %s result: %s", func, func_res) if res_path and isinstance(func_res, dict): func_res = get_safe(func_res, res_path, None) return func_res else: log.error("Unknown call target expression: %s", target) except Unauthorized as ex: # No need to log as this is not an application error, however, need to pass on the exception because # when called by the Service Gateway, the error message in the exception is required raise ex except Exception as ex: log.exception("_call_target exception") raise ex #Should to pass it back because when called by the Service Gateway, the error message in the exception is required def _get_call_args(self, func_arg_str, resource_id, res_type, value=None, cmd_args=None): args = [] func_args = func_arg_str.split(',') if func_args: for arg in func_args: arg = arg.strip() if arg == "$RESOURCE_ID": args.append(resource_id) elif arg == "$RESOURCE_TYPE": args.append(res_type) elif arg == "$VALUE" or arg == "$RESOURCE": args.append(value) elif arg == "$ARGS": if cmd_args is not None: args.extend(cmd_args) elif not arg: args.append(None) else: args.append(arg) return args # Callable functions def get_resource_size(self, resource_id): res_obj = self.container.resource_registry.rr_store.read_doc(resource_id) import json obj_str = json.dumps(res_obj) res_len = len(obj_str) log.info("Resource %s length: %s", resource_id, res_len) return res_len def set_resource_description(self, resource_id, value): res_obj = self.container.resource_registry.read(resource_id) res_obj.description = value self.container.resource_registry.update(res_obj) log.info("Resource %s description updated: %s", resource_id, value)