class IdentityRegistryService(BaseRegistryService): # Declaration of service declare = BaseService.service_declare(name='identity_service', version='0.1.0', dependencies=[]) op_clear_identity_registry = BaseRegistryService.base_clear_registry op_register_user = BaseRegistryService.base_register_resource op_update_user = BaseRegistryService.base_register_resource op_get_user = BaseRegistryService.base_get_resource op_set_identity_lcstate = BaseRegistryService.base_set_resource_lcstate op_find_users = BaseRegistryService.base_find_resource
class RegistryService(BaseRegistryService): """ @Brief Example Registry Service implementation using the base class """ # Declaration of service declare = BaseService.service_declare(name='registry_service', version='0.1.0', dependencies=[]) op_clear_registry = BaseRegistryService.base_clear_registry op_register_resource = BaseRegistryService.base_register_resource op_get_resource = BaseRegistryService.base_get_resource op_get_resource_by_id = BaseRegistryService.base_get_resource_by_id op_set_resource_lcstate = BaseRegistryService.base_set_resource_lcstate op_find_resource = BaseRegistryService.base_find_resource
class ServiceRegistryService(registry.BaseRegistryService): """ Service registry service interface @todo a service is a resource and should also be living in the resource registry """ # Declaration of service declare = BaseService.service_declare(name='service_registry', version='0.1.0', dependencies=[]) op_clear_registry = registry.BaseRegistryService.base_clear_registry op_register_service_definition = registry.BaseRegistryService.base_register_resource """ Service operation: Register a service definition with the registry. """ op_get_service_definition = registry.BaseRegistryService.base_get_resource """ Service operation: Get a service definition. """ op_register_service_instance = registry.BaseRegistryService.base_register_resource """ Service operation: Register a service instance with the registry. """ op_get_service_instance = registry.BaseRegistryService.base_get_resource """ Service operation: Get a service instance. """ op_set_service_lcstate = registry.BaseRegistryService.base_set_resource_lcstate """ Service operation: Set a service life cycle state """ op_find_registered_service_definition_from_service = registry.BaseRegistryService.base_find_resource """ Service operation: Find the definition of a service """ op_find_registered_service_definition_from_description = registry.BaseRegistryService.base_find_resource """ Service operation: Find service definitions which meet a description """ op_find_registered_service_instance_from_service = registry.BaseRegistryService.base_find_resource """ Service operation: Find the registered instance that matches the service instance """ op_find_registered_service_instance_from_description = registry.BaseRegistryService.base_find_resource """
class DataStoreService(BaseService): """ Example service interface """ # Declaration of service declare = BaseService.service_declare(name='DataStoreService', version='0.1.0', dependencies=[]) def __init__(self, receiver, spawnArgs=None): """ @brief Init method for the DataStore Frontend service @param frontend - an instance of a CAStore Frontend """ # Service class initializer. Basic config, but no yields allowed. self.frontend = spawnArgs['MyFrontend'] BaseService.__init__(self, receiver, spawnArgs) logging.info('DataStoreService.__init__()') # @defer.inlineCallbacks def slc_init(self): pass @defer.inlineCallbacks def op_push(self, content, headers, msg): logging.info('op_push: ' + str(content) + ', Remote Frontend:' + self.frontend) # The following line shows how to reply to a message yield self.reply_ok(msg) @defer.inlineCallbacks def op_pull(self, content, headers, msg): logging.info('op_pull: ' + str(content) + ', Remote Frontend:' + self.frontend) # The following line shows how to reply to a message yield self.reply_ok(msg) @defer.inlineCallbacks def op_clone(self, content, headers, msg): logging.info('op_clone: ' + str(content) + ', Remote Frontend:' + self.frontend) # The following line shows how to reply to a message yield self.reply_ok(msg)
class InstrumentRegistryService(registry.BaseRegistryService): """ Service that provides a registry for instrument devices, types etc. Based on the BaseRegistryService. """ # Declaration of service declare = BaseService.service_declare(name='instrument_registry', version='0.1.0', dependencies=[]) op_clear_instrument_registry = registry.BaseRegistryService.base_clear_registry """ Service operation: Clears all records from the instrument registry. """ op_register_instrument_instance = registry.BaseRegistryService.base_register_resource """ Service operation: Create or update an instrument instance with the registry. """ op_register_instrument_type = registry.BaseRegistryService.base_register_resource """ Service operation: Create or update an instrument type with the registry. """ op_get_instrument_instance = registry.BaseRegistryService.base_get_resource """ Service operation: Get a instrument instance. """ op_get_instrument_by_id = registry.BaseRegistryService.base_get_resource_by_id op_get_instrument_type = registry.BaseRegistryService.base_get_resource #changed """ Service operation: Get a instrument type. """ op_find_instrument_instance = registry.BaseRegistryService.base_find_resource """ Service operation: find instrument instances by characteristics """ op_find_instrument_type = registry.BaseRegistryService.base_find_resource """
class PreservationRegistryService(registry.BaseRegistryService): """ @Brief Preservation registry service interface """ # Declaration of service declare = BaseService.service_declare(name='preservation_registry', version='0.1.0', dependencies=[]) op_define_archive = registry.BaseRegistryService.base_register_resource """ Service operation: Create or update a archive resource. """ op_get_archive = registry.BaseRegistryService.base_get_resource """ Service operation: Get an archive resource """ op_find_archive = registry.BaseRegistryService.base_find_resource """
class IngestionRegistryService(registry.BaseRegistryService): """ @Brief Ingestion registry service interface """ # Declaration of service declare = BaseService.service_declare(name='ingestion_registry', version='0.1.0', dependencies=[]) op_define_ingestion_stream = registry.BaseRegistryService.base_register_resource """ Service operation: Create or update a ingestion_stream resource. """ op_get_ingestion_stream = registry.BaseRegistryService.base_get_resource """ Service operation: Get an ingestion_stream resource """ op_find_ingestion_stream = registry.BaseRegistryService.base_find_resource """
class CoordinatorService(BaseService): """ Refactor this into a BaseService that provides dap data on a looping call Make the url a parameter of the process - one per url... Brains behind preservation, and also the primary interface. """ # Define ourselves for the CC declare = BaseService.service_declare(name='coordinator', version='0.1.0', dependencies=['fetcher']) def slc_init(self): """ Service life cycle state. Initialize service here. Can use yields. @todo Create instances of clients here for later - fetcher, attr store, etc """ logging.debug('Preservation coordinator SLC init') self.fc = FetcherClient(proc=self) @defer.inlineCallbacks def op_get_url(self, content, headers, msg): """ @brief Method for proxy - request a (DAP) URL @param content URL to fetch @param headers conv-id and reply-to should point to proxy/requester @param msg Not used @todo Cache logic - right now just trapdoors all reqs to fetcher """ logging.debug('Coordinator forwarding URL request to fetcher') yield self.fc.forward_get_url(content, headers) @defer.inlineCallbacks def op_get_dap_dataset(self, content, headers, msg): """ @brief Similar to op_get_url. Fetches an entire DAP dataset. @param content URL to fetch @param headers conv-id and reply-to should point to proxy/requester @param msg Not used @todo Cache logic - right now just trapdoors all reqs to fetcher """ yield self.fc.forward_get_dap_dataset(content, headers)
class ComputationPlannerService(BaseService): """Provisioner service interface """ # Declaration of service declare = BaseService.service_declare(name='computation_planner', version='0.1.0', dependencies=[]) def op_request_computation(self, content, headers, msg): """Service operation: Request computation resources """ def op_schedule_computation(self, content, headers, msg): """Service operation: Request computation with given schedule """ def op_set_policy(self, content, headers, msg): """Service operation: Sets the policy that influences the planning of
class DataRegistryService(registry.BaseRegistryService): """ @Brief Dataset registry service interface """ # Declaration of service declare = BaseService.service_declare(name='data_registry', version='0.1.0', dependencies=[]) op_define_data = registry.BaseRegistryService.base_register_resource """ Service operation: Create or update a data resource. """ op_get_data = registry.BaseRegistryService.base_get_resource """ Service operation: Get data description """ op_find_data = registry.BaseRegistryService.base_find_resource """
class HostStatusService(BaseService): """ Host status interface """ # Declaration of service declare = BaseService.service_declare(name='host_status', version='0.1.0', dependencies=[]) def slc_init(self): self.INTERVAL = 1 # seconds self.COUNT = 1 self.count = self.COUNT self.client = xmlrpc.Proxy('http://localhost:9010') self.lc = task.LoopingCall(self.report) self.lc.start(self.INTERVAL) @defer.inlineCallbacks def report(self): self.count -= 1 if self.count < 0: logging.debug('Shutting down host status looping call') self.lc.stop() return logging.debug('Starting report query') status = yield self.client.callRemote("getStatusString", "all") logging.debug('Received report') print status def isRunning(self): return self.lc.running def op_config(self, content, headers, msg): pass @defer.inlineCallbacks def op_sendstatus(self, content, headers, msg): yield self.reply_ok(msg)
class ResponseService(BaseService): """Example service implementation """ # Declaration of service declare = BaseService.service_declare(name='responder', version='0.1.0', dependencies=[]) def slc_init(self): pass @defer.inlineCallbacks def op_respond(self, content, headers, msg): logging.info('op_respond: ' + str(content)) obj = dataobject.DataObject.decode(content) logging.info(obj) response = obj.encode() # The following line shows how to reply to a message yield self.reply_ok(msg, response)
class AttributeStoreService(store_service.StoreService): """ Service to store and retrieve key/value pairs. The Implementation is in ion.data.backends.store_service """ # Declaration of service declare = BaseService.service_declare(name='attributestore', version='0.1.0', dependencies=[]) def __init__(self, receiver, spawnArgs=None): # Service class initializer. Basic config, but no yields allowed. BaseService.__init__(self, receiver, spawnArgs) self.spawn_args['backend_class'] = self.spawn_args.get( 'backend_class', CONF.getValue('backend_class', default='ion.data.store.Store')) self.spawn_args['backend_args'] = self.spawn_args.get( 'backend_args', CONF.getValue('backend_args', default={})) logging.info('AttributeStoreService.__init__()')
class StateRepositoryService(BaseService): """Repository for service state service interface. Service state is information shared between many processes. """ # Declaration of service declare = BaseService.service_declare(name='state_repository', version='0.1.0', dependencies=[]) def op_define_state(self, content, headers, msg): """Service operation: Create a new state object (session) or update an existing one by replacing """ def op_update_state(self, content, headers, msg): """Service operation: Provide an incremental update to the service state. """ def op_retrieve_state(self, content, headers, msg): """Service operation: TBD
class LoggerService(BaseService): """Logger service interface """ # Declaration of service declare = BaseService.service_declare( name='logger', version='0.1.0', dependencies=[] ) def slc_init(self): logging.info("LoggingService initialized") def op_config(self, content, headers, msg): pass @defer.inlineCallbacks def op_logmsg(self, content, headers, msg): level = content.get('level','info') logmsg = content.get('msg','#NO MESSAGE#') # need to do something reasonable with these soon # lfrom = headers.get('sender','') # ltime = content.get('logtime') if level == 'debug': logserv.debug(logmsg) elif level == 'info': logserv.info(logmsg) elif level == 'warn': logserv.warn(logmsg) elif level == 'error': logserv.error(logmsg) elif level == 'critical': logserv.critical(logmsg) else: logging.error('Invalid log level: '+str(level)) yield self.reply_ok(msg)
class WorkerProcess(BaseService): """Worker process """ # Declaration of service declare = BaseService.service_declare(name='worker', version='0.1.0', dependencies=[]) @defer.inlineCallbacks def slc_init(self): msg_name = self.spawn_args['service-name'] scope = self.spawn_args['scope'] logging.info("slc_init name received:" + msg_name) msg_name1 = self.get_scoped_name(scope, msg_name) logging.info("slc_init name used:" + msg_name1) workReceiver = Receiver(__name__, msg_name1) self.workReceiver = workReceiver self.workReceiver.handle(self.receive) logging.info("slc_init worker receiver spawning") id = yield spawn(workReceiver) logging.info("slc_init worker receiver spawned:" + str(id)) @defer.inlineCallbacks def op_work(self, content, headers, msg): yield self._work(content) yield self.reply(msg, 'result', {'work-id': content['work-id']}, {}) @defer.inlineCallbacks def _work(self, content): myid = self.proc_name + ":" + self.receiver.spawned.id.local workid = str(content['work-id']) waittime = float(content['work']) logging.info("worker=" + myid + " job=" + workid + " work=" + str(waittime)) yield pu.asleep(waittime) logging.info("worker=" + myid + " job=" + workid + " done at=" + str(pu.currenttime_ms()))
class TaskableResourceRegistryService(BaseService): """Taskable resource registry and definition repository service interface """ # Declaration of service declare = BaseService.service_declare(name='taskable_resource_registry', version='0.1.0', dependencies=[]) def op_define_resource(self, content, headers, msg): """Service operation: Create or update taskable resource description """ def op_find_resource(self, content, headers, msg): """Service operation: Create or update taskable resource description """ def op_store_resource(self, content, headers, msg): """Service operation: Store the definition of a taskable resource, e.g. source code, virtual machine image (or a pointer to it) """ def op_retrieve_resource(self, content, headers, msg): """Service operation: Retrieve the definition of a taskable resource
class ConversationRepositoryService(BaseService): """Conversation repository service interface """ # Declaration of service declare = BaseService.service_declare(name='conversation_repository', version='0.1.0', dependencies=[]) def slc_init(self): pass def op_define_conv_type(self, content, headers, msg): """Service operation: Define a new conversation type (aka protocol, interaction pattern) """ def op_get_conv_type(self, content, headers, msg): """Service operation: Returns the description of the conversation type including the specification """ def op_define_conversation(self, content, headers, msg): """Service operation: Create a new conversation (instance) definition """ def op_bind_conversation(self, content, headers, msg): """Service operation: Add oneself to the conversation role binding """ def op_log_message(self, content, headers, msg): """Service operation: Log an occurred message with the repository """ logmsg = content['msg'] logserv.info("-----------------------------------------------\n" + str(logmsg))
class HelloService(BaseService): """ Example service interface """ # Declaration of service declare = BaseService.service_declare(name='hello', version='0.1.0', dependencies=[]) def __init__(self, receiver, spawnArgs=None): # Service class initializer. Basic config, but no yields allowed. BaseService.__init__(self, receiver, spawnArgs) logging.info('HelloService.__init__()') def slc_init(self): # Service life cycle state. Initialize service here. Can use yields. pass @defer.inlineCallbacks def op_hello(self, content, headers, msg): logging.info('op_hello: ' + str(content)) # The following line shows how to reply to a message yield self.reply_ok(msg, {'value': 'Hello there, ' + str(content)}, {})
class FetcherService(BaseService): """ Fetcher, implemented as a service. Service declaration - seems similar to the Zope methods @todo Dependencies - perhaps pub-sub? @todo refactor to use dap_tools @note These are not class methods! """ #logging.info('Declaring fetcher...') declare = BaseService.service_declare(name='fetcher', version='0.1.2', dependencies=[]) """ @todo Declare fetcher name into dns-equivalent... """ def _reassemble_headers(self, result): """ @brief Convert an array of tuples into http headers @param result HTTP result @retval Multiline string with trailing empty line @todo check for library routine to do this. @note output has an blank line at the end (\r\n) """ hstr = 'HTTP/1.0 %d %s\r\n' % (result.status, result.reason) for x in result.getheaders(): hstr = hstr + '%s: %s\r\n' % (x[0], x[1]) return hstr @defer.inlineCallbacks def _http_op(self, operation, src_url, msg): """ @brief Inner method for GET and HEAD, does fetch and reply @param operation 'GET' or 'HEAD' @param src_url Source URL @retval send_ok or send_err as required @note This routine sends the reply back to the caller! @note called by derived class ion.services.dm.cache.RetrieverService @todo Better propagation of HTTP result codes back to callers (eg 404) """ assert (operation in ['GET', 'HEAD']) logging.debug('Fetcher: %s %s' % (operation, src_url)) src = urlparse.urlsplit(src_url) try: conn = http.HTTPConnection(src.netloc) # @bug Need to merge path with query, no canned fn in urlparse lib conn.request(operation, src.path) res = conn.getresponse() except gaierror, ge: logging.exception() yield self.reply_err(msg, content=str(ge)) hstr = self._reassemble_headers(res) # Did it succeed? if res.status == 200: # @note read on HEAD returns no data hstr = hstr + '\n' + res.read() # Uncomment this to see the completed result # logging.debug(hstr) # @note base64-encoded page returned! yield self.reply_ok(msg, content=base64.b64encode(hstr)) else: logging.info('fetch error %s %s %s %s' % (operation, src_url, res.status, res.reason)) yield self.reply_err(msg, content=hstr) logging.debug('fetch completed %s' % res.status)
class StoreService(BaseService): """ Service to store and retrieve key/value pairs. """ # Declaration of service declare = BaseService.service_declare(name='store', version='0.1.0', dependencies=[]) @defer.inlineCallbacks def slc_init(self): # use spawn args to determine backend class, second config file backendcls = self.spawn_args.get( 'backend_class', CONF.getValue('backend_class', default='ion.data.store.Store')) backendargs = self.spawn_args.get( 'backend_args', CONF.getValue('backend_args', default={})) self.backend = None # self.backend holds the class which is instantiated to provide the Store if backendcls: self.backend = pu.get_class(backendcls) else: self.backend = Store assert issubclass(self.backend, IStore) # Now create an instance of the backend class # Provide rest of the spawnArgs to init the store self.store = yield self.backend.create_store(**backendargs) name = self.__class__.__name__ logging.info(name + " initialized") logging.info(name + " backend:" + str(backendcls)) logging.info(name + " backend args:" + str(backendargs)) @defer.inlineCallbacks def op_put(self, content, headers, msg): """ Service operation: Puts a value into the store identified by key. Replies with a result of this operation """ logging.info("op_put: " + str(content)) key = str(content['key']) val = content['value'] res = yield self.store.put(key, val) yield self.reply_ok(msg, {'result': res}) @defer.inlineCallbacks def op_get(self, content, headers, msg): """ Service operation: Gets a value from the store identified by key. """ logging.info("op_get: " + str(content)) key = str(content['key']) val = yield self.store.get(key) yield self.reply_ok(msg, {'value': val}) @defer.inlineCallbacks def op_query(self, content, headers, msg): """ Service operation: Look for multiple values based on a regex on key """ regex = str(content['regex']) res = yield self.store.query(regex) yield self.reply_ok(msg, {'result': res}) @defer.inlineCallbacks def op_remove(self, content, headers, msg): """ Service operation: Delete a value. """ key = str(content['key']) res = yield self.store.remove(key) yield self.reply_ok(msg, {'result': res}) @defer.inlineCallbacks def op_clear_store(self, content, headers, msg): """ Service operation: Delete a value. """ res = yield self.store.clear_store() yield self.reply_ok(msg, {'result': res})
class InstrumentManagementService(BaseService): """ Instrument management service interface. This service provides overall coordination for instrument management within an observatory context. In particular it coordinates the access to the instrument and data product registries and the interaction with instrument agents. """ # Declaration of service declare = BaseService.service_declare(name='instrument_management', version='0.1.0', dependencies=[]) def slc_init(self): self.irc = InstrumentRegistryClient(proc=self) self.dprc = DataProductRegistryClient(proc=self) self.arc = AgentRegistryClient(proc=self) self.dpsc = DataPubsubClient(proc=self) @defer.inlineCallbacks def op_create_new_instrument(self, content, headers, msg): """ Service operation: Accepts a dictionary containing user inputs. Updates the instrument registry. """ userInput = content['userInput'] newinstrument = InstrumentResource.create_new_resource() if 'name' in userInput: newinstrument.name = str(userInput['name']) if 'description' in userInput: newinstrument.description = str(userInput['description']) if 'manufacturer' in userInput: newinstrument.manufacturer = str(userInput['manufacturer']) if 'model' in userInput: newinstrument.model = str(userInput['model']) if 'serial_num' in userInput: newinstrument.serial_num = str(userInput['serial_num']) if 'fw_version' in userInput: newinstrument.fw_version = str(userInput['fw_version']) instrument_res = yield self.irc.register_instrument_instance( newinstrument) yield self.reply_ok(msg, instrument_res.encode()) @defer.inlineCallbacks def op_create_new_data_product(self, content, headers, msg): """ Service operation: Accepts a dictionary containing user inputs. Updates the data product registry. Also sets up an ingestion pipeline for an instrument """ dataProductInput = content['dataProductInput'] newdp = DataProductResource.create_new_resource() if 'instrumentID' in dataProductInput: inst_id = str(dataProductInput['instrumentID']) int_ref = ResourceReference(RegistryIdentity=inst_id, RegistryBranch='master') newdp.instrument_ref = int_ref if 'name' in dataProductInput: newdp.name = str(dataProductInput['name']) if 'description' in dataProductInput: newdp.description = str(dataProductInput['description']) if 'dataformat' in dataProductInput: newdp.dataformat = str(dataProductInput['dataformat']) # Step: Create a data stream ## Instantiate a pubsubclient #self.dpsc = DataPubsubClient(proc=self) # ## Create and Register a topic #self.topic = PubSubTopicResource.create('SBE49 Topic',"oceans, oil spill") #self.topic = yield self.dpsc.define_topic(self.topic) #logging.debug('DHE: Defined Topic') # #self.publisher = PublisherResource.create('Test Publisher', self, self.topic, 'DataObject') #self.publisher = yield self.dpsc.define_publisher(self.publisher) res = yield self.dprc.register_data_product(newdp) ref = res.reference(head=True) yield self.reply_ok(msg, res.encode()) @defer.inlineCallbacks def op_execute_command(self, content, headers, msg): """ Service operation: Execute a command on an instrument. """ # Step 1: Extract the arguments from the UI generated message content commandInput = content['commandInput'] if 'instrumentID' in commandInput: inst_id = str(commandInput['instrumentID']) else: raise ValueError("Input for instrumentID not present") command = [] if 'command' in commandInput: command_op = str(commandInput['command']) else: raise ValueError("Input for command not present") command.append(command_op) arg_idx = 0 while True: argname = 'cmdArg' + str(arg_idx) arg_idx += 1 if argname in commandInput: command.append(str(commandInput[argname])) else: break # Step 2: Find the agent id for the given instrument id agent_pid = yield self.get_agent_pid_for_instrument(inst_id) if not agent_pid: yield self.reply_err( msg, "No agent found for instrument " + str(inst_id)) defer.returnValue(None) # Step 3: Interact with the agent to execute the command iaclient = InstrumentAgentClient(proc=self, target=agent_pid) commandlist = [ command, ] logging.info("Sending command to IA: " + str(commandlist)) cmd_result = yield iaclient.execute_instrument(commandlist) yield self.reply_ok(msg, cmd_result) @defer.inlineCallbacks def op_get_instrument_state(self, content, headers, msg): """ Service operation: . """ # Step 1: Extract the arguments from the UI generated message content commandInput = content['commandInput'] if 'instrumentID' in commandInput: inst_id = str(commandInput['instrumentID']) else: raise ValueError("Input for instrumentID not present") agent_pid = yield self.get_agent_pid_for_instrument(inst_id) if not agent_pid: raise StandardError("No agent found for instrument " + str(inst_id)) iaclient = InstrumentAgentClient(proc=self, target=agent_pid) inst_cap = yield iaclient.get_capabilities() if not inst_cap: raise StandardError("No capabilities available for instrument " + str(inst_id)) ci_commands = inst_cap['ci_commands'] instrument_commands = inst_cap['instrument_commands'] instrument_parameters = inst_cap['instrument_parameters'] ci_parameters = inst_cap['ci_parameters'] values = yield iaclient.get_from_instrument(instrument_parameters) resvalues = {} if values: resvalues = values yield self.reply_ok(msg, resvalues) @defer.inlineCallbacks def op_start_instrument_agent(self, content, headers, msg): """ Service operation: Starts an instrument agent for a type of instrument. """ if 'instrumentID' in content: inst_id = str(content['instrumentID']) else: raise ValueError("Input for instrumentID not present") if 'model' in content: model = str(content['model']) else: raise ValueError("Input for model not present") if model != 'SBE49': raise ValueError("Only SBE49 supported!") agent_pid = yield self.get_agent_pid_for_instrument(inst_id) if agent_pid: raise StandardError("Agent already started for instrument " + str(inst_id)) simulator = Simulator(inst_id) simulator.start() topicname = "Inst/RAW/" + inst_id topic = PubSubTopicResource.create(topicname, "") # Use the service to create a queue and register the topic topic = yield self.dpsc.define_topic(topic) iagent_args = {} iagent_args['instrument-id'] = inst_id driver_args = {} driver_args['port'] = simulator.port driver_args['publish-to'] = topic.RegistryIdentity iagent_args['driver-args'] = driver_args iapd = ProcessDesc( **{ 'name': 'SBE49IA', 'module': 'ion.agents.instrumentagents.SBE49_IA', 'class': 'SBE49InstrumentAgent', 'spawnargs': iagent_args }) iagent_id = yield self.spawn_child(iapd) iaclient = InstrumentAgentClient(proc=self, target=iagent_id) yield iaclient.register_resource(inst_id) yield self.reply_ok(msg, "OK") @defer.inlineCallbacks def op_stop_instrument_agent(self, content, headers, msg): """ Service operation: Starts direct access mode. """ yield self.reply_err(msg, "Not yet implemented") @defer.inlineCallbacks def op_start_direct_access(self, content, headers, msg): """ Service operation: Starts direct access mode. """ yield self.reply_err(msg, "Not yet implemented") @defer.inlineCallbacks def op_stop_direct_access(self, content, headers, msg): """ Service operation: Stops direct access mode. """ yield self.reply_err(msg, "Not yet implemented") @defer.inlineCallbacks def get_agent_desc_for_instrument(self, instrument_id): logging.info("get_agent_desc_for_instrument() instrumentID=" + str(instrument_id)) int_ref = ResourceReference(RegistryIdentity=instrument_id, RegistryBranch='master') agent_query = InstrumentAgentResourceInstance() agent_query.instrument_ref = int_ref if not agent_res: defer.returnValue(None) agent_pid = agent_res.proc_id logging.info("Agent process id for instrument id %s is: %s" % (instrument_id, agent_pid)) defer.returnValue(agent_pid) @defer.inlineCallbacks def get_agent_for_instrument(self, instrument_id): logging.info("get_agent_for_instrument() instrumentID=" + str(instrument_id)) int_ref = ResourceReference(RegistryIdentity=instrument_id, RegistryBranch='master') agent_query = InstrumentAgentResourceInstance() agent_query.instrument_ref = int_ref # @todo Need to list the LC state here. WHY??? agent_query.lifecycle = LCStates.developed agents = yield self.arc.find_registered_agent_instance_from_description( agent_query, regex=False) logging.info("Found %s agent instances for instrument id %s" % (len(agents), instrument_id)) agent_res = None if len(agents) > 0: agent_res = agents[0] defer.returnValue(agent_res) @defer.inlineCallbacks def get_agent_pid_for_instrument(self, instrument_id): agent_res = yield self.get_agent_for_instrument(instrument_id) if not agent_res: defer.returnValue(None) agent_pid = agent_res.proc_id logging.info("Agent process id for instrument id %s is: %s" % (instrument_id, agent_pid)) defer.returnValue(agent_pid)
class IngestionService(BaseService): """Ingestion service interface @Note Needs work - Should create a subscription to ingest a data source What should the service interface look like for this? """ # Declaration of service declare = BaseService.service_declare(name='ingestion_service', version='0.1.0', dependencies=[]) @defer.inlineCallbacks def slc_init(self): self.reg = yield ingestion_registry.IngestionRegistryClient(proc=self) self.pubsub = yield pubsub_service.DataPubsubClient(proc=self) self.preserv = yield preservation_service.PreservationClient(proc=self) self.datareg = yield data_registry.DataRegistryClient(proc=self) @defer.inlineCallbacks def op_create_ingestion_datastream(self, content, headers, msg): """Service operation: declare new named datastream for ingestion 0) Decode content to Igestion Stream Resource 1) create inbound topic 2) create ingested topic 3) Register new data 3) start preservation of ingested 4) start preservation of inbound 5) start ingestion workflow Register progress and set lcstate along the way return the ingestiondatastream resource """ logging.debug(self.__class__.__name__ +', op_'+ headers['op'] +' Received: ' + str(headers)) isr = dataobject.DataObject.decode(content) logging.info(self.__class__.__name__ + ' recieved: op_'+ headers['op'] +', Ingestion Stream: \n' + str(publisher)) # Register the intended feed... isr = yield self.reg.define_ingestion_stream(isr) inbnd = irs.name + '.inbound' topic = PubSubTopicResource.create(name=inbnd,keywords='input') # Place holder - what to put, NOT RAW! topic = yield self.pubsub.define_topic(topic) isr.input_topic = topic.reference(heat=True) ingested = irs.name + '.ingested' topic = PubSubTopicResource.create(name=ingested,keywords='ingested') topic = yield self.pubsub.define_topic(topic) isr.ingested_topic = topic.reference(heat=True) # Register the feed topics... isr = yield self.reg.define_ingestion_stream(isr) #Create a DM Data Resource for this new feed. dmdr = DMDataResource.create_new_resource() dmdr.input_topic = isr.input_topic dmdr.ingested_topic = isr.ingested_topic # Register the data dmdr = yield self.datareg.define_data(dmdr) #Call the preservation service for the input stream arc = ArchiveResource.create_new_resource() arc.dmdataresource = dmdr.reference(head=true) arc.topic = isr.input_topic yield self.preserv.create_archive(arc) yield self.preserv.activate_persister # Register the data dmdr.input_archive = arc.reference(head=True) dmdr = yield self.datareg.define_data(dmdr) #Call the preservation service for the ingested stream arc = ArchiveResource.create_new_resource() arc.dmdataresource = dmdr.reference(head=true) arc.topic = isr.ingested_topic yield self.preserv.create_archive(arc) yield self.preserv.activate_persister # Register the data dmdr.ingested_archive = arc.reference(head=True) dmdr = yield self.datareg.define_data(dmdr) # Register the persisters isr.persisting_inout = True isr.persisting_ingested = True isr = yield self.reg.define_ingestion_stream(isr) #activate ingestion isr = yield self._activate_ingestion(isr, dmdr) isr = yield self.reg.define_ingestion_stream(isr) yield self.reply_ok(isr.encode()) @defer.inlineCallbacks def _activate_ingestion(isr, data_resource): subscription = SubscriptionResource() subscription.topic1 = t_search # More to do - tell the ingester what to do with results - what dmdr to update etc... subscription.workflow = {'consumer1':{'module':'path.to.module','consumerclass':'<ConsumerClassName>',\ 'attach':'topic1',\ 'process parameters':{'param1':'my parameter'}}} #subscription = yield self.pubsub.create_consumer_args(subscription) isr.ingesting = True defer.returnValue(isr)
class JavaIntegrationService(BaseService): """ Example service interface """ # Declaration of service declare = BaseService.service_declare(name='javaint', version='0.1.0', dependencies=[]) def __init__(self, receiver, spawnArgs=None): # Service class initializer. Basic config, but no yields allowed. BaseService.__init__(self, receiver, spawnArgs) logging.info('JavaIntegrationService.__init__()') def slc_init(self): # Service life cycle state. Initialize service here. Can use yields. #self.instruments = ["SBE49_1","SBE49_2"] #self.instruments = {'Instrument1':{'a':'1','b':'2','c':'3'},'Instrument2':{'e':'1','f':'2','g':'e'}} #self.instruments = [{'id':343,'name':'Instrument3','manufacturer':'Sony','modelNumber':'1.3','instrumentType':'aquatic','versionNumber':'232'},{'id':938,'name':'Instrument1','manufacturer':'2','modelNumber':'3','instrumentType':'a2','versionNumber':'za'},{'id':1029,'name':'Instrument2','manufacturer':'242','modelNumber':'333','instrumentType':'a2asd','versionNumber':'asdf'}] self.instruments = [{'name':'Instrument3','manufacturer':'Sony','modelNumber':'1.3','instrumentType':'aquatic','versionNumber':'232'},{'name':'Instrument1','manufacturer':'2','modelNumber':'3','instrumentType':'a2','versionNumber':'za'},{'name':'Instrument2','manufacturer':'242','modelNumber':'333','instrumentType':'a2asd','versionNumber':'asdf'}] self.datasets = [{'name':'Data product 1','instrumentType':'Atomic'},{'name':'Data product 2','instrumentType':'Nuclear'},{'name':'Data product 3','instrumentType':'Data Collection'}] self.services = [{'name':'Instrument Registry Service','status':'active'},{'name':'Data Product Service','status':'active'},{'name':'Service X','status':'active'}] @defer.inlineCallbacks def op_hello(self, content, headers, msg): logging.info('op_hello: '+str(content)) # The following line shows how to reply to a message yield self.reply_ok(msg, {'value':'Hello there, '+str(content)}, {}) @defer.inlineCallbacks def op_list_all_instruments(self, content, headers, msg): logging.info('op_list_all_instruments: '+str(content)) # The following line shows how to reply to a message yield self.reply_ok(msg, {'value':self.instruments}) @defer.inlineCallbacks def op_register_instrument(self, content, headers, msg): logging.info('op_register_instrument: '+ str(content)) newInstrument = ast.literal_eval(str(content)) # get instrument name self.instruments.append(newInstrument) # The following line shows how to reply to a message yield self.reply_ok(msg, {'value':self.instruments}) @defer.inlineCallbacks def op_list_all_datasets(self, content, headers, msg): logging.info('op_list_all_datasets: '+str(content)) # The following line shows how to reply to a message yield self.reply_ok(msg, {'value':self.datasets} ) @defer.inlineCallbacks def op_register_dataset(self, content, headers, msg): logging.info('op_register_dataset: '+ str(content)) newDataset = ast.literal_eval(str(content)) self.datasets.append(newDataset) # The following line shows how to reply to a message yield self.reply_ok(msg, {'value':self.datasets}) @defer.inlineCallbacks def op_list_all_services(self, content, headers, msg): logging.info('op_list_all_services: '+str(content)) # The following line shows how to reply to a message yield self.reply_ok(msg, {'value':self.services}) @defer.inlineCallbacks def op_register_service(self, content, headers, msg): logging.info('op_register_service: '+ str(content)) newService = ast.literal_eval(str(content)) self.services.append(newService) # The following line shows how to reply to a message yield self.reply_ok(msg, {'value':self.services})
class PreservationService(BaseService): """Preservation Service interface """ # Declaration of service declare = BaseService.service_declare(name='preservation_service', version='0.1.0', dependencies=[]) @defer.inlineCallbacks def slc_init(self): self.reg = yield preservation_registry.PreservationRegistryClient( proc=self) @defer.inlineCallbacks def op_create_archive(self, content, headers, msg): """Service operation: define a new archive object """ #arc = dm_resource_descriptions.ArchiveResource.create_new_resource() # Set stuff in arc... #Content is a topic + metadata # Pick a file name # set default policy # touch the file # Allocate storage? # Register new Archive arc = yield self.reg.define_archive(arc) @defer.inlineCallbacks def op_activate_archive_persister(self, content, headers, msg): """Service operation: create process to archive a data stream """ # Spawn persister to topic/file name persister = { 'name': 'persister 1', # Give it a new name? 'module': 'ion.services.dm.preservation.persister', 'procclass': 'Persister', 'spawnargs': { 'attach': [topic.queue.name], 'process parameters': { 'fname': arc.name } } } child1 = base_consumer.ConsumerDesc(**pd1) child1_id = yield self.test_sup.spawn_child(child1) def op_deactivate_archive_persister(self, content, headers, msg): """Service operation: kill data stream archive process """ # kill persister - Not before LCA def op_archive_data(self, content, headers, msg): """Service operation: archive an single dataset """ # Spawn persister to topic/file name def op_set_archive_cache_policy(self, content, headers, msg): """Service operation: set the cache policy for an archive """ def op_set_archive_backup_policy(self, content, headers, msg): """Service operation: set backup policy for an archive """ def op_set_archive_long_term_policy(self, content, headers, msg): """Service operation: set the long term policy for an archive
class DataPubsubService(BaseService): """ @Brief Service for Publicaiton and Subscription to topics """ # Declaration of service declare = BaseService.service_declare(name='data_pubsub', version='0.1.0', dependencies=[]) @defer.inlineCallbacks def slc_init(self): # Is this the proper way to start a client in a service? self.reg = yield pubsub_registry.DataPubsubRegistryClient(proc=self) @defer.inlineCallbacks def op_define_topic(self, content, headers, msg): """Service operation: Register a "topic" that can be published on and that can be subscribed to. Note: this has no direct connection to any AMQP topic notion. A topic is basically a data stream. """ logging.debug(self.__class__.__name__ + ', op_' + headers['op'] + ' Received: ' + str(headers)) topic = dataobject.Resource.decode(content) if topic.RegistryIdentity: yield self.update_topic_registration(topic) else: # it is a new topic and must be declared topic = yield self.create_and_register_topic(topic) logging.info(self.__class__.__name__ + ' recieved: op_' + headers['op'] + ', topic: \n' + str(topic)) #@todo call some process to update all the subscriptions? Or only on interval? if topic: logging.info(self.__class__.__name__ + ': op_' + headers['op'] + ' Success!') yield self.reply_ok(msg, topic.encode()) else: logging.info(self.__class__.__name__ + ': op_' + headers['op'] + ' Failed!') yield self.reply_err(msg, None) #@defer.inlineCallback # call back not needed def update_topic_registration(self, topic): #topic = yield self.reg.register(topic) return self.reg.register(topic) @defer.inlineCallbacks def create_and_register_topic(self, topic): """ Create a messaging name and set the queue properties to create it. @TODO fix the hack - This should come from the exchange registry! """ logging.info(self.__class__.__name__ + '; Declaring new Topic & Creating Queue.') # Give the topic anidentity topic.create_new_reference() # Declare the queue - this should be done in the exchange registry # Create a new queue object queue = yield self.create_queue() topic.queue = queue topic = yield self.reg.register(topic) defer.returnValue(topic) @defer.inlineCallbacks def create_queue(self): """ Create a queue @TODO fix the hack - This should come from the exchange registry! """ queue = dm_resource_descriptions.Queue() queue.name = dataobject.create_unique_identity() queue.type = 'fanout' queue.args = {'scope': 'global'} queue_properties = { queue.name: { 'name_type': queue.type, 'args': queue.args } } # This should come from the COI Exchange registry yield bootstrap.declare_messaging(queue_properties) defer.returnValue(queue) @defer.inlineCallbacks def op_define_publisher(self, content, headers, msg): """Service operation: Register a publisher that subsequently is authorized to publish on a topic. """ logging.debug(self.__class__.__name__ + ', op_' + headers['op'] + ' Received: ' + str(headers)) publisher = dataobject.Resource.decode(content) logging.info(self.__class__.__name__ + ' recieved: op_' + headers['op'] + ', publisher: \n' + str(publisher)) publisher = yield self.reg.register(publisher) if publisher: logging.info(self.__class__.__name__ + ': op_' + headers['op'] + ' Success!') yield self.reply_ok(msg, publisher.encode()) else: logging.info(self.__class__.__name__ + ': op_' + headers['op'] + ' Failed!') yield self.reply_err(msg, None) @defer.inlineCallbacks def op_define_subscription(self, content, headers, msg): """Service operation: Register a subscriber's intent to receive subscriptions on a topic, with the workflow described to produce the desired result """ # Subscribe should decouple the exchange point where data is published # and the subscription exchange point where a process receives it. # That allows for an intermediary process to filter it. logging.debug(self.__class__.__name__ + ', op_' + headers['op'] + ' Received: ' + str(headers)) subscription = dataobject.Resource.decode(content) logging.info(self.__class__.__name__ + ' recieved: op_' + headers['op'] + ', subscription: \n' + str(subscription)) if subscription.RegistryIdentity: # it is an existing topic and must be updated subscription = yield self.update_subscription(subscription) else: subscription = yield self.create_subscription(subscription) subscription = yield self.reg.register(subscription) if subscription: logging.info(self.__class__.__name__ + ': op_' + headers['op'] + ' Success!') yield self.reply_ok(msg, subscription.encode()) else: logging.info(self.__class__.__name__ + ': op_' + headers['op'] + ' Failed!') yield self.reply_err(msg, None) """ Subscription Object: Name - inherited # hack for now to allow naming one-three more topic descriptions topic1 = TypedAttribute(PubSubTopicResource) topic2 = TypedAttribute(PubSubTopicResource) topic3 = TypedAttribute(PubSubTopicResource) workflow = TypedAttribute(dict) ''' <consumer name>:{'module':'path.to.module','cosumeclass':'<ConsumerClassName>',\ 'attach':(<topicX>) or (<consumer name>, <consumer queue keyword>) or <list of consumers and topics>,\ 'Process Parameters':{<conumser property keyword arg>: <property value>}} ''' """ @defer.inlineCallbacks def create_subscription(self, subscription): ''' ''' subscription.create_new_reference() subscription = yield self.create_consumer_args(subscription) consumer_args = subscription.consumer_args for name, args in consumer_args.items(): child = base_consumer.ConsumerDesc(**args) child_id = yield self.spawn_child(child) #subscription.consumer_procids[name]=child_id subscription = yield self.reg.register(subscription) defer.returnValue(subscription) @defer.inlineCallbacks def create_consumer_args(self, subscription): ''' @Brief Turn the workflow argument into the arguments used to create a new consumer. This includes spawning the queues needed to deliver the product of one consumer to the attachment point of another! @Note This is probably too complex - it should be refactored into seperate methods where possible. ''' logging.info('Processing Subscription Workflow:' + str(subscription.workflow)) #A for each topic1, topic2, topic3 get the list of queues topics = {'topic1': [], 'topic2': [], 'topic3': []} if subscription.topic1.name or subscription.topic1.keywords: topics['topic1'] = yield self.find_topics(subscription.topic1) if subscription.topic2.name or subscription.topic2.keywords: topics['topic2'] = yield self.find_topics(subscription.topic2) if subscription.topic3.name or subscription.topic3.keywords: topics['topic3'] = yield self.find_topics(subscription.topic3) #B process the workflow dictionary, #0) Check for valid workflow # how can I check whether the workflow is a loop ? difficult! # 1) create new queues for workflow # 2) create ConsumerDesc for each consumer # create place holders for each consumers spawn args subscription_queues = [] consumers = {} consumer_names = subscription.workflow.keys() for consumer, args in subscription.workflow.items(): spargs = { 'attach': [], 'process parameters': args.get('process parameters', {}), 'delivery interval': args.get('delivery interval', None), 'delivery queues': args.get('delivery queues', {}) } for k, v in spargs['delivery queues']: topic = topics.get(v, None) if not topic: raise RuntimeError('Invalid delivery queue specified ') spargs['delivery queues'][k] = topic.queue.name cd = {} cd['name'] = str(consumer) cd['module'] = str(args.get('module')) cd['procclass'] = str(args.get('consumerclass')) cd['spawnargs'] = spargs consumers[consumer] = cd #logging.info('CONSUMERS:' + str(consumers)) # for each consumer and add its attachements and create delivery queues for consumer, args in subscription.workflow.items(): # list of queues to attach to for this consumer attach = [] # Get the attach to topic or consumer - make it a list and iterate attach_to = args['attach'] if not hasattr(attach_to, '__iter__'): attach_to = [attach_to] for item in attach_to: #Each item may be a topic or a consumer/keyword pair - make it a list # and extract the contents if not hasattr(item, '__iter__'): item = [item] if len(item) == 1: name = item[0] keyword = None elif len(item) == 2: name = item[0] keyword = item[1] else: raise RuntimeError('Invalid attach argument!') # is it a topic? if name in topics.keys(): topic_list = topics[name] # get the list of topics # add each queue to the list of attach_to for topic in topic_list: # Add it to the list for this consumer attach.append(topic.queue.name) # Add it to the list for this subscription subscription_queues.append(topic.queue) logging.info( '''Consumer '%s' attaches to topic name '%s' ''' % (consumer, topic.name)) # See if it is consuming another resultant elif name in consumer_names: # Could fail badly! # Does this producer already have a queue? # This is not 'safe' - lots of ways to get a key error! q = consumers[name]['spawnargs']['delivery queues'].get( keyword, None) if q: attach.append(q) logging.info( '''Consumer '%s' attaches to existing queue for producer/keyword: '%s'/'%s' ''' % (consumer, name, keyword)) else: # Create the queue! #@TODO - replace with call to exchange registry service queue = yield self.create_queue() # Add it to the list for this consumer attach.append(queue.name) # Add it to the list for this subscription subscription_queues.append(queue) # Add to the delivery list for the producer... consumers[name]['spawnargs']['delivery queues'][ keyword] = str(queue.name) logging.info( '''Consumer '%s' attaches to new queue for producer/keyword: '%s'/'%s' ''' % (consumer, name, keyword)) else: raise RuntimeError( '''Can not determine how to attach consumer '%s' \ to topic or consumer '%s' ''' % (consumer, item)) consumers[consumer]['spawnargs']['attach'].extend(attach) logging.info('CONSUMERS:' + str(consumers)) subscription.queues = subscription_queues subscription.consumer_args = consumers defer.returnValue(subscription) #@defer.inlineCallbacks def update_subscription(subscription): ''' ''' # Determine the difference between the current and existing subscription # act accordingly - but very difficult to figure out what to do! raise RuntimeError('Update Subscription is not yet implemented') pass def op_unsubscribe(self, content, headers, msg): """Service operation: Stop one's existing subscription to a topic. And remove the Queue if no one else is listening... """ @defer.inlineCallbacks def op_publish(self, content, headers, msg): """Service operation: Publish data message on a topic """ logging.debug(self.__class__.__name__ + ', op_' + headers['op'] + ' Received: ' + str(headers)) publication = dataobject.Resource.decode(content) logging.info(self.__class__.__name__ + ' recieved: op_' + headers['op'] + ', publication') #logging.info(self.__class__.__name__ + ' recieved: op_'+ headers['op'] +', publication: \n' + str(publication)) #Get the data data = publication.data # Get the Topic topic_ref = publication.topic_ref topic = yield self.reg.get(topic_ref.reference(head=True)) if not topic: logging.info(self.__class__.__name__ + ' recieved: op_' + headers['op'] + ', topic invalid!') yield self.reply_err(msg, 'Topic does not exist') return #Check publisher is valid! publisher = dm_resource_descriptions.PublisherResource() publisher.publisher = publication.publisher # Get the publications which this process is registered fro reg_pubs = yield self.reg.find(publisher, regex=False, attnames=['publisher']) valid = False for pub in reg_pubs: if topic.reference(head=True) in pub.topics: valid = True logging.debug(self.__class__.__name__ + '; Publishing to topic: \n' + str(topic)) break if not valid: logging.info(self.__class__.__name__ + ' recieved: op_' + headers['op'] + ', publisher not registered for topic!') yield self.reply_err(msg, 'Publisher not registered for topic!') return if not data.notification: data.notification = 'Data topic: ' + topic.name data.timestamp = pu.currenttime() # Todo: impersonate message as from sender - #@Todo - Move the actual publish command back to the client side! yield self.send(topic.queue.name, 'data', data.encode(), {}) logging.info(self.__class__.__name__ + ': op_' + headers['op'] + ' Success!') yield self.reply_ok(msg, '') #@defer.inlineCallbacks def find_topics(self, topic_description): """Service operation: For a given resource, find the topic that contains updates to the resource or resource description. @Notes - should this create a topic if none yet exist? """ #Note - AOI does not do anything yet and keyword needs to be improved... #topic_list = yield self.reg.find(topic_description,regex=True, attnames=['name','keywords','aoi']) return self.reg.find(topic_description, regex=True, attnames=['name', 'keywords', 'aoi'])