setattr(control.cc, r.label, r.procinst) control.cc.ps = ps def nodes(): nodes = {} for c in self.containers.values(): nodes[str(c['node'])] = 1 return nodes.keys() control.cc.nodes = nodes control.cc.cont = lambda: [str(k) for k in self.containers.keys()] control.cc.info = lambda: self.containers[str(Container.id)] control.cc.identify = lambda: self.send(self.ann_name, 'identify', '', {'quiet': True}) control.cc.getinfo = lambda n: self.send(_get_node(n), 'get_info', '') control.cc.ping = lambda n: self.send(_get_node(n), 'ping', '', {'quiet': True}) control.cc.help = "CC Helpers. ATTRS: agent, config, pids, svcs, procs, help FUNC: send, rpc_send, spawn, svc, ps, nodes, cont, info, identify, ping" # Spawn of the process using the module name factory = ProtocolFactory(CCAgent) """ twistd -n --pidfile t1.pid magnet -h amoeba.ucsd.edu -a sysname=mm res/scripts/newcc.py twistd -n --pidfile t2.pid magnet -h amoeba.ucsd.edu -a sysname=mm res/scripts/newcc.py send (2, {'op':'identify','content':''}) send (2, {'op':'spawn','content':{'module':'ion.play.hello_service'}}) """
@brief service for provisioning new VM instances """ import logging logging = logging.getLogger(__name__) from twisted.internet import defer from magnet.spawnable import Receiver import ion.util.procutils as pu from ion.core.base_process import ProtocolFactory from ion.services.base_service import BaseService, BaseServiceClient class ProvisionerService(BaseService): """Provisioner service interface """ # Declaration of service declare = BaseService.service_declare(name='provisioner', version='0.1.0', dependencies=[]) def op_provision(self, content, headers, msg): """Service operation: Provision a taskable resource """ def op_terminate(self, content, headers, msg): """Service operation: Terminate a taskable resource """ # Spawn of the process using the module name factory = ProtocolFactory(ProvisionerService)
class ProxyService(BaseService): """ Proxy service. Stub, really, since the proxy listens on a plain tcp port. """ # Declaration of service declare = BaseService.service_declare(name='proxy', version='0.1.0', dependencies=['controller']) @defer.inlineCallbacks def slc_init(self): """ Use this hook to bind to listener TCP port. """ logging.info('starting proxy on port %d' % PROXY_PORT) self.proxy_port = yield reactor.listenTCP(PROXY_PORT, DAPProxyFactory()) logging.info('Proxy listener running.') @defer.inlineCallbacks def slc_shutdown(self): """ Close TCP listener """ logging.info('Shutting down proxy') yield self.proxy_port.stopListening() factory = ProtocolFactory(ProxyService)
Class for the client accessing the data product registry. """ def __init__(self, proc=None, **kwargs): if not 'targetname' in kwargs: kwargs['targetname'] = "data_product_registry" BaseServiceClient.__init__(self, proc, **kwargs) def clear_registry(self): return self.base_clear_registry('clear_data_product_registry') def register_data_product(self, data_product_instance): """ Client method to Register a Resource instance """ return self.base_register_resource('register_data_product', data_product_instance) def get_data_product(self, data_product_instance_reference): """ Get a resource instance """ return self.base_get_resource('get_data_product', resource_reference) def find_data_product(self, description, regex=True, ignore_defaults=True): return self.base_find_resource('find_data_product', description, regex, ignore_defaults) # Spawn of the process using the module name factory = ProtocolFactory(DataProductRegistryService)
op_register = registry.BaseRegistryService.base_register_resource """ Service operation: Create or update a pubsub. """ op_get = registry.BaseRegistryService.base_get_resource """ Service operation: Get pubsub """ op_find = registry.BaseRegistryService.base_find_resource """ Service operation: Find pubsub by characteristics """ # Spawn of the process using the module name factory = ProtocolFactory(DataPubsubRegistryService) class DataPubsubRegistryClient(registry.BaseRegistryClient): """ @Brief Class for the client accessing the Data PubSub Registry. @Todo clean up the interface for specific pubsub resource objects Need to specify topic, publisher, subscriber """ def __init__(self, proc=None, **kwargs): if not 'targetname' in kwargs: kwargs['targetname'] = "datapubsub_registry" BaseServiceClient.__init__(self, proc, **kwargs) def clear_registry(self): return self.base_clear_registry('clear_registry')
str(pu.currenttime_ms())) class WorkerClient(BaseProcess): """Class for the client accessing the object store. """ def __init__(self, *args): BaseProcess.__init__(self, *args) self.workresult = {} self.worker = {} def op_result(self, content, headers, msg): ts = pu.currenttime_ms() logging.info("Work result received " + str(content) + " at " + str(ts)) workid = content['work-id'] worker = headers['sender'] self.workresult[workid] = ts if worker in self.worker: wcnt = self.worker[worker] + 1 else: wcnt = 1 self.worker[worker] = wcnt @defer.inlineCallbacks def submit_work(self, to, workid, work): yield self.send(str(to), 'work', {'work-id': workid, 'work': work}, {}) # Spawn of the process using the module name factory = ProtocolFactory(WorkerProcess)
def __translator(input): """ A function (to be returned upon request) that will translate the very raw data from the instrument into the common archive format """ return input @defer.inlineCallbacks def op_get_translator(self, content, headers, msg): """ Return the translator function that will convert the very raw format of the instrument into a common OOI repository-ready format """ yield self.reply_err(msg, "Not Implemented!") # yield self.reply_ok(msg, self.__translator) @defer.inlineCallbacks def op_get_capabilities(self, content, headers, msg): """ Obtain a list of capabilities that this instrument has. This is simply a command and parameter list at this point """ yield self.reply(msg, 'get_capabilities', {IA.instrument_commands: const.instrument_commands, IA.instrument_parameters: const.instrument_parameters, IA.ci_commands: const.ci_commands, IA.ci_parameters: const.ci_parameters}, {}) # Spawn of the process using the module name factory = ProtocolFactory(SBE49InstrumentAgent)
yield self._shutdown_processes() class EchoProcess(BaseProcess): @defer.inlineCallbacks def plc_noinit(self): logging.info("In init: " + self.proc_state) yield pu.asleep(1) logging.info("Leaving init: " + self.proc_state) @defer.inlineCallbacks def op_echo(self, content, headers, msg): logging.info("Message received: " + str(content)) yield self.reply_ok(msg, content) @defer.inlineCallbacks def op_echofail1(self, content, headers, msg): logging.info("Message received: " + str(content)) ex = RuntimeError("I'm supposed to fail") yield self.reply_err(msg, ex) @defer.inlineCallbacks def op_echofail2(self, content, headers, msg): logging.info("Message received: " + str(content)) raise RuntimeError("I'm supposed to fail") yield self.reply_ok(msg, content) # Spawn of the process using the module name factory = ProtocolFactory(EchoProcess)
"""Service operation: """ """ Begin experimental methods RU create backend module.py files if needed. keep the "business logic" separate from the message interface """ def op_authenticate_user(self, content, headers, msg): """ RU Service operation: . """ def op_authenticate_service_provider(self, content, headers, msg): """ RU Service operation: . """ def op_add_service_provider(self, content, headers, msg): """ RU Service operation: . """ def op_update_service_provider(self, content, headers, msg): """ RU Service operation: . """ def op_revoke_service_provider(self, content, headers, msg): """ RU Service operation: . """ # Spawn of the process using the module name factory = ProtocolFactory(AuthorizationService)
self.port = reactor.listenTCP(port, self.site) self.root.print_string = None self.values = [] logging.info('Website started') @defer.inlineCallbacks def plc_shutdown(self): logging.info('Shutdown triggered') if self.port: yield self.port.stopListening() #@defer.inlineCallbacks def ondata(self, data, notification, timestamp, **args): """ """ logging.info('Updating google viz chart data!') if isinstance(data, str): self.root.print_string = data else: raise RuntimeError( 'Invalid data (Not a String) passed to WebVizConsumer ondata method!' ) logging.info('Update complete!') # Spawn of the process using the module name factory = ProtocolFactory(WebVizConsumer)
ingest data from a stream and update registry """ @defer.inlineCallbacks def customize_consumer(self): logging.info('Setting up Ingestion Consumer') ref_str = self.params.get('data_resource_ref') enc_ref = json.loads(ref_str) self.params['data_resource_ref'] = dataobject.DataObject.decode(enc_ref) self.datareg = yield data_registry.DataRegistryClient(proc=self) #@defer.inlineCallbacks def ondata(self, data, notification, timestamp, data_resource_ref): ''' examine data extract meta data get data resource put updated data resource ''' pass # Spawn of the process using the module name factory = ProtocolFactory(IngestionConsumer)
def op_set_archive_cache_policy(self, content, headers, msg): """Service operation: set the cache policy for an archive """ def op_set_archive_backup_policy(self, content, headers, msg): """Service operation: set backup policy for an archive """ def op_set_archive_long_term_policy(self, content, headers, msg): """Service operation: set the long term policy for an archive """ # Spawn of the process using the module name factory = ProtocolFactory(PreservationService) class PreservationClient(BaseServiceClient): def __init__(self, proc=None, **kwargs): if not 'targetname' in kwargs: kwargs['targetname'] = 'preservation_service' BaseServiceClient.__init__(self, proc, **kwargs) def create_archive(self, data_reg_resource): ''' @Brief create a new archive @param dataresource is a DM Data Resource which is registered @return IngestionDataStreamResource object '''
class LatestConsumer(base_consumer.BaseConsumer): """ This is an simple consumer to forward the latest messages at most once every 'delivery interval' to 'queues' from spawn_args, 'delivery queues' which is passed as **kwargs to ondata """ def ondata(self, data, notification, timestamp, queues=[]): # Wipe the list of messages to send then add the latest ones self.msgs_to_send = [] # Queue new messages to send if not hasattr(queues, '__iter__'): queues = [queues] for queue in queues: self.queue_result(queue, data, notification) def onschedule(self, **kwargs): ''' This method is called when it is time to actually send the results in this case it is not needed, but must be over-riden... ''' pass # Spawn of the process using the module name factory = ProtocolFactory(LatestConsumer)
from ion.core.base_process import ProtocolFactory from ion.services.base_service import BaseService, BaseServiceClient class StateRepositoryService(BaseService): """Repository for service state service interface. Service state is information shared between many processes. """ # Declaration of service declare = BaseService.service_declare(name='state_repository', version='0.1.0', dependencies=[]) def op_define_state(self, content, headers, msg): """Service operation: Create a new state object (session) or update an existing one by replacing """ def op_update_state(self, content, headers, msg): """Service operation: Provide an incremental update to the service state. """ def op_retrieve_state(self, content, headers, msg): """Service operation: TBD """ # Spawn of the process using the module name factory = ProtocolFactory(StateRepositoryService)
def set_identity_lcstate_retired(self, identity_reference): return self.set_identity_lcstate(identity_reference, dataobject.LCStates.retired) def set_identity_lcstate_developed(self, identity_reference): return self.set_identity_lcstate(identity_reference, dataobject.LCStates.developed) def set_identity_lcstate_commissioned(self, identity_reference): return self.set_identity_lcstate(identity_reference, dataobject.LCStates.commissioned) class IdentityRegistryService(BaseRegistryService): # Declaration of service declare = BaseService.service_declare(name='identity_service', version='0.1.0', dependencies=[]) op_clear_identity_registry = BaseRegistryService.base_clear_registry op_register_user = BaseRegistryService.base_register_resource op_update_user = BaseRegistryService.base_register_resource op_get_user = BaseRegistryService.base_get_resource op_set_identity_lcstate = BaseRegistryService.base_set_resource_lcstate op_find_users = BaseRegistryService.base_find_resource # Spawn of the process using the module name factory = ProtocolFactory(IdentityRegistryService)
class TaskableResourceRegistryService(BaseService): """Taskable resource registry and definition repository service interface """ # Declaration of service declare = BaseService.service_declare(name='taskable_resource_registry', version='0.1.0', dependencies=[]) def op_define_resource(self, content, headers, msg): """Service operation: Create or update taskable resource description """ def op_find_resource(self, content, headers, msg): """Service operation: Create or update taskable resource description """ def op_store_resource(self, content, headers, msg): """Service operation: Store the definition of a taskable resource, e.g. source code, virtual machine image (or a pointer to it) """ def op_retrieve_resource(self, content, headers, msg): """Service operation: Retrieve the definition of a taskable resource """ # Spawn of the process using the module name factory = ProtocolFactory(TaskableResourceRegistryService)
dataset.data = DapUnpacker(xdrdata, dataset).getvalue() logging.debug('pydap object creation complete') fname = generate_filename(source_url, local_dir=local_dir) logging.info('Saving DAP dataset "%s" to "%s"' % (source_url, fname)) try: netcdf.save(dataset, fname) except UnicodeDecodeError, ude: logging.exception('save error: %s ' % ude) return 1 ''' # Must change name after refactor factory = ProtocolFactory(PersisterConsumer) ''' There is no client for a consumer... class PersisterClient(BaseServiceClient): # Not really needed - consumers don't have clients def __init__(self, proc=None, **kwargs): if not 'targetname' in kwargs: kwargs['targetname'] = 'persister' BaseServiceClient.__init__(self, proc, **kwargs) @defer.inlineCallbacks def persist_dap_dataset(self, dap_message): """ @brief Invoke persister, assumes a single dataset per message @param dap_message Message with das/dds/dods in das/dds/dods keys @retval ok or error via rpc mechanism """
version='0.1.0', dependencies=[]) def __init__(self, receiver, spawnArgs=None): # Service class initializer. Basic config, but no yields allowed. BaseService.__init__(self, receiver, spawnArgs) self.spawn_args['backend_class'] = self.spawn_args.get( 'backend_class', CONF.getValue('backend_class', default='ion.data.store.Store')) self.spawn_args['backend_args'] = self.spawn_args.get( 'backend_args', CONF.getValue('backend_args', default={})) logging.info('AttributeStoreService.__init__()') class AttributeStoreClient(store_service.StoreServiceClient): """ Class for the client accessing the attribute store via Exchange The Implementation is in ion.data.backends.store_service The client provides the IStore interface """ def __init__(self, proc=None, **kwargs): if not 'targetname' in kwargs: kwargs['targetname'] = "attributestore" BaseServiceClient.__init__(self, proc, **kwargs) # Spawn of the process using the module name factory = ProtocolFactory(AttributeStoreService)
except KeyError, ke: logging.exception('missing header!') raise ke return new_headers @defer.inlineCallbacks def forward_get_url(self, content, headers): """ Forward a message to the fetcher. Reach in and rewrite the reply-to and conversation id headers, so that the fetcher can reply directly to the proxy and bypass the coordinator. """ yield self._check_init() logging.debug('Fetcher forwarding URL') yield self.send('get_url', content, self._rewrite_headers(headers)) @defer.inlineCallbacks def forward_get_dap_dataset(self, content, headers): """ Same as forward_get_url, different verb. """ yield self._check_init() yield self.send('get_dap_dataset', content, self._rewrite_headers(headers)) # If loaded as a module, spawn the process factory = ProtocolFactory(FetcherService)
def op_register_dataset(self, content, headers, msg): logging.info('op_register_dataset: '+ str(content)) newDataset = ast.literal_eval(str(content)) self.datasets.append(newDataset) # The following line shows how to reply to a message yield self.reply_ok(msg, {'value':self.datasets}) @defer.inlineCallbacks def op_list_all_services(self, content, headers, msg): logging.info('op_list_all_services: '+str(content)) # The following line shows how to reply to a message yield self.reply_ok(msg, {'value':self.services}) @defer.inlineCallbacks def op_register_service(self, content, headers, msg): logging.info('op_register_service: '+ str(content)) newService = ast.literal_eval(str(content)) self.services.append(newService) # The following line shows how to reply to a message yield self.reply_ok(msg, {'value':self.services}) # Spawn of the process using the module name factory = ProtocolFactory(JavaIntegrationService)
BaseServiceClient.__init__(self, proc, **kwargs) @defer.inlineCallbacks def send_data_object(self, obj): yield self._check_init() #print obj msg = obj.encode() logging.info('Sending Encoded resource:' + str(msg)) (content, headers, msg) = yield self.rpc_send('respond', msg, {}) logging.info('Responder replied: ' + str(content)) response = dataobject.DataObject.decode(content['value']) defer.returnValue(response) # Spawn of the process using the module name factory = ProtocolFactory(ResponseService) class TestSendDataObject(IonTestCase): """Testing service classes of resource registry """ @defer.inlineCallbacks def setUp(self): obj = SimpleObject() obj.key = 'seabird' obj.name = 'David' self.obj = obj yield self._start_container() @defer.inlineCallbacks def tearDown(self):
Steps: Send the current commit DAG to the service Receive the Commits which need to be pulled """ (content, headers, msg) = yield self.rpc_send('pull', repository_name) logging.info('Service reply: ' + str(content)) defer.returnValue(str(content)) @defer.inlineCallbacks def clone(self, repository_name): """ @brief Clone a repository to the local object store from the remote datastore @param repository_name - the name (key) of a repository in the remote datastore """ yield self._check_init() logging.info('cloning: ' + repository_name + ', Local Frontend:' + self.frontend) """ Steps: Send the repo name Receive the Commits which need to be pulled """ (content, headers, msg) = yield self.rpc_send('clone', repository_name) logging.info('Service reply: ' + str(content)) defer.returnValue(str(content)) # Spawn of the process using the module name factory = ProtocolFactory(DataStoreService)
""" src_url = content new_url = rewrite_url(src_url) logging.debug('Old url: %s New url: %s' % (src_url, new_url)) # Note that _http_op is inherited fetcher code... return self._http_op('GET', new_url, msg) class RetrieverClient(BaseServiceClient): """ Client interface to the retriever. """ def __init__(self, proc=None, **kwargs): if not 'targetname' in kwargs: kwargs['targetname'] = 'retriever' BaseServiceClient.__init__(self, proc, **kwargs) @defer.inlineCallbacks def get_url(self, url): """ The interface is just a collection of DAP URLs. """ yield self._check_init() (content, headers, msg) = yield self.rpc_send('get_url', url) defer.returnValue(content) factory = ProtocolFactory(RetrieverService)
This is an simple consumer to loggin messages on a queue/topic """ def ondata(self, data, notification, timestamp): if isinstance(data, str): info = 'String Dataset: ' + data elif isinstance(data, DatasetType): info = 'Dap Dataset Name:' + data.name elif isinstance(data, dict): info = 'Dict Dataset Name:' + data.get( 'name', 'No name attribute in data dictionary?') elif data == None: info = 'Received empty Data Message' else: info = 'Unknown dataset type' + str(data) logging.info('LoggingConsumer recieved a data message: \n %s' % info) if notification: logging.info('Data Message Notification:\n %s' % notification) else: logging.debug('Data Message Notification Empty!\n') logging.info('Data Message Timestamp:%s' % timestamp) logging.debug('Data Message Data:' + str(data)) # Something more useful? # Spawn of the process using the module name factory = ProtocolFactory(LoggingConsumer)
""" @Brief Example Registry Service implementation using the base class """ # Declaration of service declare = BaseService.service_declare(name='registry_service', version='0.1.0', dependencies=[]) op_clear_registry = BaseRegistryService.base_clear_registry op_register_resource = BaseRegistryService.base_register_resource op_get_resource = BaseRegistryService.base_get_resource op_get_resource_by_id = BaseRegistryService.base_get_resource_by_id op_set_resource_lcstate = BaseRegistryService.base_set_resource_lcstate op_find_resource = BaseRegistryService.base_find_resource # Spawn of the process using the module name factory = ProtocolFactory(RegistryService) class BaseRegistryClient(BaseServiceClient): """ @brief BaseRegistryClient is the base class used to simplify implementation of Registry Service Clients. The client for a particular registry should inherit from this base class and use the base methods to communicate with the service. The client can do what ever business logic is neccissary before calling the base client methods. """ @defer.inlineCallbacks def base_clear_registry(self,op_name): yield self._check_init()
def op_config(self, content, headers, msg): pass @defer.inlineCallbacks def op_sendstatus(self, content, headers, msg): yield self.reply_ok(msg) class HostStatusClient(BaseServiceClient): """ Class for client to sent log message to service """ def __init__(self, proc=None, **kwargs): if not 'targetname' in kwargs: kwargs['targetname'] = "host_status" BaseServiceClient.__init__(self, proc, **kwargs) @defer.inlineCallbacks def logmsg(self, level, msg, sender, logtime): yield self._check_init() defer.returnValue(0) # Spawn of the process using the module name factory = ProtocolFactory(HostStatusService) """ from ion.services.coi import logger spawn(logger) """
# Declaration of service declare = BaseService.service_declare(name='data_processing', version='0.1.0', dependencies=[]) def op_define_process(self, content, headers, msg): """ Service operation: Create or update a data process. A data process works on data messages and is assumed to have parameterizable input and output """ self.reply_err(msg, "Not yet implemented") def op_schedule_processing(self, content, headers, msg): """ Service operation: Defines processing based on schedule or event trigger, given a data process and required input and output streams. """ self.reply_err(msg, "Not yet implemented") def op_cancel_processing(self, content, headers, msg): """ Service operation: Remove scheduled processing. """ self.reply_err(msg, "Not yet implemented") # Spawn of the process using the module name factory = ProtocolFactory(DataProcessingService)
result = yield self._base_command('get_instrument_state', reqcont) defer.returnValue(result) @defer.inlineCallbacks def execute_command(self, instrumentID, command, arglist): reqcont = {} commandInput = {} commandInput['instrumentID'] = instrumentID commandInput['command'] = command if arglist: argnum = 0 for arg in arglist: commandInput['cmdArg' + str(argnum)] = arg argnum += 1 reqcont['commandInput'] = commandInput result = yield self._base_command('execute_command', reqcont) defer.returnValue(result) @defer.inlineCallbacks def _base_command(self, op, content): (cont, hdrs, msg) = yield self.rpc_send(op, content) if cont.get('status') == 'OK': defer.returnValue(cont) else: defer.returnValue(None) # Spawn of the process using the module name factory = ProtocolFactory(InstrumentManagementService)
""" op_find_registered_agent_definition_from_description = registry.BaseRegistryService.base_find_resource """ Service operation: Find agent definitions which meet a description """ op_find_registered_agent_instance_from_agent = registry.BaseRegistryService.base_find_resource """ Service operation: Find the registered instance that matches the agent instance """ op_find_registered_agent_instance_from_description = registry.BaseRegistryService.base_find_resource """ Service operation: Find all the registered agent instances which match a description """ # Spawn of the process using the module name factory = ProtocolFactory(AgentRegistryService) class AgentRegistryClient(registry.BaseRegistryClient): """ Client class for accessing the agent registry. This is most important for finding and accessing any other agents. This client knows how to find the agent registry - what does that mean, don't all clients have targetname assigned? """ def __init__(self, proc=None, **kwargs): if not 'targetname' in kwargs: kwargs['targetname'] = "agent_registry" BaseServiceClient.__init__(self, proc, **kwargs) def clear_registry(self):
pass def op_define_conv_type(self, content, headers, msg): """Service operation: Define a new conversation type (aka protocol, interaction pattern) """ def op_get_conv_type(self, content, headers, msg): """Service operation: Returns the description of the conversation type including the specification """ def op_define_conversation(self, content, headers, msg): """Service operation: Create a new conversation (instance) definition """ def op_bind_conversation(self, content, headers, msg): """Service operation: Add oneself to the conversation role binding """ def op_log_message(self, content, headers, msg): """Service operation: Log an occurred message with the repository """ logmsg = content['msg'] logserv.info("-----------------------------------------------\n" + str(logmsg)) # Spawn of the process using the module name factory = ProtocolFactory(ConversationRepositoryService)