def _start_output_stream_listener(self, data_product_stream_ids, message_count_per_stream=10): cc = self.container assertions = self.assertTrue ### ### Make a subscriber in the test to listen for transformed data ### salinity_subscription_id = self.pubsubclient.create_subscription( query=StreamQuery(data_product_stream_ids), exchange_name='workflow_test', name="test workflow transformations", ) pid = cc.spawn_process(name='dummy_process_for_test', module='pyon.ion.process', cls='SimpleProcess', config={}) dummy_process = cc.proc_manager.procs[pid] subscriber_registrar = StreamSubscriberRegistrar(process=dummy_process, node=cc.node) result = gevent.event.AsyncResult() results = [] def message_received(message, headers): # Heads log.warn(' data received!') results.append(message) if len(results) >= len( data_product_stream_ids ) * message_count_per_stream: #Only wait for so many messages - per stream result.set(True) subscriber = subscriber_registrar.create_subscriber( exchange_name='workflow_test', callback=message_received) subscriber.start() # after the queue has been created it is safe to activate the subscription self.pubsubclient.activate_subscription( subscription_id=salinity_subscription_id) # Assert that we have received data assertions(result.get(timeout=30)) self.pubsubclient.deactivate_subscription( subscription_id=salinity_subscription_id) subscriber.stop() return results
def setUp(self): self._start_container() self.container.start_rel_from_url('res/deploy/r2dm.yml') self.pubsub_cli = PubsubManagementServiceClient(node=self.container.node) self.ctd_stream1_id = self.pubsub_cli.create_stream(name="SampleStream1", description="Sample Stream 1 Description") self.ctd_stream2_id = self.pubsub_cli.create_stream(name="SampleStream2", description="Sample Stream 2 Description") # Make a subscription to two input streams exchange_name = "a_queue" query = StreamQuery([self.ctd_stream1_id, self.ctd_stream2_id]) self.ctd_subscription_id = self.pubsub_cli.create_subscription(query, exchange_name, "SampleSubscription", "Sample Subscription Description") # Make a subscription to all streams on an exchange point exchange_name = "another_queue" query = ExchangeQuery() self.exchange_subscription_id = self.pubsub_cli.create_subscription(query, exchange_name, "SampleExchangeSubscription", "Sample Exchange Subscription Description") pid = self.container.spawn_process(name='dummy_process_for_test', module='pyon.ion.process', cls='SimpleProcess', config={}) dummy_process = self.container.proc_manager.procs[pid] # Normally the user does not see or create the publisher, this is part of the containers business. # For the test we need to set it up explicitly publisher_registrar = StreamPublisherRegistrar(process=dummy_process, node=self.container.node) self.ctd_stream1_publisher = publisher_registrar.create_publisher(stream_id=self.ctd_stream1_id) self.ctd_stream2_publisher = publisher_registrar.create_publisher(stream_id=self.ctd_stream2_id) # Cheat and use the cc as the process - I don't think it is used for anything... self.stream_subscriber = StreamSubscriberRegistrar(process=dummy_process, node=self.container.node)
def _start_data_subscribers(self): """ """ # Create a pubsub client to create streams. pubsub_client = PubsubManagementServiceClient(node=self.container.node) # A callback for processing subscribed-to data. def consume_data(message, headers): log.info('Subscriber received data message: %s.', str(message)) self._samples_received.append(message) if self._no_samples and self._no_samples == len(self._samples_received): self._async_data_result.set() # Create a stream subscriber registrar to create subscribers. subscriber_registrar = StreamSubscriberRegistrar(process=self.container, node=self.container.node) # Create streams and subscriptions for each stream named in driver. self._stream_config = {} self._data_subscribers = [] for (stream_name, val) in PACKET_CONFIG.iteritems(): stream_def = ctd_stream_definition(stream_id=None) stream_def_id = pubsub_client.create_stream_definition( container=stream_def) stream_id = pubsub_client.create_stream( name=stream_name, stream_definition_id=stream_def_id, original=True, encoding='ION R2') self._stream_config[stream_name] = stream_id # Create subscriptions for each stream. exchange_name = '%s_queue' % stream_name sub = subscriber_registrar.create_subscriber(exchange_name=exchange_name, callback=consume_data) self._listen(sub) self._data_subscribers.append(sub) query = StreamQuery(stream_ids=[stream_id]) sub_id = pubsub_client.create_subscription(\ query=query, exchange_name=exchange_name) pubsub_client.activate_subscription(sub_id)
def on_start(self): pubsub_cli = PubsubManagementServiceProcessClient( process=self, node=self.container.node) # Get the stream(s) stream_id = self.CFG.get_safe('process.stream_id', '') query = StreamQuery(stream_ids=[ stream_id, ]) exchange_name = 'dispatcher_%s' % self.id subscription_id = pubsub_cli.create_subscription( query=query, exchange_name=exchange_name, name="SampleSubscription", description="Sample Subscription Description") stream_subscriber = StreamSubscriberRegistrar(process=self, node=self.container.node) def message_received(granule, h): rdt = RecordDictionaryTool.load_from_granule(granule) log.warn( 'Logging Record Dictionary received in logger subscription \n%s', rdt.pretty_print()) subscriber = stream_subscriber.create_subscriber( exchange_name=exchange_name, callback=message_received) subscriber.start() pubsub_cli.activate_subscription(subscription_id)
def instrument_test_driver(container): org_client = OrgManagementServiceClient(node=container.node) id_client = IdentityManagementServiceClient(node=container.node) system_actor = id_client.find_actor_identity_by_name(name=CFG.system.system_actor) log.info('system actor:' + system_actor._id) sa_header_roles = get_role_message_headers(org_client.find_all_roles_by_user(system_actor._id)) # Names of agent data streams to be configured. parsed_stream_name = 'ctd_parsed' raw_stream_name = 'ctd_raw' # Driver configuration. #Simulator driver_config = { 'svr_addr': 'localhost', 'cmd_port': 5556, 'evt_port': 5557, 'dvr_mod': 'ion.agents.instrument.drivers.sbe37.sbe37_driver', 'dvr_cls': 'SBE37Driver', 'comms_config': { SBE37Channel.CTD: { 'method':'ethernet', 'device_addr': CFG.device.sbe37.host, 'device_port': CFG.device.sbe37.port, 'server_addr': 'localhost', 'server_port': 8888 } } } #Hardware _container_client = ContainerAgentClient(node=container.node, name=container.name) # Create a pubsub client to create streams. _pubsub_client = PubsubManagementServiceClient(node=container.node) # A callback for processing subscribed-to data. def consume(message, headers): log.info('Subscriber received message: %s', str(message)) # Create a stream subscriber registrar to create subscribers. subscriber_registrar = StreamSubscriberRegistrar(process=container, node=container.node) subs = [] # Create streams for each stream named in driver. stream_config = {} for (stream_name, val) in PACKET_CONFIG.iteritems(): stream_def = ctd_stream_definition(stream_id=None) stream_def_id = _pubsub_client.create_stream_definition( container=stream_def) stream_id = _pubsub_client.create_stream( name=stream_name, stream_definition_id=stream_def_id, original=True, encoding='ION R2', headers={'ion-actor-id': system_actor._id, 'ion-actor-roles': sa_header_roles }) stream_config[stream_name] = stream_id # Create subscriptions for each stream. exchange_name = '%s_queue' % stream_name sub = subscriber_registrar.create_subscriber(exchange_name=exchange_name, callback=consume) sub.start() query = StreamQuery(stream_ids=[stream_id]) sub_id = _pubsub_client.create_subscription(\ query=query, exchange_name=exchange_name ) _pubsub_client.activate_subscription(sub_id) subs.append(sub) # Create agent config. agent_resource_id = '123xyz' agent_config = { 'driver_config' : driver_config, 'stream_config' : stream_config, 'agent' : {'resource_id': agent_resource_id} } # Launch an instrument agent process. _ia_name = 'agent007' _ia_mod = 'ion.agents.instrument.instrument_agent' _ia_class = 'InstrumentAgent' _ia_pid = _container_client.spawn_process(name=_ia_name, module=_ia_mod, cls=_ia_class, config=agent_config) log.info('got pid=%s for resource_id=%s' % (str(_ia_pid), str(agent_resource_id)))
def setUp(self): """ Setup the test environment to exersice use of instrumet agent, including: * define driver_config parameters. * create container with required services and container client. * create publication stream ids for each driver data stream. * create stream_config parameters. * create and activate subscriptions for agent data streams. * spawn instrument agent process and create agent client. * add cleanup functions to cause subscribers to get stopped. """ # params = { ('CTD', 'TA2'): -1.9434316e-05, # ('CTD', 'PTCA1'): 1.3206866, # ('CTD', 'TCALDATE'): [8, 11, 2006] } # for tup in params: # print tup self.addCleanup(self.customCleanUp) # Names of agent data streams to be configured. parsed_stream_name = 'ctd_parsed' raw_stream_name = 'ctd_raw' # Driver configuration. #Simulator self.driver_config = { 'svr_addr': 'localhost', 'cmd_port': 5556, 'evt_port': 5557, 'dvr_mod': 'ion.agents.instrument.drivers.sbe37.sbe37_driver', 'dvr_cls': 'SBE37Driver', 'comms_config': { SBE37Channel.CTD: { 'method': 'ethernet', 'device_addr': CFG.device.sbe37.host, 'device_port': CFG.device.sbe37.port, 'server_addr': 'localhost', 'server_port': 8888 } } } #Hardware ''' self.driver_config = { 'svr_addr': 'localhost', 'cmd_port': 5556, 'evt_port': 5557, 'dvr_mod': 'ion.agents.instrument.drivers.sbe37.sbe37_driver', 'dvr_cls': 'SBE37Driver', 'comms_config': { SBE37Channel.CTD: { 'method':'ethernet', 'device_addr': '137.110.112.119', 'device_port': 4001, 'server_addr': 'localhost', 'server_port': 8888 } } } ''' # Start container. self._start_container() # Establish endpoint with container (used in tests below) self._container_client = ContainerAgentClient(node=self.container.node, name=self.container.name) # Bring up services in a deploy file (no need to message) self.container.start_rel_from_url('res/deploy/r2dm.yml') # Create a pubsub client to create streams. self._pubsub_client = PubsubManagementServiceClient( node=self.container.node) # A callback for processing subscribed-to data. def consume(message, headers): log.info('Subscriber received message: %s', str(message)) # Create a stream subscriber registrar to create subscribers. subscriber_registrar = StreamSubscriberRegistrar( process=self.container, node=self.container.node) self.subs = [] # Create streams for each stream named in driver. self.stream_config = {} for (stream_name, val) in PACKET_CONFIG.iteritems(): stream_def = ctd_stream_definition(stream_id=None) stream_def_id = self._pubsub_client.create_stream_definition( container=stream_def) stream_id = self._pubsub_client.create_stream( name=stream_name, stream_definition_id=stream_def_id, original=True, encoding='ION R2') self.stream_config[stream_name] = stream_id # Create subscriptions for each stream. exchange_name = '%s_queue' % stream_name sub = subscriber_registrar.create_subscriber( exchange_name=exchange_name, callback=consume) sub.start() query = StreamQuery(stream_ids=[stream_id]) sub_id = self._pubsub_client.create_subscription(\ query=query, exchange_name=exchange_name) self._pubsub_client.activate_subscription(sub_id) self.subs.append(sub) # Add cleanup function to stop subscribers. def stop_subscriber(sub_list): for sub in sub_list: sub.stop() self.addCleanup(stop_subscriber, self.subs) # Create agent config. self.agent_resource_id = '123xyz' self.agent_config = { 'driver_config': self.driver_config, 'stream_config': self.stream_config, 'agent': { 'resource_id': self.agent_resource_id } } # Launch an instrument agent process. self._ia_name = 'agent007' self._ia_mod = 'ion.agents.instrument.instrument_agent' self._ia_class = 'InstrumentAgent' self._ia_pid = self._container_client.spawn_process( name=self._ia_name, module=self._ia_mod, cls=self._ia_class, config=self.agent_config) log.info('got pid=%s', str(self._ia_pid)) self._ia_client = None # Start a resource agent client to talk with the instrument agent. self._ia_client = ResourceAgentClient(self.agent_resource_id, process=FakeProcess()) log.info('got ia client %s', str(self._ia_client))
def on_start(self): rr_cli = ResourceRegistryServiceProcessClient(process=self, node=self.container.node) pubsub_cli = PubsubManagementServiceProcessClient( process=self, node=self.container.node) # Get the stream(s) data_product_id = self.CFG.get_safe('dispatcher.data_product_id', '') stream_ids, _ = rr_cli.find_objects(subject=data_product_id, predicate=PRED.hasStream, id_only=True) log.info('Got Stream Ids: "%s"', stream_ids) assert stream_ids, 'No streams found for this data product!' query = StreamQuery(stream_ids=stream_ids) exchange_name = 'dispatcher_%s' % str(os.getpid()) subscription_id = pubsub_cli.create_subscription( query=query, exchange_name=exchange_name, name="SampleSubscription", description="Sample Subscription Description") stream_subscriber = StreamSubscriberRegistrar(process=self, node=self.container.node) stream_defs = {} def message_received(granule, h): stream_id = granule.stream_resource_id data_stream_id = granule.data_stream_id data_stream = granule.identifiables[data_stream_id] tstamp = get_datetime(data_stream.timestamp.value) records = granule.identifiables['record_count'].value log.info( 'Received a message from stream %s with time stamp %s and %d records' % (stream_id, tstamp, records)) if stream_id not in stream_defs: stream_defs[stream_id] = pubsub_cli.find_stream_definition( stream_id, id_only=False).container stream_def = stream_defs.get(stream_id) sp = PointSupplementStreamParser(stream_definition=stream_def, stream_granule=granule) last_data = {} for field in sp.list_field_names(): last_data[field] = sp.get_values(field)[-1] log.info('Last values in the message: %s' % str(last_data)) subscriber = stream_subscriber.create_subscriber( exchange_name=exchange_name, callback=message_received) subscriber.start() pubsub_cli.activate_subscription(subscription_id)
def on_start(self): # The data dictionary object holds a copy of all the viz products created by the service. The viz # products are indexed by the viz_product_type and data_product_id (which could be google_datatables or # mpl_graphs self.viz_data_dictionary = {} self.viz_data_dictionary['google_dt'] = {} self.viz_data_dictionary['google_realtime_dt'] = {} self.viz_data_dictionary['matplotlib_graphs'] = {} # Kind of redundant but we will maintain a separate list of data product_ids registered with the viz_service self.data_products = [] # Create clients to interface with PubSub, Transform Management Service and Resource Registry self.pubsub_cli = self.clients.pubsub_management self.tms_cli = self.clients.transform_management self.rr_cli = self.clients.resource_registry self.dr_cli = self.clients.data_retriever self.dsm_cli = self.clients.dataset_management """ # Create process definitions which will used to spawn off the transform processes self.matplotlib_proc_def = IonObject(RT.ProcessDefinition, name='viz_transform_process'+'.'+self.random_id_generator()) self.matplotlib_proc_def.executable = { 'module': 'ion.services.ans.visualization_service', 'class':'VizTransformProcForMatplotlibGraphs' } self.matplotlib_proc_def_id, _ = self.rr_cli.create(self.matplotlib_proc_def) self.google_dt_proc_def = IonObject(RT.ProcessDefinition, name='viz_transform_process'+'.'+self.random_id_generator()) self.google_dt_proc_def.executable = { 'module': 'ion.services.ans.visualization_service', 'class':'VizTransformProcForGoogleDT' } self.google_dt_proc_def_id, _ = self.rr_cli.create(self.google_dt_proc_def) """ # Query resource registry to get process definitions and streams ids made by the bootstrap proc_def_ids, _ = self.rr_cli.find_resources( restype=RT.ProcessDefinition, lcstate=None, name="viz_matplotlib_transform_process", id_only=True) self.matplotlib_proc_def_id = proc_def_ids[0] proc_def_ids, _ = self.rr_cli.find_resources( restype=RT.ProcessDefinition, lcstate=None, name="viz_google_dt_transform_process", id_only=True) self.google_dt_proc_def_id = proc_def_ids[0] # Create a stream that all the transform processes will use to submit data back to the viz service self.viz_service_submit_stream_id = self.pubsub_cli.create_stream( name="visualization_service_submit_stream." + self.random_id_generator()) # subscribe to this stream since all the results from transforms will be submitted here query = StreamQuery(stream_ids=[ self.viz_service_submit_stream_id, ]) self.viz_service_submit_stream_sub_id = self.pubsub_cli.create_subscription( query=query, exchange_name="visualization_service_submit_queue") submit_stream_subscriber_registrar = StreamSubscriberRegistrar( process=self.container, node=self.container.node) submit_stream_subscriber = submit_stream_subscriber_registrar.create_subscriber( exchange_name='visualization_service_submit_queue', callback=self.process_submission) submit_stream_subscriber.start() self.pubsub_cli.activate_subscription( self.viz_service_submit_stream_sub_id) # Discover the existing data_product_ids active in the system sys_prod_ids, _ = self.rr_cli.find_resources(RT.DataProduct, None, None, True) # Register all the streams in the system, which will in turn start transform processes for dp_id in sys_prod_ids: self.register_new_data_product(dp_id) # listen for events when new data_products show up self.event_subscriber = EventSubscriber( event_type="ResourceModifiedEvent", origin_type="DataProduct", sub_type="UPDATE", callback=self.receive_new_dataproduct_event) self.event_subscriber.activate() return
def test_dm_integration(self): ''' test_salinity_transform Test full DM Services Integration ''' cc = self.container assertions = self.assertTrue #----------------------------- # Copy below here to run as a script (don't forget the imports of course!) #----------------------------- # Create some service clients... pubsub_management_service = PubsubManagementServiceClient(node=cc.node) ingestion_management_service = IngestionManagementServiceClient( node=cc.node) dataset_management_service = DatasetManagementServiceClient( node=cc.node) data_retriever_service = DataRetrieverServiceClient(node=cc.node) transform_management_service = TransformManagementServiceClient( node=cc.node) process_dispatcher = ProcessDispatcherServiceClient(node=cc.node) # declare some handy variables datastore_name = 'test_dm_integration' ### ### In the beginning there were two stream definitions... ### # create a stream definition for the data from the ctd simulator ctd_stream_def = SBE37_CDM_stream_definition() ctd_stream_def_id = pubsub_management_service.create_stream_definition( container=ctd_stream_def, name='Simulated CTD data') # create a stream definition for the data from the salinity Transform sal_stream_def_id = pubsub_management_service.create_stream_definition( container=SalinityTransform.outgoing_stream_def, name='Scalar Salinity data stream') ### ### And two process definitions... ### # one for the ctd simulator... producer_definition = ProcessDefinition() producer_definition.executable = { 'module': 'ion.processes.data.ctd_stream_publisher', 'class': 'SimpleCtdPublisher' } ctd_sim_procdef_id = process_dispatcher.create_process_definition( process_definition=producer_definition) # one for the salinity transform producer_definition = ProcessDefinition() producer_definition.executable = { 'module': 'ion.processes.data.transforms.ctd.ctd_L2_salinity', 'class': 'SalinityTransform' } salinity_transform_procdef_id = process_dispatcher.create_process_definition( process_definition=producer_definition) #--------------------------- # Set up ingestion - this is an operator concern - not done by SA in a deployed system #--------------------------- # Configure ingestion using eight workers, ingesting to test_dm_integration datastore with the SCIDATA profile log.debug('Calling create_ingestion_configuration') ingestion_configuration_id = ingestion_management_service.create_ingestion_configuration( exchange_point_id='science_data', couch_storage=CouchStorage(datastore_name=datastore_name, datastore_profile='SCIDATA'), number_of_workers=1) # ingestion_management_service.activate_ingestion_configuration( ingestion_configuration_id=ingestion_configuration_id) #--------------------------- # Set up the producer (CTD Simulator) #--------------------------- # Create the stream ctd_stream_id = pubsub_management_service.create_stream( stream_definition_id=ctd_stream_def_id) # Set up the datasets ctd_dataset_id = dataset_management_service.create_dataset( stream_id=ctd_stream_id, datastore_name=datastore_name, view_name='datasets/stream_join_granule') # Configure ingestion of this dataset ctd_dataset_config_id = ingestion_management_service.create_dataset_configuration( dataset_id=ctd_dataset_id, archive_data=True, archive_metadata=True, ingestion_configuration_id= ingestion_configuration_id, # you need to know the ingestion configuration id! ) # Hold onto ctd_dataset_config_id if you want to stop/start ingestion of that dataset by the ingestion service #--------------------------- # Set up the salinity transform #--------------------------- # Create the stream sal_stream_id = pubsub_management_service.create_stream( stream_definition_id=sal_stream_def_id) # Set up the datasets sal_dataset_id = dataset_management_service.create_dataset( stream_id=sal_stream_id, datastore_name=datastore_name, view_name='datasets/stream_join_granule') # Configure ingestion of the salinity as a dataset sal_dataset_config_id = ingestion_management_service.create_dataset_configuration( dataset_id=sal_dataset_id, archive_data=True, archive_metadata=True, ingestion_configuration_id= ingestion_configuration_id, # you need to know the ingestion configuration id! ) # Hold onto sal_dataset_config_id if you want to stop/start ingestion of that dataset by the ingestion service # Create a subscription as input to the transform sal_transform_input_subscription_id = pubsub_management_service.create_subscription( query=StreamQuery(stream_ids=[ ctd_stream_id, ]), exchange_name='salinity_transform_input' ) # how do we make these names??? i.e. Should they be anonymous? # create the salinity transform sal_transform_id = transform_management_service.create_transform( name='example salinity transform', in_subscription_id=sal_transform_input_subscription_id, out_streams={ 'output': sal_stream_id, }, process_definition_id=salinity_transform_procdef_id, # no configuration needed at this time... ) # start the transform - for a test case it makes sense to do it before starting the producer but it is not required transform_management_service.activate_transform( transform_id=sal_transform_id) # Start the ctd simulator to produce some data configuration = { 'process': { 'stream_id': ctd_stream_id, } } ctd_sim_pid = process_dispatcher.schedule_process( process_definition_id=ctd_sim_procdef_id, configuration=configuration) ### ### Make a subscriber in the test to listen for salinity data ### salinity_subscription_id = pubsub_management_service.create_subscription( query=StreamQuery([ sal_stream_id, ]), exchange_name='salinity_test', name="test salinity subscription", ) pid = cc.spawn_process(name='dummy_process_for_test', module='pyon.ion.process', cls='SimpleProcess', config={}) dummy_process = cc.proc_manager.procs[pid] subscriber_registrar = StreamSubscriberRegistrar(process=dummy_process, node=cc.node) result = gevent.event.AsyncResult() results = [] def message_received(message, headers): # Heads log.warn('Salinity data received!') results.append(message) if len(results) > 3: result.set(True) subscriber = subscriber_registrar.create_subscriber( exchange_name='salinity_test', callback=message_received) subscriber.start() # after the queue has been created it is safe to activate the subscription pubsub_management_service.activate_subscription( subscription_id=salinity_subscription_id) # Assert that we have received data assertions(result.get(timeout=10)) # stop the flow parse the messages... process_dispatcher.cancel_process( ctd_sim_pid ) # kill the ctd simulator process - that is enough data for message in results: psd = PointSupplementStreamParser( stream_definition=SalinityTransform.outgoing_stream_def, stream_granule=message) # Test the handy info method for the names of fields in the stream def assertions('salinity' in psd.list_field_names()) # you have to know the name of the coverage in stream def salinity = psd.get_values('salinity') import numpy assertions(isinstance(salinity, numpy.ndarray)) assertions(numpy.nanmin(salinity) > 0.0) # salinity should always be greater than 0
def test_raw_stream_integration(self): cc = self.container assertions = self.assertTrue #----------------------------- # Copy below here to run as a script (don't forget the imports of course!) #----------------------------- # Create some service clients... pubsub_management_service = PubsubManagementServiceClient(node=cc.node) ingestion_management_service = IngestionManagementServiceClient( node=cc.node) dataset_management_service = DatasetManagementServiceClient( node=cc.node) process_dispatcher = ProcessDispatcherServiceClient(node=cc.node) # declare some handy variables datastore_name = 'test_dm_integration' ### ### In the beginning there was one stream definitions... ### # create a stream definition for the data from the ctd simulator raw_ctd_stream_def = SBE37_RAW_stream_definition() raw_ctd_stream_def_id = pubsub_management_service.create_stream_definition( container=raw_ctd_stream_def, name='Simulated RAW CTD data') ### ### And two process definitions... ### # one for the ctd simulator... producer_definition = ProcessDefinition() producer_definition.executable = { 'module': 'ion.processes.data.raw_stream_publisher', 'class': 'RawStreamPublisher' } raw_ctd_sim_procdef_id = process_dispatcher.create_process_definition( process_definition=producer_definition) #--------------------------- # Set up ingestion - this is an operator concern - not done by SA in a deployed system #--------------------------- # Configure ingestion using eight workers, ingesting to test_dm_integration datastore with the SCIDATA profile log.debug('Calling create_ingestion_configuration') ingestion_configuration_id = ingestion_management_service.create_ingestion_configuration( exchange_point_id='science_data', couch_storage=CouchStorage(datastore_name=datastore_name, datastore_profile='SCIDATA'), number_of_workers=1) # ingestion_management_service.activate_ingestion_configuration( ingestion_configuration_id=ingestion_configuration_id) #--------------------------- # Set up the producer (CTD Simulator) #--------------------------- # Create the stream raw_ctd_stream_id = pubsub_management_service.create_stream( stream_definition_id=raw_ctd_stream_def_id) # Set up the datasets raw_ctd_dataset_id = dataset_management_service.create_dataset( stream_id=raw_ctd_stream_id, datastore_name=datastore_name, view_name='datasets/stream_join_granule') # Configure ingestion of this dataset raw_ctd_dataset_config_id = ingestion_management_service.create_dataset_configuration( dataset_id=raw_ctd_dataset_id, archive_data=True, archive_metadata=True, ingestion_configuration_id= ingestion_configuration_id, # you need to know the ingestion configuration id! ) # Hold onto ctd_dataset_config_id if you want to stop/start ingestion of that dataset by the ingestion service # Start the ctd simulator to produce some data configuration = { 'process': { 'stream_id': raw_ctd_stream_id, } } raw_sim_pid = process_dispatcher.schedule_process( process_definition_id=raw_ctd_sim_procdef_id, configuration=configuration) ### ### Make a subscriber in the test to listen for salinity data ### raw_subscription_id = pubsub_management_service.create_subscription( query=StreamQuery([ raw_ctd_stream_id, ]), exchange_name='raw_test', name="test raw subscription", ) # this is okay - even in cei mode! pid = cc.spawn_process(name='dummy_process_for_test', module='pyon.ion.process', cls='SimpleProcess', config={}) dummy_process = cc.proc_manager.procs[pid] subscriber_registrar = StreamSubscriberRegistrar(process=dummy_process, node=cc.node) result = gevent.event.AsyncResult() results = [] def message_received(message, headers): # Heads log.warn('Raw data received!') results.append(message) if len(results) > 3: result.set(True) subscriber = subscriber_registrar.create_subscriber( exchange_name='raw_test', callback=message_received) subscriber.start() # after the queue has been created it is safe to activate the subscription pubsub_management_service.activate_subscription( subscription_id=raw_subscription_id) # Assert that we have received data assertions(result.get(timeout=10)) # stop the flow parse the messages... process_dispatcher.cancel_process( raw_sim_pid ) # kill the ctd simulator process - that is enough data gevent.sleep(1) for message in results: sha1 = message.identifiables['stream_encoding'].sha1 data = message.identifiables['data_stream'].values filename = FileSystem.get_hierarchical_url(FS.CACHE, sha1, ".raw") with open(filename, 'r') as f: assertions(data == f.read())
def test_integrated_transform(self): ''' This example script runs a chained three way transform: B A < C Where A is the even_odd transform (generates a stream of even and odd numbers from input) and B and C are the basic transforms that receive even and odd input ''' cc = self.container assertions = self.assertTrue pubsub_cli = PubsubManagementServiceClient(node=cc.node) rr_cli = ResourceRegistryServiceClient(node=cc.node) tms_cli = TransformManagementServiceClient(node=cc.node) #------------------------------- # Process Definition #------------------------------- # Create the process definition for the basic transform process_definition = IonObject(RT.ProcessDefinition, name='basic_transform_definition') process_definition.executable = { 'module': 'ion.processes.data.transforms.transform_example', 'class':'TransformExample' } basic_transform_definition_id, _ = rr_cli.create(process_definition) # Create The process definition for the TransformEvenOdd process_definition = IonObject(RT.ProcessDefinition, name='evenodd_transform_definition') process_definition.executable = { 'module': 'ion.processes.data.transforms.transform_example', 'class':'TransformEvenOdd' } evenodd_transform_definition_id, _ = rr_cli.create(process_definition) #------------------------------- # Streams #------------------------------- streams = [pubsub_cli.create_stream() for i in xrange(5)] #------------------------------- # Subscriptions #------------------------------- query = StreamQuery(stream_ids=[streams[0]]) input_subscription_id = pubsub_cli.create_subscription(query=query, exchange_name='input_queue') query = StreamQuery(stream_ids = [streams[1]]) # even output even_subscription_id = pubsub_cli.create_subscription(query=query, exchange_name='even_queue') query = StreamQuery(stream_ids = [streams[2]]) # odd output odd_subscription_id = pubsub_cli.create_subscription(query=query, exchange_name='odd_queue') #------------------------------- # Launch the EvenOdd Transform #------------------------------- evenodd_id = tms_cli.create_transform(name='even_odd', in_subscription_id=input_subscription_id, out_streams={'even':streams[1], 'odd':streams[2]}, process_definition_id=evenodd_transform_definition_id, configuration={}) tms_cli.activate_transform(evenodd_id) #------------------------------- # Launch the Even Processing Transform #------------------------------- even_transform_id = tms_cli.create_transform(name='even_transform', in_subscription_id = even_subscription_id, out_streams={'even_plus1':streams[3]}, process_definition_id=basic_transform_definition_id, configuration={}) tms_cli.activate_transform(even_transform_id) #------------------------------- # Launch the Odd Processing Transform #------------------------------- odd_transform_id = tms_cli.create_transform(name='odd_transform', in_subscription_id = odd_subscription_id, out_streams={'odd_plus1':streams[4]}, process_definition_id=basic_transform_definition_id, configuration={}) tms_cli.activate_transform(odd_transform_id) #------------------------------- # Set up final subscribers #------------------------------- evenplus1_subscription_id = pubsub_cli.create_subscription( query=StreamQuery([streams[3]]), exchange_name='evenplus1_queue', name='EvenPlus1Subscription', description='EvenPlus1 SubscriptionDescription' ) oddplus1_subscription_id = pubsub_cli.create_subscription( query=StreamQuery([streams[4]]), exchange_name='oddplus1_queue', name='OddPlus1Subscription', description='OddPlus1 SubscriptionDescription' ) total_msg_count = 2 msgs = gevent.queue.Queue() def even1_message_received(message, headers): input = int(message.get('num')) assertions( (input % 2) ) # Assert it is odd (transform adds 1) msgs.put(True) def odd1_message_received(message, headers): input = int(message.get('num')) assertions(not (input % 2)) # Assert it is even msgs.put(True) subscriber_registrar = StreamSubscriberRegistrar(process=cc, node=cc.node) even_subscriber = subscriber_registrar.create_subscriber(exchange_name='evenplus1_queue', callback=even1_message_received) odd_subscriber = subscriber_registrar.create_subscriber(exchange_name='oddplus1_queue', callback=odd1_message_received) # Start subscribers even_subscriber.start() odd_subscriber.start() # Activate subscriptions pubsub_cli.activate_subscription(evenplus1_subscription_id) pubsub_cli.activate_subscription(oddplus1_subscription_id) #------------------------------- # Set up fake stream producer #------------------------------- pid = cc.spawn_process(name='dummy_process_for_test', module='pyon.ion.process', cls='SimpleProcess', config={}) dummy_process = cc.proc_manager.procs[pid] # Normally the user does not see or create the publisher, this is part of the containers business. # For the test we need to set it up explicitly publisher_registrar = StreamPublisherRegistrar(process=dummy_process, node=cc.node) stream_publisher = publisher_registrar.create_publisher(stream_id=streams[0]) #------------------------------- # Start test #------------------------------- # Publish a stream for i in xrange(total_msg_count): stream_publisher.publish({'num':str(i)}) time.sleep(0.5) for i in xrange(total_msg_count * 2): try: msgs.get() except Empty: assertions(False, "Failed to process all messages correctly.")
def test_activateInstrumentStream(self): # Create InstrumentModel instModel_obj = IonObject(RT.InstrumentModel, name='SBE37IMModel', description="SBE37IMModel", model_label="SBE37IMModel") try: instModel_id = self.imsclient.create_instrument_model( instModel_obj) except BadRequest as ex: self.fail("failed to create new InstrumentModel: %s" % ex) print 'new InstrumentModel id = ', instModel_id # Create InstrumentAgent instAgent_obj = IonObject( RT.InstrumentAgent, name='agent007', description="SBE37IMAgent", driver_module="ion.agents.instrument.instrument_agent", driver_class="InstrumentAgent") try: instAgent_id = self.imsclient.create_instrument_agent( instAgent_obj) except BadRequest as ex: self.fail("failed to create new InstrumentAgent: %s" % ex) print 'new InstrumentAgent id = ', instAgent_id self.imsclient.assign_instrument_model_to_instrument_agent( instModel_id, instAgent_id) # Create InstrumentDevice log.debug( 'test_activateInstrumentStream: Create instrument resource to represent the SBE37 (SA Req: L4-CI-SA-RQ-241) ' ) instDevice_obj = IonObject(RT.InstrumentDevice, name='SBE37IMDevice', description="SBE37IMDevice", serial_number="12345") try: instDevice_id = self.imsclient.create_instrument_device( instrument_device=instDevice_obj) self.imsclient.assign_instrument_model_to_instrument_device( instModel_id, instDevice_id) except BadRequest as ex: self.fail("failed to create new InstrumentDevice: %s" % ex) log.debug( "test_activateInstrumentStream: new InstrumentDevice id = %s (SA Req: L4-CI-SA-RQ-241) ", instDevice_id) driver_config = { 'dvr_mod': 'ion.agents.instrument.drivers.sbe37.sbe37_driver', 'dvr_cls': 'SBE37Driver', 'workdir': '/tmp/', } instAgentInstance_obj = IonObject( RT.InstrumentAgentInstance, name='SBE37IMAgentInstance', description="SBE37IMAgentInstance", driver_config=driver_config, comms_device_address='sbe37-simulator.oceanobservatories.org', comms_device_port=4001, port_agent_work_dir='/tmp/', port_agent_delimeter=['<<', '>>']) instAgentInstance_id = self.imsclient.create_instrument_agent_instance( instAgentInstance_obj, instAgent_id, instDevice_id) # create a stream definition for the data from the ctd simulator ctd_stream_def = SBE37_CDM_stream_definition() ctd_stream_def_id = self.pubsubcli.create_stream_definition( container=ctd_stream_def) log.debug( 'test_activateInstrumentStream new Stream Definition id = %s', instDevice_id) log.debug( 'test_activateInstrumentStream Creating new CDM data product with a stream definition' ) dp_obj = IonObject(RT.DataProduct, name='the parsed data', description='ctd stream test') try: data_product_id1 = self.dpclient.create_data_product( dp_obj, ctd_stream_def_id) except BadRequest as ex: self.fail("failed to create new data product: %s" % ex) log.debug('test_activateInstrumentStream new dp_id = %s', str(data_product_id1)) self.damsclient.assign_data_product(input_resource_id=instDevice_id, data_product_id=data_product_id1) self.dpclient.activate_data_product_persistence( data_product_id=data_product_id1, persist_data=True, persist_metadata=True) # Retrieve the id of the OUTPUT stream from the out Data Product stream_ids, _ = self.rrclient.find_objects(data_product_id1, PRED.hasStream, None, True) log.debug('test_activateInstrumentStream Data product streams1 = %s', str(stream_ids)) simdata_subscription_id = self.pubsubcli.create_subscription( query=StreamQuery([stream_ids[0]]), exchange_name='Sim_data_queue', name='SimDataSubscription', description='SimData SubscriptionDescription') def simdata_message_received(message, headers): input = str(message) log.debug("test_activateInstrumentStream: granule received: %s", input) subscriber_registrar = StreamSubscriberRegistrar( process=self.container, node=self.container.node) simdata_subscriber = subscriber_registrar.create_subscriber( exchange_name='Sim_data_queue', callback=simdata_message_received) # Start subscribers simdata_subscriber.start() # Activate subscriptions self.pubsubcli.activate_subscription(simdata_subscription_id) log.debug( 'test_activateInstrumentStream Creating new RAW data product with a stream definition' ) raw_stream_def = SBE37_RAW_stream_definition() raw_stream_def_id = self.pubsubcli.create_stream_definition( container=raw_stream_def) dp_obj = IonObject(RT.DataProduct, name='the raw data', description='raw stream test') try: data_product_id2 = self.dpclient.create_data_product( dp_obj, raw_stream_def_id) except BadRequest as ex: self.fail("failed to create new data product: %s" % ex) log.debug('test_activateInstrumentStream new dp_id = %s', str(data_product_id2)) self.damsclient.assign_data_product(input_resource_id=instDevice_id, data_product_id=data_product_id2) self.dpclient.activate_data_product_persistence( data_product_id=data_product_id2, persist_data=True, persist_metadata=True) # Retrieve the id of the OUTPUT stream from the out Data Product stream_ids, _ = self.rrclient.find_objects(data_product_id2, PRED.hasStream, None, True) log.debug('test_activateInstrumentStream Data product streams2 = %s', str(stream_ids)) self.imsclient.start_instrument_agent_instance( instrument_agent_instance_id=instAgentInstance_id) inst_agent_instance_obj = self.imsclient.read_instrument_agent_instance( instAgentInstance_id) log.debug( 'test_activateInstrumentStream Instrument agent instance obj: = %s', str(inst_agent_instance_obj)) # Start a resource agent client to talk with the instrument agent. #self._ia_client = ResourceAgentClient('123xyz', name=inst_agent_instance_obj.agent_process_id, process=FakeProcess()) self._ia_client = ResourceAgentClient(instDevice_id, process=FakeProcess()) log.debug('test_activateInstrumentStream: got ia client %s', str(self._ia_client)) log.debug("test_activateInstrumentStream: got ia client %s", str(self._ia_client)) cmd = AgentCommand(command='initialize') retval = self._ia_client.execute_agent(cmd) log.debug("test_activateInstrumentStream: initialize %s", str(retval)) time.sleep(2) log.debug( "test_activateInstrumentStream: Sending go_active command (L4-CI-SA-RQ-334)" ) cmd = AgentCommand(command='go_active') reply = self._ia_client.execute_agent(cmd) log.debug( "test_activateInstrumentStream: return value from go_active %s", str(reply)) time.sleep(2) cmd = AgentCommand(command='get_current_state') retval = self._ia_client.execute_agent(cmd) state = retval.result log.debug( "test_activateInstrumentStream: current state after sending go_active command %s (L4-CI-SA-RQ-334)", str(state)) cmd = AgentCommand(command='run') reply = self._ia_client.execute_agent(cmd) log.debug("test_activateInstrument: run %s", str(reply)) time.sleep(2) log.debug("test_activateInstrumentStream: calling go_streaming ") cmd = AgentCommand(command='go_streaming') reply = self._ia_client.execute(cmd) log.debug("test_activateInstrumentStream: return from go_streaming %s", str(reply)) time.sleep(15) log.debug("test_activateInstrumentStream: calling go_observatory") cmd = AgentCommand(command='go_observatory') reply = self._ia_client.execute(cmd) log.debug( "test_activateInstrumentStream: return from go_observatory %s", str(reply)) time.sleep(2) log.debug("test_activateInstrumentStream: calling reset ") cmd = AgentCommand(command='reset') reply = self._ia_client.execute_agent(cmd) log.debug("test_activateInstrumentStream: return from reset %s", str(reply)) time.sleep(2) #------------------------------- # Deactivate InstrumentAgentInstance #------------------------------- self.imsclient.stop_instrument_agent_instance( instrument_agent_instance_id=instAgentInstance_id)