class TestDMEnd2End(IonIntegrationTestCase): def setUp(self): # Love the non pep-8 convention self._start_container() self.container.start_rel_from_url('res/deploy/r2deploy.yml') self.process_dispatcher = ProcessDispatcherServiceClient() self.pubsub_management = PubsubManagementServiceClient() self.resource_registry = ResourceRegistryServiceClient() self.dataset_management = DatasetManagementServiceClient() self.ingestion_management = IngestionManagementServiceClient() self.data_retriever = DataRetrieverServiceClient() self.pids = [] self.event = Event() self.exchange_space_name = 'test_granules' self.exchange_point_name = 'science_data' self.purge_queues() self.queue_buffer = [] def purge_queues(self): xn = self.container.ex_manager.create_xn_queue('science_granule_ingestion') xn.purge() def tearDown(self): self.purge_queues() for pid in self.pids: self.container.proc_manager.terminate_process(pid) IngestionManagementIntTest.clean_subscriptions() for queue in self.queue_buffer: if isinstance(queue, ExchangeNameQueue): queue.delete() elif isinstance(queue, str): xn = self.container.ex_manager.create_xn_queue(queue) xn.delete() def launch_producer(self, stream_id=''): #-------------------------------------------------------------------------------- # Launch the producer #-------------------------------------------------------------------------------- pid = self.container.spawn_process('better_data_producer', 'ion.processes.data.example_data_producer', 'BetterDataProducer', {'process':{'stream_id':stream_id}}) self.pids.append(pid) def get_ingestion_config(self): #-------------------------------------------------------------------------------- # Grab the ingestion configuration from the resource registry #-------------------------------------------------------------------------------- # The ingestion configuration should have been created by the bootstrap service # which is configured through r2deploy.yml ingest_configs, _ = self.resource_registry.find_resources(restype=RT.IngestionConfiguration,id_only=True) return ingest_configs[0] def publish_hifi(self,stream_id,stream_route,offset=0): pub = StandaloneStreamPublisher(stream_id, stream_route) stream_def = self.pubsub_management.read_stream_definition(stream_id=stream_id) stream_def_id = stream_def._id rdt = RecordDictionaryTool(stream_definition_id=stream_def_id) rdt['time'] = np.arange(10) + (offset * 10) rdt['temp'] = np.arange(10) + (offset * 10) pub.publish(rdt.to_granule()) def publish_fake_data(self,stream_id, route): for i in xrange(4): self.publish_hifi(stream_id,route,i) def get_datastore(self, dataset_id): dataset = self.dataset_management.read_dataset(dataset_id) datastore_name = dataset.datastore_name datastore = self.container.datastore_manager.get_datastore(datastore_name, DataStore.DS_PROFILE.SCIDATA) return datastore def validate_granule_subscription(self, msg, route, stream_id): if msg == {}: return rdt = RecordDictionaryTool.load_from_granule(msg) log.info('%s', rdt.pretty_print()) self.assertIsInstance(msg,Granule,'Message is improperly formatted. (%s)' % type(msg)) self.event.set() def make_file_data(self): from interface.objects import File import uuid data = 'hello world\n' rand = str(uuid.uuid4())[:8] meta = File(name='/examples/' + rand + '.txt', group_id='example1') return {'body': data, 'meta':meta} def publish_file(self, stream_id, stream_route): publisher = StandaloneStreamPublisher(stream_id,stream_route) publisher.publish(self.make_file_data()) def wait_until_we_have_enough_granules(self, dataset_id='',granules=4): datastore = self.get_datastore(dataset_id) dataset = self.dataset_management.read_dataset(dataset_id) with gevent.timeout.Timeout(40): success = False while not success: success = len(datastore.query_view(dataset.view_name)) >= granules gevent.sleep(0.1) log.info(datastore.query_view(dataset.view_name)) def wait_until_we_have_enough_files(self): datastore = self.container.datastore_manager.get_datastore('filesystem', DataStore.DS_PROFILE.FILESYSTEM) now = time.time() timeout = now + 10 done = False while not done: if now >= timeout: raise Timeout('Files are not populating in time.') if len(datastore.query_view('catalog/file_by_owner')) >= 1: done = True now = time.time() def create_dataset(self, parameter_dict_id=''): tdom, sdom = time_series_domain() sdom = sdom.dump() tdom = tdom.dump() if not parameter_dict_id: parameter_dict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True) dataset_id = self.dataset_management.create_dataset('test_dataset', parameter_dictionary_id=parameter_dict_id, spatial_domain=sdom, temporal_domain=tdom) return dataset_id @unittest.skip('Doesnt work') @attr('LOCOINT') @unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False), 'Skip test while in CEI LAUNCH mode') def test_replay_pause(self): # Get a precompiled parameter dictionary with basic ctd fields pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict',id_only=True) context_ids = self.dataset_management.read_parameter_contexts(pdict_id, id_only=True) # Add a field that supports binary data input. bin_context = ParameterContext('binary', param_type=ArrayType()) context_ids.append(self.dataset_management.create_parameter_context('binary', bin_context.dump())) # Add another field that supports dictionary elements. rec_context = ParameterContext('records', param_type=RecordType()) context_ids.append(self.dataset_management.create_parameter_context('records', rec_context.dump())) pdict_id = self.dataset_management.create_parameter_dictionary('replay_pdict', parameter_context_ids=context_ids, temporal_context='time') stream_def_id = self.pubsub_management.create_stream_definition('replay_stream', parameter_dictionary_id=pdict_id) replay_stream, replay_route = self.pubsub_management.create_stream('replay', 'xp1', stream_definition_id=stream_def_id) dataset_id = self.create_dataset(pdict_id) scov = DatasetManagementService._get_coverage(dataset_id) bb = CoverageCraft(scov) bb.rdt['time'] = np.arange(100) bb.rdt['temp'] = np.random.random(100) + 30 bb.sync_with_granule() DatasetManagementService._persist_coverage(dataset_id, bb.coverage) # This invalidates it for multi-host configurations # Set up the subscriber to verify the data subscriber = StandaloneStreamSubscriber(self.exchange_space_name, self.validate_granule_subscription) xp = self.container.ex_manager.create_xp('xp1') self.queue_buffer.append(self.exchange_space_name) subscriber.start() subscriber.xn.bind(replay_route.routing_key, xp) # Set up the replay agent and the client wrapper # 1) Define the Replay (dataset and stream to publish on) self.replay_id, process_id = self.data_retriever.define_replay(dataset_id=dataset_id, stream_id=replay_stream) # 2) Make a client to the interact with the process (optionall provide it a process to bind with) replay_client = ReplayClient(process_id) # 3) Start the agent (launch the process) self.data_retriever.start_replay_agent(self.replay_id) # 4) Start replaying... replay_client.start_replay() # Wait till we get some granules self.assertTrue(self.event.wait(5)) # We got granules, pause the replay, clear the queue and allow the process to finish consuming replay_client.pause_replay() gevent.sleep(1) subscriber.xn.purge() self.event.clear() # Make sure there's no remaining messages being consumed self.assertFalse(self.event.wait(1)) # Resume the replay and wait until we start getting granules again replay_client.resume_replay() self.assertTrue(self.event.wait(5)) # Stop the replay, clear the queues replay_client.stop_replay() gevent.sleep(1) subscriber.xn.purge() self.event.clear() # Make sure that it did indeed stop self.assertFalse(self.event.wait(1)) subscriber.stop() @attr('SMOKE') def test_dm_end_2_end(self): #-------------------------------------------------------------------------------- # Set up a stream and have a mock instrument (producer) send data #-------------------------------------------------------------------------------- self.event.clear() # Get a precompiled parameter dictionary with basic ctd fields pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict',id_only=True) context_ids = self.dataset_management.read_parameter_contexts(pdict_id, id_only=True) # Add a field that supports binary data input. bin_context = ParameterContext('binary', param_type=ArrayType()) context_ids.append(self.dataset_management.create_parameter_context('binary', bin_context.dump())) # Add another field that supports dictionary elements. rec_context = ParameterContext('records', param_type=RecordType()) context_ids.append(self.dataset_management.create_parameter_context('records', rec_context.dump())) pdict_id = self.dataset_management.create_parameter_dictionary('replay_pdict', parameter_context_ids=context_ids, temporal_context='time') stream_definition = self.pubsub_management.create_stream_definition('ctd data', parameter_dictionary_id=pdict_id) stream_id, route = self.pubsub_management.create_stream('producer', exchange_point=self.exchange_point_name, stream_definition_id=stream_definition) #-------------------------------------------------------------------------------- # Start persisting the data on the stream # - Get the ingestion configuration from the resource registry # - Create the dataset # - call persist_data_stream to setup the subscription for the ingestion workers # on the stream that you specify which causes the data to be persisted #-------------------------------------------------------------------------------- ingest_config_id = self.get_ingestion_config() dataset_id = self.create_dataset(pdict_id) self.ingestion_management.persist_data_stream(stream_id=stream_id, ingestion_configuration_id=ingest_config_id, dataset_id=dataset_id) #-------------------------------------------------------------------------------- # Now the granules are ingesting and persisted #-------------------------------------------------------------------------------- self.launch_producer(stream_id) self.wait_until_we_have_enough_granules(dataset_id,4) #-------------------------------------------------------------------------------- # Now get the data in one chunk using an RPC Call to start_retreive #-------------------------------------------------------------------------------- replay_data = self.data_retriever.retrieve(dataset_id) self.assertIsInstance(replay_data, Granule) rdt = RecordDictionaryTool.load_from_granule(replay_data) self.assertTrue((rdt['time'][:10] == np.arange(10)).all(),'%s' % rdt['time'][:]) self.assertTrue((rdt['binary'][:10] == np.array(['hi']*10, dtype='object')).all()) #-------------------------------------------------------------------------------- # Now to try the streamed approach #-------------------------------------------------------------------------------- replay_stream_id, replay_route = self.pubsub_management.create_stream('replay_out', exchange_point=self.exchange_point_name, stream_definition_id=stream_definition) self.replay_id, process_id = self.data_retriever.define_replay(dataset_id=dataset_id, stream_id=replay_stream_id) log.info('Process ID: %s', process_id) replay_client = ReplayClient(process_id) #-------------------------------------------------------------------------------- # Create the listening endpoint for the the retriever to talk to #-------------------------------------------------------------------------------- xp = self.container.ex_manager.create_xp(self.exchange_point_name) subscriber = StandaloneStreamSubscriber(self.exchange_space_name, self.validate_granule_subscription) self.queue_buffer.append(self.exchange_space_name) subscriber.start() subscriber.xn.bind(replay_route.routing_key, xp) self.data_retriever.start_replay_agent(self.replay_id) self.assertTrue(replay_client.await_agent_ready(5), 'The process never launched') replay_client.start_replay() self.assertTrue(self.event.wait(10)) subscriber.stop() self.data_retriever.cancel_replay_agent(self.replay_id) #-------------------------------------------------------------------------------- # Test the slicing capabilities #-------------------------------------------------------------------------------- granule = self.data_retriever.retrieve(dataset_id=dataset_id, query={'tdoa':slice(0,5)}) rdt = RecordDictionaryTool.load_from_granule(granule) b = rdt['time'] == np.arange(5) self.assertTrue(b.all() if not isinstance(b,bool) else b) def test_retrieve_and_transform(self): # Stream definition for the CTD data pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True) stream_def_id = self.pubsub_management.create_stream_definition('ctd data', parameter_dictionary_id=pdict_id) ctd_stream_id, route = self.pubsub_management.create_stream('ctd stream', 'xp1', stream_definition_id=stream_def_id) # Stream definition for the salinity data salinity_pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True) sal_stream_def_id = self.pubsub_management.create_stream_definition('sal data', parameter_dictionary_id=salinity_pdict_id) ingest_config_id = self.get_ingestion_config() dataset_id = self.create_dataset(pdict_id) #-------------------------------------------------------------------------------- # Again with this ridiculous problem #-------------------------------------------------------------------------------- self.get_datastore(dataset_id) self.ingestion_management.persist_data_stream(stream_id=ctd_stream_id, ingestion_configuration_id=ingest_config_id, dataset_id=dataset_id) rdt = RecordDictionaryTool(stream_definition_id=stream_def_id) rdt['time'] = np.arange(10) rdt['temp'] = np.random.randn(10) * 10 + 30 rdt['conductivity'] = np.random.randn(10) * 2 + 10 publisher = StandaloneStreamPublisher(ctd_stream_id, route) publisher.publish(rdt.to_granule()) rdt['time'] = np.arange(10,20) publisher.publish(rdt.to_granule()) self.wait_until_we_have_enough_granules(dataset_id, 2) granule = self.data_retriever.retrieve(dataset_id, None, None, 'ion.processes.data.transforms.ctd.ctd_L2_salinity', 'CTDL2SalinityTransformAlgorithm', kwargs=dict(params=sal_stream_def_id)) rdt = RecordDictionaryTool.load_from_granule(granule) for i in rdt['salinity']: self.assertNotEquals(i,0) def test_last_granule(self): #-------------------------------------------------------------------------------- # Create the necessary configurations for the test #-------------------------------------------------------------------------------- pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True) stream_def_id = self.pubsub_management.create_stream_definition('ctd parsed', parameter_dictionary_id=pdict_id) stream_id, route = self.pubsub_management.create_stream('last_granule', exchange_point=self.exchange_point_name, stream_definition_id=stream_def_id) config_id = self.get_ingestion_config() dataset_id = self.create_dataset(pdict_id) self.ingestion_management.persist_data_stream(stream_id=stream_id, ingestion_configuration_id=config_id, dataset_id=dataset_id) #-------------------------------------------------------------------------------- # Create the datastore first, #-------------------------------------------------------------------------------- self.get_datastore(dataset_id) self.publish_hifi(stream_id,route, 0) self.publish_hifi(stream_id,route, 1) self.wait_until_we_have_enough_granules(dataset_id,2) # I just need two success = False def verifier(): replay_granule = self.data_retriever.retrieve_last_granule(dataset_id) rdt = RecordDictionaryTool.load_from_granule(replay_granule) comp = rdt['time'] == np.arange(10) + 10 if not isinstance(comp,bool): return comp.all() return False success = poll(verifier) self.assertTrue(success) success = False def verify_points(): replay_granule = self.data_retriever.retrieve_last_data_points(dataset_id,5) rdt = RecordDictionaryTool.load_from_granule(replay_granule) comp = rdt['time'] == np.arange(15,20) if not isinstance(comp,bool): return comp.all() return False success = poll(verify_points) self.assertTrue(success) def test_replay_with_parameters(self): #-------------------------------------------------------------------------------- # Create the configurations and the dataset #-------------------------------------------------------------------------------- # Get a precompiled parameter dictionary with basic ctd fields pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict',id_only=True) context_ids = self.dataset_management.read_parameter_contexts(pdict_id, id_only=True) # Add a field that supports binary data input. bin_context = ParameterContext('binary', param_type=ArrayType()) context_ids.append(self.dataset_management.create_parameter_context('binary', bin_context.dump())) # Add another field that supports dictionary elements. rec_context = ParameterContext('records', param_type=RecordType()) context_ids.append(self.dataset_management.create_parameter_context('records', rec_context.dump())) pdict_id = self.dataset_management.create_parameter_dictionary('replay_pdict', parameter_context_ids=context_ids, temporal_context='time') stream_def_id = self.pubsub_management.create_stream_definition('replay_stream', parameter_dictionary_id=pdict_id) stream_id, route = self.pubsub_management.create_stream('replay_with_params', exchange_point=self.exchange_point_name, stream_definition_id=stream_def_id) config_id = self.get_ingestion_config() dataset_id = self.create_dataset(pdict_id) self.ingestion_management.persist_data_stream(stream_id=stream_id, ingestion_configuration_id=config_id, dataset_id=dataset_id) #-------------------------------------------------------------------------------- # Coerce the datastore into existence (beats race condition) #-------------------------------------------------------------------------------- self.get_datastore(dataset_id) self.launch_producer(stream_id) self.wait_until_we_have_enough_granules(dataset_id,4) query = { 'start_time': 0, 'end_time': 20, 'stride_time' : 2, 'parameters': ['time','temp'] } retrieved_data = self.data_retriever.retrieve(dataset_id=dataset_id,query=query) rdt = RecordDictionaryTool.load_from_granule(retrieved_data) comp = np.arange(0,20,2) == rdt['time'] self.assertTrue(comp.all(),'%s' % rdt.pretty_print()) self.assertEquals(set(rdt.iterkeys()), set(['time','temp'])) extents = self.dataset_management.dataset_extents(dataset_id=dataset_id, parameters=['time','temp']) self.assertTrue(extents['time']>=20) self.assertTrue(extents['temp']>=20) def test_repersist_data(self): pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict',id_only=True) stream_def_id = self.pubsub_management.create_stream_definition(name='ctd', parameter_dictionary_id=pdict_id) stream_id, route = self.pubsub_management.create_stream(name='repersist', exchange_point=self.exchange_point_name, stream_definition_id=stream_def_id) config_id = self.get_ingestion_config() dataset_id = self.create_dataset(pdict_id) self.ingestion_management.persist_data_stream(stream_id=stream_id, ingestion_configuration_id=config_id, dataset_id=dataset_id) self.get_datastore(dataset_id) self.publish_hifi(stream_id,route,0) self.publish_hifi(stream_id,route,1) self.wait_until_we_have_enough_granules(dataset_id,2) self.ingestion_management.unpersist_data_stream(stream_id=stream_id,ingestion_configuration_id=config_id) self.ingestion_management.persist_data_stream(stream_id=stream_id,ingestion_configuration_id=config_id,dataset_id=dataset_id) self.publish_hifi(stream_id,route,2) self.publish_hifi(stream_id,route,3) self.wait_until_we_have_enough_granules(dataset_id,4) success = False with gevent.timeout.Timeout(5): while not success: replay_granule = self.data_retriever.retrieve(dataset_id) rdt = RecordDictionaryTool.load_from_granule(replay_granule) comp = rdt['time'] == np.arange(0,40) if not isinstance(comp,bool): success = comp.all() gevent.sleep(1) self.assertTrue(success)
class VisualizationIntegrationTestHelper(IonIntegrationTestCase): def create_ctd_input_stream_and_data_product(self, data_product_name='ctd_parsed'): cc = self.container assertions = self.assertTrue # Now create client to DataProductManagementService self.rrclient = ResourceRegistryServiceClient(node=self.container.node) self.damsclient = DataAcquisitionManagementServiceClient(node=self.container.node) self.pubsubclient = PubsubManagementServiceClient(node=self.container.node) self.ingestclient = IngestionManagementServiceClient(node=self.container.node) self.imsclient = InstrumentManagementServiceClient(node=self.container.node) self.dataproductclient = DataProductManagementServiceClient(node=self.container.node) self.dataprocessclient = DataProcessManagementServiceClient(node=self.container.node) self.datasetclient = DatasetManagementServiceClient(node=self.container.node) self.workflowclient = WorkflowManagementServiceClient(node=self.container.node) self.process_dispatcher = ProcessDispatcherServiceClient(node=self.container.node) self.vis_client = VisualizationServiceClient(node=self.container.node) #------------------------------- # Create CTD Parsed as the initial data product #------------------------------- # create a stream definition for the data from the ctd simulator ctd_stream_def = SBE37_CDM_stream_definition() ctd_stream_def_id = self.pubsubclient.create_stream_definition(container=ctd_stream_def, name='Simulated CTD data') log.debug('Creating new CDM data product with a stream definition') craft = CoverageCraft sdom, tdom = craft.create_domains() sdom = sdom.dump() tdom = tdom.dump() parameter_dictionary = craft.create_parameters() parameter_dictionary = parameter_dictionary.dump() dp_obj = IonObject(RT.DataProduct, name=data_product_name, description='ctd stream test', temporal_domain = tdom, spatial_domain = sdom) ctd_parsed_data_product_id = self.dataproductclient.create_data_product(dp_obj, ctd_stream_def_id, parameter_dictionary) log.debug('new ctd_parsed_data_product_id = %s' % ctd_parsed_data_product_id) #Only ever need one device for testing purposes. instDevice_obj,_ = self.rrclient.find_resources(restype=RT.InstrumentDevice, name='SBE37IMDevice') if instDevice_obj: instDevice_id = instDevice_obj[0]._id else: instDevice_obj = IonObject(RT.InstrumentDevice, name='SBE37IMDevice', description="SBE37IMDevice", serial_number="12345" ) instDevice_id = self.imsclient.create_instrument_device(instrument_device=instDevice_obj) self.damsclient.assign_data_product(input_resource_id=instDevice_id, data_product_id=ctd_parsed_data_product_id) self.dataproductclient.activate_data_product_persistence(data_product_id=ctd_parsed_data_product_id) # Retrieve the id of the OUTPUT stream from the out Data Product stream_ids, _ = self.rrclient.find_objects(ctd_parsed_data_product_id, PRED.hasStream, None, True) assertions(len(stream_ids) > 0 ) ctd_stream_id = stream_ids[0] return ctd_stream_id, ctd_parsed_data_product_id def create_data_product(self, dp_name = "", dp_description = ""): craft = CoverageCraft sdom, tdom = craft.create_domains() sdom = sdom.dump() tdom = tdom.dump() parameter_dictionary = craft.create_parameters() # this creates a ParameterDictionary object parameter_dictionary = parameter_dictionary.dump() # this returns a python dictionary data_prod_obj = IonObject(RT.DataProduct, name=dp_name, description=dp_description, temporal_domain = tdom, spatial_domain = sdom) data_prod_id = self.create_data_product(data_prod_obj, stream_definition_id, parameter_dictionary) return data_prod_id, data_prod_obj def start_simple_input_stream_process(self, ctd_stream_id): return self.start_input_stream_process(ctd_stream_id) def start_sinusoidal_input_stream_process(self, ctd_stream_id): return self.start_input_stream_process(ctd_stream_id, 'ion.processes.data.sinusoidal_stream_publisher', 'SinusoidalCtdPublisher') def start_input_stream_process(self, ctd_stream_id, module = 'ion.processes.data.ctd_stream_publisher', class_name= 'SimpleCtdPublisher'): ### ### Start the process for producing the CTD data ### # process definition for the ctd simulator... producer_definition = ProcessDefinition() producer_definition.executable = { 'module':module, 'class':class_name } ctd_sim_procdef_id = self.process_dispatcher.create_process_definition(process_definition=producer_definition) # Start the ctd simulator to produce some data configuration = { 'process':{ 'stream_id':ctd_stream_id, } } ctd_sim_pid = self.process_dispatcher.schedule_process(process_definition_id=ctd_sim_procdef_id, configuration=configuration) return ctd_sim_pid def start_output_stream_and_listen(self, ctd_stream_id, data_product_stream_ids, message_count_per_stream=10): cc = self.container assertions = self.assertTrue ### ### Make a subscriber in the test to listen for transformed data ### salinity_subscription_id = self.pubsubclient.create_subscription( query=StreamQuery(data_product_stream_ids), exchange_name = 'workflow_test', exchange_point = 'science_data', name = "test workflow transformations", ) pid = cc.spawn_process(name='dummy_process_for_test', module='pyon.ion.process', cls='SimpleProcess', config={}) dummy_process = cc.proc_manager.procs[pid] subscriber_registrar = StreamSubscriberRegistrar(process=dummy_process, container=cc) result = gevent.event.AsyncResult() results = [] message_count = len(data_product_stream_ids) * message_count_per_stream def message_received(message, headers): # Heads results.append(message) if len(results) >= message_count: #Only wait for so many messages - per stream result.set(True) subscriber = subscriber_registrar.create_subscriber(exchange_name='workflow_test', callback=message_received) subscriber.start() # after the queue has been created it is safe to activate the subscription self.pubsubclient.activate_subscription(subscription_id=salinity_subscription_id) #Start the input stream process if ctd_stream_id is not None: ctd_sim_pid = self.start_simple_input_stream_process(ctd_stream_id) # Assert that we have received data assertions(result.get(timeout=30)) # stop the flow parse the messages... if ctd_stream_id is not None: self.process_dispatcher.cancel_process(ctd_sim_pid) # kill the ctd simulator process - that is enough data self.pubsubclient.deactivate_subscription(subscription_id=salinity_subscription_id) subscriber.stop() return results def validate_messages(self, results): cc = self.container assertions = self.assertTrue first_salinity_values = None for message in results: rdt = RecordDictionaryTool.load_from_granule(message) try: temp = get_safe(rdt, 'temp') # psd = PointSupplementStreamParser(stream_definition=self.ctd_stream_def, stream_granule=message) # temp = psd.get_values('temperature') # log.info(psd.list_field_names()) except KeyError as ke: temp = None if temp is not None: assertions(isinstance(temp, numpy.ndarray)) log.info( 'temperature=' + str(numpy.nanmin(temp))) first_salinity_values = None else: #psd = PointSupplementStreamParser(stream_definition=SalinityTransform.outgoing_stream_def, stream_granule=message) #log.info( psd.list_field_names()) # Test the handy info method for the names of fields in the stream def #assertions('salinity' in psd.list_field_names()) # you have to know the name of the coverage in stream def salinity = get_safe(rdt, 'salinity') #salinity = psd.get_values('salinity') log.info( 'salinity=' + str(numpy.nanmin(salinity))) # Check to see if salinity has values assertions(salinity != None) assertions(isinstance(salinity, numpy.ndarray)) assertions(numpy.nanmin(salinity) > 0.0) # salinity should always be greater than 0 if first_salinity_values is None: first_salinity_values = salinity.tolist() else: second_salinity_values = salinity.tolist() assertions(len(first_salinity_values) == len(second_salinity_values)) for idx in range(0,len(first_salinity_values)): assertions(first_salinity_values[idx]*2.0 == second_salinity_values[idx]) def validate_data_ingest_retrieve(self, dataset_id): assertions = self.assertTrue self.data_retriever = DataRetrieverServiceClient(node=self.container.node) #validate that data was ingested replay_granule = self.data_retriever.retrieve_last_granule(dataset_id) rdt = RecordDictionaryTool.load_from_granule(replay_granule) salinity = get_safe(rdt, 'salinity') assertions(salinity != None) #retrieve all the granules from the database and check the values replay_granule_all = self.data_retriever.retrieve(dataset_id) rdt = RecordDictionaryTool.load_from_granule(replay_granule_all) for k, v in rdt.iteritems(): if k == 'salinity': for val in numpy.nditer(v): assertions(val > 0) def create_salinity_data_process_definition(self): # Salinity: Data Process Definition #First look to see if it exists and if not, then create it dpd,_ = self.rrclient.find_resources(restype=RT.DataProcessDefinition, name='ctd_salinity') if len(dpd) > 0: return dpd[0] log.debug("Create data process definition SalinityTransform") dpd_obj = IonObject(RT.DataProcessDefinition, name='ctd_salinity', description='create a salinity data product', module='ion.processes.data.transforms.ctd.ctd_L2_salinity', class_name='SalinityTransform', process_source='SalinityTransform source code here...') try: ctd_L2_salinity_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj) except Excpetion as ex: self.fail("failed to create new SalinityTransform data process definition: %s" %ex) # create a stream definition for the data from the salinity Transform sal_stream_def_id = self.pubsubclient.create_stream_definition(container=SalinityTransform.outgoing_stream_def, name='Salinity') self.dataprocessclient.assign_stream_definition_to_data_process_definition(sal_stream_def_id, ctd_L2_salinity_dprocdef_id ) return ctd_L2_salinity_dprocdef_id def create_salinity_doubler_data_process_definition(self): #First look to see if it exists and if not, then create it dpd,_ = self.rrclient.find_resources(restype=RT.DataProcessDefinition, name='salinity_doubler') if len(dpd) > 0: return dpd[0] # Salinity Doubler: Data Process Definition log.debug("Create data process definition SalinityDoublerTransform") dpd_obj = IonObject(RT.DataProcessDefinition, name='salinity_doubler', description='create a salinity doubler data product', module='ion.processes.data.transforms.example_double_salinity', class_name='SalinityDoubler', process_source='SalinityDoubler source code here...') try: salinity_doubler_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj) except Exception as ex: self.fail("failed to create new SalinityDoubler data process definition: %s" %ex) # create a stream definition for the data from the salinity Transform salinity_double_stream_def_id = self.pubsubclient.create_stream_definition(container=SalinityDoubler.outgoing_stream_def, name='SalinityDoubler') self.dataprocessclient.assign_stream_definition_to_data_process_definition(salinity_double_stream_def_id, salinity_doubler_dprocdef_id ) return salinity_doubler_dprocdef_id def create_transform_process(self, data_process_definition_id, data_process_input_dp_id): data_process_definition = self.rrclient.read(data_process_definition_id) # Find the link between the output Stream Definition resource and the Data Process Definition resource stream_ids,_ = self.rrclient.find_objects(data_process_definition._id, PRED.hasStreamDefinition, RT.StreamDefinition, id_only=True) if not stream_ids: raise Inconsistent("The data process definition %s is missing an association to an output stream definition" % data_process_definition._id ) process_output_stream_def_id = stream_ids[0] #Concatenate the name of the workflow and data process definition for the name of the data product output data_process_name = data_process_definition.name # Create the output data product of the transform transform_dp_obj = IonObject(RT.DataProduct, name=data_process_name,description=data_process_definition.description) transform_dp_id = self.dataproductclient.create_data_product(transform_dp_obj, process_output_stream_def_id) self.dataproductclient.activate_data_product_persistence(data_product_id=transform_dp_id) #last one out of the for loop is the output product id output_data_product_id = transform_dp_id # Create the transform data process log.debug("create data_process and start it") data_process_id = self.dataprocessclient.create_data_process(data_process_definition._id, [data_process_input_dp_id], {'output':transform_dp_id}) self.dataprocessclient.activate_data_process(data_process_id) #Find the id of the output data stream stream_ids, _ = self.rrclient.find_objects(transform_dp_id, PRED.hasStream, None, True) if not stream_ids: raise Inconsistent("The data process %s is missing an association to an output stream" % data_process_id ) return data_process_id, output_data_product_id def create_google_dt_data_process_definition(self): #First look to see if it exists and if not, then create it dpd,_ = self.rrclient.find_resources(restype=RT.DataProcessDefinition, name='google_dt_transform') if len(dpd) > 0: return dpd[0] # Data Process Definition log.debug("Create data process definition GoogleDtTransform") dpd_obj = IonObject(RT.DataProcessDefinition, name='google_dt_transform', description='Convert data streams to Google DataTables', module='ion.processes.data.transforms.viz.google_dt', class_name='VizTransformGoogleDT', process_source='VizTransformGoogleDT source code here...') try: procdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj) except Exception as ex: self.fail("failed to create new VizTransformGoogleDT data process definition: %s" %ex) # create a stream definition for the data from the stream_def_id = self.pubsubclient.create_stream_definition(container=VizTransformGoogleDT.outgoing_stream_def, name='VizTransformGoogleDT') self.dataprocessclient.assign_stream_definition_to_data_process_definition(stream_def_id, procdef_id ) return procdef_id def validate_google_dt_transform_results(self, results): cc = self.container assertions = self.assertTrue # if its just one granule, wrap it up in a list so we can use the following for loop for a couple of cases if isinstance(results,Granule): results =[results] for g in results: if isinstance(g,Granule): tx = TaxyTool.load_from_granule(g) rdt = RecordDictionaryTool.load_from_granule(g) gdt_data = get_safe(rdt, 'google_dt_components') # IF this granule does not contains google dt, skip if gdt_data == None: continue gdt = gdt_data[0] assertions(gdt['viz_product_type'] == 'google_dt' ) assertions(len(gdt['data_description']) >= 0) # Need to come up with a better check assertions(len(gdt['data_content']) >= 0) def create_mpl_graphs_data_process_definition(self): #First look to see if it exists and if not, then create it dpd,_ = self.rrclient.find_resources(restype=RT.DataProcessDefinition, name='mpl_graphs_transform') if len(dpd) > 0: return dpd[0] #Data Process Definition log.debug("Create data process definition MatplotlibGraphsTransform") dpd_obj = IonObject(RT.DataProcessDefinition, name='mpl_graphs_transform', description='Convert data streams to Matplotlib graphs', module='ion.processes.data.transforms.viz.matplotlib_graphs', class_name='VizTransformMatplotlibGraphs', process_source='VizTransformMatplotlibGraphs source code here...') try: procdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj) except Exception as ex: self.fail("failed to create new VizTransformMatplotlibGraphs data process definition: %s" %ex) # create a stream definition for the data stream_def_id = self.pubsubclient.create_stream_definition(container=VizTransformMatplotlibGraphs.outgoing_stream_def, name='VizTransformMatplotlibGraphs') self.dataprocessclient.assign_stream_definition_to_data_process_definition(stream_def_id, procdef_id ) return procdef_id def validate_mpl_graphs_transform_results(self, results): cc = self.container assertions = self.assertTrue # if its just one granule, wrap it up in a list so we can use the following for loop for a couple of cases if isinstance(results,Granule): results =[results] for g in results: if isinstance(g,Granule): tx = TaxyTool.load_from_granule(g) rdt = RecordDictionaryTool.load_from_granule(g) graphs = get_safe(rdt, 'matplotlib_graphs') if graphs == None: continue for graph in graphs[0]: # At this point only dictionaries containing image data should be passed # For some reason non dictionary values are filtering through. if not isinstance(graph, dict): continue assertions(graph['viz_product_type'] == 'matplotlib_graphs' ) # check to see if the list (numpy array) contains actual images assertions(imghdr.what(graph['image_name'], h = graph['image_obj']) == 'png') def validate_vis_service_google_dt_results(self, results): assertions = self.assertTrue assertions(results) gdt_str = (results.lstrip("google.visualization.Query.setResponse(")).rstrip(")") assertions(len(gdt_str) > 0) return def validate_vis_service_mpl_graphs_results(self, results): assertions = self.assertTrue assertions(results) # check to see if the object passed is a dictionary with a valid image object in it image_format = results["content_type"].lstrip("image/") assertions(imghdr.what(results['image_name'], h = base64.decodestring(results['image_obj'])) == image_format) return
class TestDMEnd2End(IonIntegrationTestCase): def setUp(self): # Love the non pep-8 convention self._start_container() self.container.start_rel_from_url("res/deploy/r2deploy.yml") self.process_dispatcher = ProcessDispatcherServiceClient() self.pubsub_management = PubsubManagementServiceClient() self.resource_registry = ResourceRegistryServiceClient() self.dataset_management = DatasetManagementServiceClient() self.ingestion_management = IngestionManagementServiceClient() self.data_retriever = DataRetrieverServiceClient() self.pids = [] self.event = Event() self.exchange_space_name = "test_granules" self.exchange_point_name = "science_data" self.purge_queues() def purge_queues(self): xn = self.container.ex_manager.create_xn_queue("science_granule_ingestion") xn.purge() def tearDown(self): self.purge_queues() for pid in self.pids: self.process_dispatcher.cancel_process(pid) IngestionManagementIntTest.clean_subscriptions() def launch_producer(self, stream_id=""): # -------------------------------------------------------------------------------- # Create the process definition for the producer # -------------------------------------------------------------------------------- producer_definition = ProcessDefinition(name="Example Data Producer") producer_definition.executable = { "module": "ion.processes.data.example_data_producer", "class": "BetterDataProducer", } process_definition_id = self.process_dispatcher.create_process_definition( process_definition=producer_definition ) # -------------------------------------------------------------------------------- # Launch the producer # -------------------------------------------------------------------------------- config = DotDict() config.process.stream_id = stream_id pid = self.process_dispatcher.schedule_process( process_definition_id=process_definition_id, configuration=config ) self.pids.append(pid) def get_ingestion_config(self): # -------------------------------------------------------------------------------- # Grab the ingestion configuration from the resource registry # -------------------------------------------------------------------------------- # The ingestion configuration should have been created by the bootstrap service # which is configured through r2deploy.yml ingest_configs, _ = self.resource_registry.find_resources(restype=RT.IngestionConfiguration, id_only=True) return ingest_configs[0] def publish_hifi(self, stream_id, offset=0): pub = SimpleStreamPublisher.new_publisher(self.container, self.exchange_point_name, stream_id) black_box = CoverageCraft() black_box.rdt["time"] = np.arange(10) + (offset * 10) black_box.rdt["temp"] = (np.arange(10) + (offset * 10)) * 2 granule = black_box.to_granule() pub.publish(granule) def publish_fake_data(self, stream_id): for i in xrange(4): self.publish_hifi(stream_id, i) def get_datastore(self, dataset_id): dataset = self.dataset_management.read_dataset(dataset_id) datastore_name = dataset.datastore_name datastore = self.container.datastore_manager.get_datastore(datastore_name, DataStore.DS_PROFILE.SCIDATA) return datastore def validate_granule_subscription(self, msg, header): if msg == {}: return self.assertIsInstance(msg, Granule, "Message is improperly formatted. (%s)" % type(msg)) self.event.set() def wait_until_we_have_enough_granules(self, dataset_id="", granules=4): datastore = self.get_datastore(dataset_id) dataset = self.dataset_management.read_dataset(dataset_id) now = time.time() timeout = now + 10 done = False while not done: if now >= timeout: raise Timeout("Granules are not populating in time.") if len(datastore.query_view(dataset.view_name)) >= granules: done = True now = time.time() def create_dataset(self): craft = CoverageCraft sdom, tdom = craft.create_domains() sdom = sdom.dump() tdom = tdom.dump() pdict = craft.create_parameters() pdict = pdict.dump() dataset_id = self.dataset_management.create_dataset( "test_dataset", parameter_dict=pdict, spatial_domain=sdom, temporal_domain=tdom ) return dataset_id def test_coverage_ingest(self): stream_id = self.pubsub_management.create_stream() dataset_id = self.create_dataset() # I freaking hate this bug self.get_datastore(dataset_id) ingestion_config_id = self.get_ingestion_config() self.ingestion_management.persist_data_stream( stream_id=stream_id, ingestion_configuration_id=ingestion_config_id, dataset_id=dataset_id ) black_box = CoverageCraft() black_box.rdt["time"] = np.arange(20) black_box.rdt["temp"] = np.random.random(20) * 10 black_box.sync_with_granule() granule = black_box.to_granule() publisher = SimpleStreamPublisher.new_publisher(self.container, self.exchange_point_name, stream_id) publisher.publish(granule) self.wait_until_we_have_enough_granules(dataset_id, 1) coverage = DatasetManagementService._get_coverage(dataset_id) black_box = CoverageCraft(coverage) black_box.sync_rdt_with_coverage() comp = black_box.rdt["time"] == np.arange(20) self.assertTrue(comp.all()) black_box = CoverageCraft() black_box.rdt["time"] = np.arange(20) + 20 black_box.rdt["temp"] = np.random.random(20) * 10 black_box.sync_with_granule() granule = black_box.to_granule() publisher.publish(granule) self.wait_until_we_have_enough_granules(dataset_id, 2) coverage = DatasetManagementService._get_coverage(dataset_id) black_box = CoverageCraft(coverage) black_box.sync_rdt_with_coverage() comp = black_box.rdt["time"] == np.arange(40) self.assertTrue(comp.all()) granule = self.data_retriever.retrieve(dataset_id) black_box = CoverageCraft() black_box.sync_rdt_with_granule(granule) comp = black_box.rdt["time"] == np.arange(40) self.assertTrue(comp.all()) @attr("SMOKE") def test_dm_end_2_end(self): # -------------------------------------------------------------------------------- # Set up a stream and have a mock instrument (producer) send data # -------------------------------------------------------------------------------- stream_id = self.pubsub_management.create_stream() self.launch_producer(stream_id) # -------------------------------------------------------------------------------- # Start persisting the data on the stream # - Get the ingestion configuration from the resource registry # - Create the dataset # - call persist_data_stream to setup the subscription for the ingestion workers # on the stream that you specify which causes the data to be persisted # -------------------------------------------------------------------------------- ingest_config_id = self.get_ingestion_config() dataset_id = self.create_dataset() self.ingestion_management.persist_data_stream( stream_id=stream_id, ingestion_configuration_id=ingest_config_id, dataset_id=dataset_id ) # -------------------------------------------------------------------------------- # Now the granules are ingesting and persisted # -------------------------------------------------------------------------------- self.wait_until_we_have_enough_granules(dataset_id, 4) # -------------------------------------------------------------------------------- # Now get the data in one chunk using an RPC Call to start_retreive # -------------------------------------------------------------------------------- replay_data = self.data_retriever.retrieve(dataset_id) self.assertIsInstance(replay_data, Granule) # -------------------------------------------------------------------------------- # Now to try the streamed approach # -------------------------------------------------------------------------------- replay_id, stream_id = self.data_retriever.define_replay(dataset_id) # -------------------------------------------------------------------------------- # Create the listening endpoint for the the retriever to talk to # -------------------------------------------------------------------------------- xp = self.container.ex_manager.create_xp(self.exchange_point_name) xn = self.container.ex_manager.create_xn_queue(self.exchange_space_name) xn.bind("%s.data" % stream_id, xp) subscriber = SimpleStreamSubscriber.new_subscriber( self.container, self.exchange_space_name, self.validate_granule_subscription ) subscriber.start() self.data_retriever.start_replay(replay_id) fail = False try: self.event.wait(10) except gevent.Timeout: fail = True subscriber.stop() self.assertTrue(not fail, "Failed to validate the data.") def test_replay_by_time(self): log.info("starting test...") # -------------------------------------------------------------------------------- # Create the necessary configurations for the test # -------------------------------------------------------------------------------- stream_id = self.pubsub_management.create_stream() config_id = self.get_ingestion_config() dataset_id = self.create_dataset() self.ingestion_management.persist_data_stream( stream_id=stream_id, ingestion_configuration_id=config_id, dataset_id=dataset_id ) # -------------------------------------------------------------------------------- # Create the datastore first, # -------------------------------------------------------------------------------- # There is a race condition sometimes between the services and the process for # the creation of the datastore and it's instance, this ensures the datastore # exists before the process is even subscribing to data. self.get_datastore(dataset_id) self.publish_fake_data(stream_id) self.wait_until_we_have_enough_granules(dataset_id, 2) # I just need two replay_granule = self.data_retriever.retrieve(dataset_id, {"start_time": 0, "end_time": 6}) rdt = RecordDictionaryTool.load_from_granule(replay_granule) comp = rdt["time"] == np.array([0, 1, 2, 3, 4, 5]) try: log.info("Compared granule: %s", replay_granule.__dict__) log.info("Granule tax: %s", replay_granule.taxonomy.__dict__) except: pass self.assertTrue(comp.all()) def test_last_granule(self): # -------------------------------------------------------------------------------- # Create the necessary configurations for the test # -------------------------------------------------------------------------------- stream_id = self.pubsub_management.create_stream() config_id = self.get_ingestion_config() dataset_id = self.create_dataset() self.ingestion_management.persist_data_stream( stream_id=stream_id, ingestion_configuration_id=config_id, dataset_id=dataset_id ) # -------------------------------------------------------------------------------- # Create the datastore first, # -------------------------------------------------------------------------------- self.get_datastore(dataset_id) self.publish_hifi(stream_id, 0) self.publish_hifi(stream_id, 1) self.wait_until_we_have_enough_granules(dataset_id, 2) # I just need two replay_granule = self.data_retriever.retrieve_last_granule(dataset_id) rdt = RecordDictionaryTool.load_from_granule(replay_granule) comp = rdt["time"] == np.arange(10) + 10 self.assertTrue(comp.all()) def test_replay_with_parameters(self): # -------------------------------------------------------------------------------- # Create the configurations and the dataset # -------------------------------------------------------------------------------- stream_id = self.pubsub_management.create_stream() config_id = self.get_ingestion_config() dataset_id = self.create_dataset() self.ingestion_management.persist_data_stream( stream_id=stream_id, ingestion_configuration_id=config_id, dataset_id=dataset_id ) # -------------------------------------------------------------------------------- # Coerce the datastore into existence (beats race condition) # -------------------------------------------------------------------------------- self.get_datastore(dataset_id) self.launch_producer(stream_id) self.wait_until_we_have_enough_granules(dataset_id, 4) query = {"start_time": 0, "end_time": 20, "parameters": ["time", "temp"]} retrieved_data = self.data_retriever.retrieve(dataset_id=dataset_id, query=query) rdt = RecordDictionaryTool.load_from_granule(retrieved_data) comp = np.arange(20) == rdt["time"] self.assertTrue(comp.all(), "%s" % rdt.pretty_print()) self.assertEquals(set(rdt.iterkeys()), set(["time", "temp"])) def test_repersist_data(self): stream_id = self.pubsub_management.create_stream() config_id = self.get_ingestion_config() dataset_id = self.create_dataset() self.ingestion_management.persist_data_stream( stream_id=stream_id, ingestion_configuration_id=config_id, dataset_id=dataset_id ) self.get_datastore(dataset_id) self.publish_hifi(stream_id, 0) self.publish_hifi(stream_id, 1) self.wait_until_we_have_enough_granules(dataset_id, 2) self.ingestion_management.unpersist_data_stream(stream_id=stream_id, ingestion_configuration_id=config_id) self.ingestion_management.persist_data_stream( stream_id=stream_id, ingestion_configuration_id=config_id, dataset_id=dataset_id ) self.publish_hifi(stream_id, 2) self.publish_hifi(stream_id, 3) self.wait_until_we_have_enough_granules(dataset_id, 4) retrieved_granule = self.data_retriever.retrieve(dataset_id) rdt = RecordDictionaryTool.load_from_granule(retrieved_granule) comp = rdt["time"] == np.arange(0, 40) self.assertTrue(comp.all(), "Uh-oh: %s" % rdt["time"])