class BootstrapQCPostProcessor(BootstrapPlugin): ''' Sets up one QC Post Processing worker and initiates the Scheduler Service's interval every 24 hours. ''' def on_initial_bootstrap(self, process, config, **kwargs): if self.process_exists(process, 'qc_post_processor'): # Short circuit the bootstrap to make sure not more than one is ever started return self.scheduler_service = SchedulerServiceProcessClient(process=process) self.process_dispatcher = ProcessDispatcherServiceProcessClient(process=process) interval_key = uuid4().hex # Unique identifier for this process config = DotDict() config.process.interval_key = interval_key process_definition = ProcessDefinition(name='qc_post_processor', executable={'module':'ion.processes.data.transforms.qc_post_processing', 'class':'QCPostProcessing'}) process_definition_id = self.process_dispatcher.create_process_definition(process_definition) process_id = self.process_dispatcher.create_process(process_definition_id) self.process_dispatcher.schedule_process(process_definition_id, process_id=process_id, configuration=config) timer_id = self.scheduler_service.create_interval_timer(start_time=time.time(), end_time=-1, #Run FOREVER interval=3600*24, event_origin=interval_key) def process_exists(self, process, name): proc_ids, meta = process.container.resource_registry.find_resources(restype=RT.Process, id_only=True) return any([name in p['name'] for p in meta])
def setUp(self): # Start container logging.disable(logging.ERROR) self._start_container() self.container.start_rel_from_url('res/deploy/r2deploy.yml') # simulate preloading preload_ion_params(self.container) logging.disable(logging.NOTSET) #Instantiate a process to represent the test process=VisualizationServiceTestProcess() # Now create client to DataProductManagementService self.rrclient = ResourceRegistryServiceProcessClient(node=self.container.node, process=process) self.damsclient = DataAcquisitionManagementServiceProcessClient(node=self.container.node, process=process) self.pubsubclient = PubsubManagementServiceProcessClient(node=self.container.node, process=process) self.ingestclient = IngestionManagementServiceProcessClient(node=self.container.node, process=process) self.imsclient = InstrumentManagementServiceProcessClient(node=self.container.node, process=process) self.dataproductclient = DataProductManagementServiceProcessClient(node=self.container.node, process=process) self.dataprocessclient = DataProcessManagementServiceProcessClient(node=self.container.node, process=process) self.datasetclient = DatasetManagementServiceProcessClient(node=self.container.node, process=process) self.workflowclient = WorkflowManagementServiceProcessClient(node=self.container.node, process=process) self.process_dispatcher = ProcessDispatcherServiceProcessClient(node=self.container.node, process=process) self.data_retriever = DataRetrieverServiceProcessClient(node=self.container.node, process=process) self.vis_client = VisualizationServiceProcessClient(node=self.container.node, process=process) self.ctd_stream_def = SBE37_CDM_stream_definition()
def on_initial_bootstrap(self, process, config, **kwargs): if os.environ.get('PYCC_MODE'): # This environment is an ion integration test log.info('PYCC_MODE: skipping qc_post_processor launch') return if self.process_exists(process, 'qc_post_processor'): # Short circuit the bootstrap to make sure not more than one is ever started return self.scheduler_service = SchedulerServiceProcessClient(process=process) self.process_dispatcher = ProcessDispatcherServiceProcessClient(process=process) self.run_interval = CFG.get_safe('service.qc_processing.run_interval', 24) interval_key = uuid4().hex # Unique identifier for this process config = DotDict() config.process.interval_key = interval_key process_definition = ProcessDefinition(name='qc_post_processor', executable={'module':'ion.processes.data.transforms.qc_post_processing', 'class':'QCPostProcessing'}) process_definition_id = self.process_dispatcher.create_process_definition(process_definition) process_id = self.process_dispatcher.create_process(process_definition_id) self.process_dispatcher.schedule_process(process_definition_id, process_id=process_id, configuration=config) timer_id = self.scheduler_service.create_interval_timer(start_time=str(time.time()), end_time='-1', #Run FOREVER interval=3600*self.run_interval, event_origin=interval_key)
def on_initial_bootstrap(self, process, config, **kwargs): self.pds_client = ProcessDispatcherServiceProcessClient( process=process) self.resource_registry = ResourceRegistryServiceProcessClient( process=process) self.ingestion_worker(process, config) self.replay_defs(process, config) self.notification_worker(process, config) self.registration_worker(process, config) self.pydap_server(process, config)
class BootstrapQCPostProcessor(BootstrapPlugin): ''' Sets up one QC Post Processing worker and initiates the Scheduler Service's interval every 24 hours. ''' def on_initial_bootstrap(self, process, config, **kwargs): if os.environ.get('PYCC_MODE'): # This environment is an ion integration test log.info('PYCC_MODE: skipping qc_post_processor launch') return if self.process_exists(process, 'qc_post_processor'): # Short circuit the bootstrap to make sure not more than one is ever started return self.scheduler_service = SchedulerServiceProcessClient(process=process) self.process_dispatcher = ProcessDispatcherServiceProcessClient( process=process) self.run_interval = CFG.get_safe('service.qc_processing.run_interval', 24) interval_key = uuid4().hex # Unique identifier for this process config = DotDict() config.process.interval_key = interval_key process_definition = ProcessDefinition( name='qc_post_processor', executable={ 'module': 'ion.processes.data.transforms.qc_post_processing', 'class': 'QCPostProcessing' }) process_definition_id = self.process_dispatcher.create_process_definition( process_definition) process_id = self.process_dispatcher.create_process( process_definition_id) self.process_dispatcher.schedule_process(process_definition_id, process_id=process_id, configuration=config) timer_id = self.scheduler_service.create_interval_timer( start_time=str(time.time()), end_time='-1', #Run FOREVER interval=3600 * self.run_interval, event_origin=interval_key) def process_exists(self, process, name): proc_ids, meta = process.container.resource_registry.find_resources( restype=RT.Process, id_only=True) return any([name in p['name'] for p in meta if p['name']])
def on_initial_bootstrap(self, process, config, **kwargs): if self.process_exists(process, 'qc_post_processor'): # Short circuit the bootstrap to make sure not more than one is ever started return self.scheduler_service = SchedulerServiceProcessClient(process=process) self.process_dispatcher = ProcessDispatcherServiceProcessClient(process=process) interval_key = uuid4().hex # Unique identifier for this process config = DotDict() config.process.interval_key = interval_key process_definition = ProcessDefinition(name='qc_post_processor', executable={'module':'ion.processes.data.transforms.qc_post_processing', 'class':'QCPostProcessing'}) process_definition_id = self.process_dispatcher.create_process_definition(process_definition) process_id = self.process_dispatcher.create_process(process_definition_id) self.process_dispatcher.schedule_process(process_definition_id, process_id=process_id, configuration=config) timer_id = self.scheduler_service.create_interval_timer(start_time=time.time(), end_time=-1, #Run FOREVER interval=3600*24, event_origin=interval_key)
def setUp(self): # Start container logging.disable(logging.ERROR) self._start_container() self.container.start_rel_from_url('res/deploy/r2deploy.yml') # simulate preloading #preload_ion_params(self.container) logging.disable(logging.NOTSET) #Instantiate a process to represent the test process=VisualizationServiceTestProcess() # Now create client to DataProductManagementService self.rrclient = ResourceRegistryServiceProcessClient(node=self.container.node, process=process) self.damsclient = DataAcquisitionManagementServiceProcessClient(node=self.container.node, process=process) self.pubsubclient = PubsubManagementServiceProcessClient(node=self.container.node, process=process) self.ingestclient = IngestionManagementServiceProcessClient(node=self.container.node, process=process) self.imsclient = InstrumentManagementServiceProcessClient(node=self.container.node, process=process) self.dataproductclient = DataProductManagementServiceProcessClient(node=self.container.node, process=process) self.dataprocessclient = DataProcessManagementServiceProcessClient(node=self.container.node, process=process) self.datasetclient = DatasetManagementServiceProcessClient(node=self.container.node, process=process) self.process_dispatcher = ProcessDispatcherServiceProcessClient(node=self.container.node, process=process) self.data_retriever = DataRetrieverServiceProcessClient(node=self.container.node, process=process) self.vis_client = VisualizationServiceProcessClient(node=self.container.node, process=process) self.ctd_stream_def = SBE37_CDM_stream_definition()
def on_restart(self, process, config, **kwargs): pds_client = ProcessDispatcherServiceProcessClient(process=process) def restart_transform(transform_id): transform = process.container.resource_registry.read(transform_id) configuration = transform.configuration proc_def_ids, other = process.container.resource_registry.find_objects( subject=transform_id, predicate=PRED.hasProcessDefinition, id_only=True) if len(proc_def_ids) < 1: log.warning( 'Transform did not have a correct process definition.') return pid = pds_client.schedule_process( process_definition_id=proc_def_ids[0], configuration=configuration) transform.process_id = pid process.container.resource_registry.update(transform) restart_flag = config.get_safe('service.transform_management.restart', False) if restart_flag: transform_ids, meta = process.container.resource_registry.find_resources( restype=RT.Transform, id_only=True) for transform_id in transform_ids: restart_transform(transform_id)
def on_initial_bootstrap(self, process, config, **kwargs): self.pds_client = ProcessDispatcherServiceProcessClient(process=process) self.resource_registry = ResourceRegistryServiceProcessClient(process=process) self.ingestion_worker(process,config) self.replay_defs(process,config) self.notification_worker(process,config) self.registration_worker(process,config)
class BootstrapQCPostProcessor(BootstrapPlugin): """ Sets up one QC Post Processing worker and initiates the Scheduler Service's interval every 24 hours. """ def on_initial_bootstrap(self, process, config, **kwargs): if os.environ.get("PYCC_MODE"): # This environment is an ion integration test log.info("PYCC_MODE: skipping qc_post_processor launch") return if self.process_exists(process, "qc_post_processor"): # Short circuit the bootstrap to make sure not more than one is ever started return self.scheduler_service = SchedulerServiceProcessClient(process=process) self.process_dispatcher = ProcessDispatcherServiceProcessClient(process=process) self.run_interval = CFG.get_safe("service.qc_processing.run_interval", 24) interval_key = uuid4().hex # Unique identifier for this process config = DotDict() config.process.interval_key = interval_key process_definition = ProcessDefinition( name="qc_post_processor", executable={"module": "ion.processes.data.transforms.qc_post_processing", "class": "QCPostProcessing"}, ) process_definition_id = self.process_dispatcher.create_process_definition(process_definition) process_id = self.process_dispatcher.create_process(process_definition_id) self.process_dispatcher.schedule_process(process_definition_id, process_id=process_id, configuration=config) timer_id = self.scheduler_service.create_interval_timer( start_time=str(time.time()), end_time="-1", # Run FOREVER interval=3600 * self.run_interval, event_origin=interval_key, ) def process_exists(self, process, name): proc_ids, meta = process.container.resource_registry.find_resources(restype=RT.Process, id_only=True) return any([name in p["name"] for p in meta if p["name"]])
class BootstrapQCPostProcessor(BootstrapPlugin): ''' Sets up one QC Post Processing worker and initiates the Scheduler Service's interval every 24 hours. ''' def on_initial_bootstrap(self, process, config, **kwargs): # TODO: Temporary skip while refactoring QC work for M088 return if os.environ.get('PYCC_MODE'): # This environment is an ion integration test log.info('PYCC_MODE: skipping qc_post_processor launch') return if self.process_exists(process, 'qc_post_processor'): # Short circuit the bootstrap to make sure not more than one is ever started return self.scheduler_service = SchedulerServiceProcessClient(process=process) self.process_dispatcher = ProcessDispatcherServiceProcessClient(process=process) self.run_interval = CFG.get_safe('service.qc_processing.run_interval', 24) interval_key = uuid4().hex # Unique identifier for this process config = DotDict() config.process.interval_key = interval_key process_definition = ProcessDefinition(name='qc_post_processor', executable={'module':'ion.processes.data.transforms.qc_post_processing', 'class':'QCPostProcessing'}) process_definition_id = self.process_dispatcher.create_process_definition(process_definition) process_id = self.process_dispatcher.create_process(process_definition_id) self.process_dispatcher.schedule_process(process_definition_id, process_id=process_id, configuration=config) timer_id = self.scheduler_service.create_interval_timer(start_time=str(time.time()), end_time='-1', #Run FOREVER interval=3600*self.run_interval, event_origin=interval_key) def process_exists(self, process, name): proc_ids, meta = process.container.resource_registry.find_resources(restype=RT.Process, id_only=True) return any([name in p['name'] for p in meta if p['name']])
def on_initial_bootstrap(self, process, config, **kwargs): pds_client = ProcessDispatcherServiceProcessClient(process=process) ingestion_module = config.get_safe('bootstrap.processes.ingestion.module','ion.processes.data.ingestion.science_granule_ingestion_worker') ingestion_class = config.get_safe('bootstrap.processes.ingestion.class' ,'ScienceGranuleIngestionWorker') ingestion_datastore = config.get_safe('bootstrap.processes.ingestion.datastore_name', 'datasets') ingestion_queue = config.get_safe('bootstrap.processes.ingestion.queue' , 'science_granule_ingestion') ingestion_workers = config.get_safe('bootstrap.processes.ingestion.workers', 2) replay_module = config.get_safe('bootstrap.processes.replay.module', 'ion.processes.data.replay.replay_process') replay_class = config.get_safe('bootstrap.processes.replay.class' , 'ReplayProcess') process_definition = ProcessDefinition( name='ingestion_worker_process', description='Worker transform process for ingestion of datasets') process_definition.executable['module']= ingestion_module process_definition.executable['class'] = ingestion_class ingestion_procdef_id = pds_client.create_process_definition(process_definition=process_definition) #-------------------------------------------------------------------------------- # Simulate a HA ingestion worker by creating two of them #-------------------------------------------------------------------------------- config = DotDict() config.process.datastore_name = ingestion_datastore config.process.queue_name = ingestion_queue for i in xrange(ingestion_workers): pds_client.schedule_process(process_definition_id=ingestion_procdef_id, configuration=config) process_definition = ProcessDefinition(name='data_replay_process', description='Process for the replay of datasets') process_definition.executable['module']= replay_module process_definition.executable['class'] = replay_class pds_client.create_process_definition(process_definition=process_definition)
def await_agent_ready(self, replay_timeout=5): ''' Determines if the process has been started @param replay_timeout Time to wait before raising a timeout @retval True if the process has been started ''' if self.process: pd_cli = ProcessDispatcherServiceProcessClient( process=self.process) else: pd_cli = ProcessDispatcherServiceClient() process_gate = ProcessStateGate(pd_cli.read_process, self.replay_process_id, ProcessStateEnum.RUNNING) return process_gate. await (replay_timeout)
def setUp(self): # Start container self._start_container() self.container.start_rel_from_url('res/deploy/r2deploy.yml') #Instantiate a process to represent the test process=WorkflowServiceTestProcess() # Now create client to DataProductManagementService self.rrclient = ResourceRegistryServiceProcessClient(node=self.container.node, process=process) self.damsclient = DataAcquisitionManagementServiceProcessClient(node=self.container.node, process=process) self.pubsubclient = PubsubManagementServiceProcessClient(node=self.container.node, process=process) self.ingestclient = IngestionManagementServiceProcessClient(node=self.container.node, process=process) self.imsclient = InstrumentManagementServiceProcessClient(node=self.container.node, process=process) self.dataproductclient = DataProductManagementServiceProcessClient(node=self.container.node, process=process) self.dataprocessclient = DataProcessManagementServiceProcessClient(node=self.container.node, process=process) self.datasetclient = DatasetManagementServiceProcessClient(node=self.container.node, process=process) self.workflowclient = WorkflowManagementServiceProcessClient(node=self.container.node, process=process) self.process_dispatcher = ProcessDispatcherServiceProcessClient(node=self.container.node, process=process) self.data_retriever = DataRetrieverServiceProcessClient(node=self.container.node, process=process) self.ctd_stream_def = SBE37_CDM_stream_definition()
class TestVisualizationServiceIntegration(VisualizationIntegrationTestHelper): def setUp(self): # Start container logging.disable(logging.ERROR) self._start_container() self.container.start_rel_from_url('res/deploy/r2deploy.yml') # simulate preloading #preload_ion_params(self.container) logging.disable(logging.NOTSET) #Instantiate a process to represent the test process=VisualizationServiceTestProcess() # Now create client to DataProductManagementService self.rrclient = ResourceRegistryServiceProcessClient(node=self.container.node, process=process) self.damsclient = DataAcquisitionManagementServiceProcessClient(node=self.container.node, process=process) self.pubsubclient = PubsubManagementServiceProcessClient(node=self.container.node, process=process) self.ingestclient = IngestionManagementServiceProcessClient(node=self.container.node, process=process) self.imsclient = InstrumentManagementServiceProcessClient(node=self.container.node, process=process) self.dataproductclient = DataProductManagementServiceProcessClient(node=self.container.node, process=process) self.dataprocessclient = DataProcessManagementServiceProcessClient(node=self.container.node, process=process) self.datasetclient = DatasetManagementServiceProcessClient(node=self.container.node, process=process) self.process_dispatcher = ProcessDispatcherServiceProcessClient(node=self.container.node, process=process) self.data_retriever = DataRetrieverServiceProcessClient(node=self.container.node, process=process) self.vis_client = VisualizationServiceProcessClient(node=self.container.node, process=process) self.ctd_stream_def = SBE37_CDM_stream_definition() def validate_messages(self, msgs): msg = msgs rdt = RecordDictionaryTool.load_from_granule(msg.body) vardict = {} vardict['temp'] = get_safe(rdt, 'temp') vardict['time'] = get_safe(rdt, 'time') print vardict['time'] print vardict['temp'] @attr('LOCOINT') #@patch.dict('pyon.ion.exchange.CFG', {'container':{'exchange':{'auto_register': False}}}) @unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False),'Not integrated for CEI') def test_visualization_queue(self): #The list of data product streams to monitor data_product_stream_ids = list() #Create the input data product ctd_stream_id, ctd_parsed_data_product_id = self.create_ctd_input_stream_and_data_product() data_product_stream_ids.append(ctd_stream_id) user_queue_name = USER_VISUALIZATION_QUEUE xq = self.container.ex_manager.create_xn_queue(user_queue_name) salinity_subscription_id = self.pubsubclient.create_subscription( stream_ids=data_product_stream_ids, exchange_name = user_queue_name, name = "user visualization queue" ) subscriber = Subscriber(from_name=xq) subscriber.initialize() # after the queue has been created it is safe to activate the subscription self.pubsubclient.activate_subscription(subscription_id=salinity_subscription_id) #Start the output stream listener to monitor and collect messages #results = self.start_output_stream_and_listen(None, data_product_stream_ids) #Not sure why this is needed - but it is #subscriber._chan.stop_consume() ctd_sim_pid = self.start_simple_input_stream_process(ctd_stream_id) gevent.sleep(10.0) # Send some messages - don't care how many msg_count,_ = xq.get_stats() log.info('Messages in user queue 1: %s ' % msg_count) #Validate the data from each of the messages along the way #self.validate_messages(results) # for x in range(msg_count): # mo = subscriber.get_one_msg(timeout=1) # print mo.body # mo.ack() msgs = subscriber.get_all_msgs(timeout=2) for x in range(len(msgs)): msgs[x].ack() self.validate_messages(msgs[x]) # print msgs[x].body #Should be zero after pulling all of the messages. msg_count,_ = xq.get_stats() log.info('Messages in user queue 2: %s ' % msg_count) #Trying to continue to receive messages in the queue gevent.sleep(5.0) # Send some messages - don't care how many #Turning off after everything - since it is more representative of an always on stream of data! self.process_dispatcher.cancel_process(ctd_sim_pid) # kill the ctd simulator process - that is enough data #Should see more messages in the queue msg_count,_ = xq.get_stats() log.info('Messages in user queue 3: %s ' % msg_count) msgs = subscriber.get_all_msgs(timeout=2) for x in range(len(msgs)): msgs[x].ack() self.validate_messages(msgs[x]) #Should be zero after pulling all of the messages. msg_count,_ = xq.get_stats() log.info('Messages in user queue 4: %s ' % msg_count) subscriber.close() self.container.ex_manager.delete_xn(xq) @patch.dict(CFG, {'user_queue_monitor_timeout': 5}) def test_realtime_visualization(self): # #Start up multiple vis service workers if not a CEI launch # if not os.getenv('CEI_LAUNCH_TEST', False): # vpid1 = self.container.spawn_process('visualization_service1','ion.services.ans.visualization_service','VisualizationService', CFG ) # self.addCleanup(self.container.terminate_process, vpid1) # vpid2 = self.container.spawn_process('visualization_service2','ion.services.ans.visualization_service','VisualizationService', CFG ) # self.addCleanup(self.container.terminate_process, vpid2) ## Create the Highcharts workflow definition since there is no preload for the test #workflow_def_id = self.create_highcharts_workflow_def() #Create the input data product ctd_stream_id, ctd_parsed_data_product_id = self.create_ctd_input_stream_and_data_product() ctd_sim_pid = self.start_sinusoidal_input_stream_process(ctd_stream_id) vis_params ={} vis_token_resp = self.vis_client.initiate_realtime_visualization_data(data_product_id=ctd_parsed_data_product_id, visualization_parameters=simplejson.dumps(vis_params)) vis_token = ast.literal_eval(vis_token_resp)["rt_query_token"] result = gevent.event.AsyncResult() def get_vis_messages(get_data_count=7): #SHould be an odd number for round robbin processing by service workers get_cnt = 0 while get_cnt < get_data_count: vis_data = self.vis_client.get_realtime_visualization_data(vis_token) if (vis_data): self.validate_highcharts_transform_results(vis_data) get_cnt += 1 gevent.sleep(5) # simulates the polling from UI result.set(get_cnt) gevent.spawn(get_vis_messages) result.get(timeout=90) #Trying to continue to receive messages in the queue gevent.sleep(2.0) # Send some messages - don't care how many # Cleanup self.vis_client.terminate_realtime_visualization_data(vis_token) #Turning off after everything - since it is more representative of an always on stream of data! self.process_dispatcher.cancel_process(ctd_sim_pid) # kill the ctd simulator process - that is enough data @patch.dict(CFG, {'user_queue_monitor_timeout': 5}) @patch.dict(CFG, {'user_queue_monitor_size': 25}) @attr('CLEANUP') @unittest.skipIf(os.getenv('PYCC_MODE', False),'Not integrated for CEI') def test_realtime_visualization_cleanup(self): # #Start up multiple vis service workers if not a CEI launch # if not os.getenv('CEI_LAUNCH_TEST', False): # vpid1 = self.container.spawn_process('visualization_service1','ion.services.ans.visualization_service','VisualizationService', CFG ) # self.addCleanup(self.container.terminate_process, vpid1) # vpid2 = self.container.spawn_process('visualization_service2','ion.services.ans.visualization_service','VisualizationService', CFG ) # self.addCleanup(self.container.terminate_process, vpid2) #get the list of queues and message counts on the broker for the user vis queues try: queues = self.container.ex_manager.list_queues(name=USER_VISUALIZATION_QUEUE, return_columns=['name', 'messages']) q_names = [ q['name'] for q in queues if q['name']] #Get a list of only the queue names original_queue_count = len(q_names) except Exception, e: log.warn('Unable to get queue information from broker management plugin: ' + e.message) pass ## Create the highcharts workflow definition since there is no preload for the test #workflow_def_id = self.create_highcharts_workflow_def() #Create the input data product ctd_stream_id, ctd_parsed_data_product_id = self.create_ctd_input_stream_and_data_product() ctd_sim_pid = self.start_sinusoidal_input_stream_process(ctd_stream_id) #Start up a number of requests - and queues - to start accumulating messages. THe test will not clean them up #but instead check to see if the monitoring thread will. vis_token_resp = self.vis_client.initiate_realtime_visualization_data(data_product_id=ctd_parsed_data_product_id) bad_vis_token1 = ast.literal_eval(vis_token_resp)["rt_query_token"] vis_token_resp = self.vis_client.initiate_realtime_visualization_data(data_product_id=ctd_parsed_data_product_id) bad_vis_token2 = ast.literal_eval(vis_token_resp)["rt_query_token"] vis_token_resp = self.vis_client.initiate_realtime_visualization_data(data_product_id=ctd_parsed_data_product_id) bad_vis_token3 = ast.literal_eval(vis_token_resp)["rt_query_token"] vis_token_resp = self.vis_client.initiate_realtime_visualization_data(data_product_id=ctd_parsed_data_product_id) vis_token = ast.literal_eval(vis_token_resp)["rt_query_token"] #Get the default exchange space exchange = self.container.ex_manager.default_xs.exchange #get the list of queues and message counts on the broker for the user vis queues try: queues = self.container.ex_manager.list_queues(name=USER_VISUALIZATION_QUEUE, return_columns=['name', 'messages']) q_names = [ q['name'] for q in queues if q['name']] #Get a list of only the queue names self.assertIn(exchange + "." + bad_vis_token1, q_names) self.assertIn(exchange + "." + bad_vis_token2, q_names) self.assertIn(exchange + "." + bad_vis_token3, q_names) self.assertIn(exchange + "." + vis_token, q_names) except Exception, e: log.warn('Unable to get queue information from broker management plugin: ' + e.message) pass
class TestWorkflowManagementIntegration(VisualizationIntegrationTestHelper): def setUp(self): # Start container self._start_container() self.container.start_rel_from_url('res/deploy/r2deploy.yml') # simulate preloading preload_ion_params(self.container) #Instantiate a process to represent the test process = WorkflowServiceTestProcess() # Now create client to DataProductManagementService self.rrclient = ResourceRegistryServiceProcessClient( node=self.container.node, process=process) self.damsclient = DataAcquisitionManagementServiceProcessClient( node=self.container.node, process=process) self.pubsubclient = PubsubManagementServiceProcessClient( node=self.container.node, process=process) self.ingestclient = IngestionManagementServiceProcessClient( node=self.container.node, process=process) self.imsclient = InstrumentManagementServiceProcessClient( node=self.container.node, process=process) self.dataproductclient = DataProductManagementServiceProcessClient( node=self.container.node, process=process) self.dataprocessclient = DataProcessManagementServiceProcessClient( node=self.container.node, process=process) self.datasetclient = DatasetManagementServiceProcessClient( node=self.container.node, process=process) self.workflowclient = WorkflowManagementServiceProcessClient( node=self.container.node, process=process) self.process_dispatcher = ProcessDispatcherServiceProcessClient( node=self.container.node, process=process) self.data_retriever = DataRetrieverServiceProcessClient( node=self.container.node, process=process) self.ctd_stream_def = SBE37_CDM_stream_definition() @attr('LOCOINT') @unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False), 'Not integrated for CEI') def test_SA_transform_components(self): assertions = self.assertTrue #The list of data product streams to monitor data_product_stream_ids = list() #Create the input data product ctd_stream_id, ctd_parsed_data_product_id = self.create_ctd_input_stream_and_data_product( ) data_product_stream_ids.append(ctd_stream_id) ### ### Setup the first transformation ### # Salinity: Data Process Definition ctd_L2_salinity_dprocdef_id = self.create_salinity_data_process_definition( ) l2_salinity_all_data_process_id, ctd_l2_salinity_output_dp_id = self.create_transform_process( ctd_L2_salinity_dprocdef_id, ctd_parsed_data_product_id, 'salinity') ## get the stream id for the transform outputs stream_ids, _ = self.rrclient.find_objects( ctd_l2_salinity_output_dp_id, PRED.hasStream, None, True) assertions(len(stream_ids) > 0) sal_stream_id = stream_ids[0] data_product_stream_ids.append(sal_stream_id) ### ### Setup the second transformation ### # Salinity Doubler: Data Process Definition salinity_doubler_dprocdef_id = self.create_salinity_doubler_data_process_definition( ) salinity_double_data_process_id, salinity_doubler_output_dp_id = self.create_transform_process( salinity_doubler_dprocdef_id, ctd_l2_salinity_output_dp_id, 'salinity') stream_ids, _ = self.rrclient.find_objects( salinity_doubler_output_dp_id, PRED.hasStream, None, True) assertions(len(stream_ids) > 0) sal_dbl_stream_id = stream_ids[0] data_product_stream_ids.append(sal_dbl_stream_id) #Start the output stream listener to monitor and collect messages results = self.start_output_stream_and_listen(ctd_stream_id, data_product_stream_ids) #Stop the transform processes self.dataprocessclient.deactivate_data_process( salinity_double_data_process_id) self.dataprocessclient.deactivate_data_process( l2_salinity_all_data_process_id) #Validate the data from each of the messages along the way self.validate_messages(results) @attr('LOCOINT') @attr('SMOKE') @unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False), 'Not integrated for CEI') def test_transform_workflow(self): assertions = self.assertTrue log.debug("Building the workflow definition") workflow_def_obj = IonObject( RT.WorkflowDefinition, name='Salinity_Test_Workflow', description='tests a workflow of multiple transform data processes' ) workflow_data_product_name = 'TEST-Workflow_Output_Product' #Set a specific output product name #------------------------------------------------------------------------------------------------------------------------- log.debug("Adding a transformation process definition for salinity") #------------------------------------------------------------------------------------------------------------------------- ctd_L2_salinity_dprocdef_id = self.create_salinity_data_process_definition( ) workflow_step_obj = IonObject( 'DataProcessWorkflowStep', data_process_definition_id=ctd_L2_salinity_dprocdef_id, persist_process_output_data=False ) #Don't persist the intermediate data product workflow_def_obj.workflow_steps.append(workflow_step_obj) #------------------------------------------------------------------------------------------------------------------------- log.debug( "Adding a transformation process definition for salinity doubler") #------------------------------------------------------------------------------------------------------------------------- salinity_doubler_dprocdef_id = self.create_salinity_doubler_data_process_definition( ) workflow_step_obj = IonObject( 'DataProcessWorkflowStep', data_process_definition_id=salinity_doubler_dprocdef_id, ) workflow_def_obj.workflow_steps.append(workflow_step_obj) log.debug("Creating workflow def in the resource registry") workflow_def_id = self.workflowclient.create_workflow_definition( workflow_def_obj) aids = self.rrclient.find_associations(workflow_def_id, PRED.hasDataProcessDefinition) assertions(len(aids) == 2) #The list of data product streams to monitor data_product_stream_ids = list() log.debug("Creating the input data product") ctd_stream_id, ctd_parsed_data_product_id = self.create_ctd_input_stream_and_data_product( ) data_product_stream_ids.append(ctd_stream_id) log.debug("Creating and starting the workflow") workflow_id, workflow_product_id = self.workflowclient.create_data_process_workflow( workflow_def_id, ctd_parsed_data_product_id, persist_workflow_data_product=True, output_data_product_name=workflow_data_product_name, timeout=300) workflow_output_ids, _ = self.rrclient.find_subjects( RT.Workflow, PRED.hasOutputProduct, workflow_product_id, True) assertions(len(workflow_output_ids) == 1) log.debug("persisting the output product") #self.dataproductclient.activate_data_product_persistence(workflow_product_id) dataset_ids, _ = self.rrclient.find_objects(workflow_product_id, PRED.hasDataset, RT.Dataset, True) assertions(len(dataset_ids) == 1) dataset_id = dataset_ids[0] log.debug( "Verifying the output data product name matches what was specified in the workflow definition" ) workflow_product = self.rrclient.read(workflow_product_id) assertions( workflow_product.name.startswith(workflow_data_product_name), 'Nope: %s != %s' % (workflow_product.name, workflow_data_product_name)) log.debug( "Walking the associations to find the appropriate output data streams to validate the messages" ) workflow_dp_ids, _ = self.rrclient.find_objects( workflow_id, PRED.hasDataProduct, RT.DataProduct, True) assertions(len(workflow_dp_ids) == 2) for dp_id in workflow_dp_ids: stream_ids, _ = self.rrclient.find_objects(dp_id, PRED.hasStream, None, True) assertions(len(stream_ids) == 1) data_product_stream_ids.append(stream_ids[0]) log.debug("data_product_stream_ids: %s" % data_product_stream_ids) log.debug( "Starting the output stream listener to monitor to collect messages" ) results = self.start_output_stream_and_listen(ctd_stream_id, data_product_stream_ids) log.debug("results::: %s" % results) log.debug("Stopping the workflow processes") self.workflowclient.terminate_data_process_workflow( workflow_id, False, timeout=250) # Should test true at some point log.debug("Making sure the Workflow object was removed") objs, _ = self.rrclient.find_resources(restype=RT.Workflow) assertions(len(objs) == 0) log.debug( "Validating the data from each of the messages along the way") self.validate_messages(results) log.debug( "Checking to see if dataset id = %s, was persisted, and that it can be retrieved...." % dataset_id) self.validate_data_ingest_retrieve(dataset_id) log.debug("Cleaning up to make sure delete is correct.") self.workflowclient.delete_workflow_definition(workflow_def_id) """ workflow_def_ids,_ = self.rrclient.find_resources(restype=RT.WorkflowDefinition) assertions(len(workflow_def_ids) == 0 ) """ @attr('LOCOINT') @unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False), 'Not integrated for CEI') def test_highcharts_transform_workflow(self): assertions = self.assertTrue # Build the workflow definition workflow_def_obj = IonObject( RT.WorkflowDefinition, name='HighCharts_Test_Workflow', description= 'Tests the workflow of converting stream data to HighCharts') #Add a transformation process definition highcharts_procdef_id = self.create_highcharts_data_process_definition( ) workflow_step_obj = IonObject( 'DataProcessWorkflowStep', data_process_definition_id=highcharts_procdef_id, persist_process_output_data=False) workflow_def_obj.workflow_steps.append(workflow_step_obj) #Create it in the resource registry workflow_def_id = self.workflowclient.create_workflow_definition( workflow_def_obj) #The list of data product streams to monitor data_product_stream_ids = list() #Create the input data product ctd_stream_id, ctd_parsed_data_product_id = self.create_ctd_input_stream_and_data_product( ) data_product_stream_ids.append(ctd_stream_id) #Create and start the workflow workflow_id, workflow_product_id = self.workflowclient.create_data_process_workflow( workflow_def_id, ctd_parsed_data_product_id, timeout=60) workflow_output_ids, _ = self.rrclient.find_subjects( RT.Workflow, PRED.hasOutputProduct, workflow_product_id, True) assertions(len(workflow_output_ids) == 1) #Walk the associations to find the appropriate output data streams to validate the messages workflow_dp_ids, _ = self.rrclient.find_objects( workflow_id, PRED.hasDataProduct, RT.DataProduct, True) assertions(len(workflow_dp_ids) == 1) for dp_id in workflow_dp_ids: stream_ids, _ = self.rrclient.find_objects(dp_id, PRED.hasStream, None, True) assertions(len(stream_ids) == 1) data_product_stream_ids.append(stream_ids[0]) #Start the output stream listener to monitor and collect messages results = self.start_output_stream_and_listen(ctd_stream_id, data_product_stream_ids) #Stop the workflow processes self.workflowclient.terminate_data_process_workflow( workflow_id=workflow_id, delete_data_products=False, timeout=60) # Should test true at some point #Validate the data from each of the messages along the way self.validate_highcharts_transform_results(results) #Cleanup to make sure delete is correct. self.workflowclient.delete_workflow_definition(workflow_def_id) """ workflow_def_ids,_ = self.rrclient.find_resources(restype=RT.WorkflowDefinition) assertions(len(workflow_def_ids) == 0 ) """ @attr('LOCOINT') @unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False), 'Not integrated for CEI') def test_mpl_graphs_transform_workflow(self): assertions = self.assertTrue # Build the workflow definition workflow_def_obj = IonObject( RT.WorkflowDefinition, name='Mpl_Graphs_Test_Workflow', description= 'Tests the workflow of converting stream data to Matplotlib graphs' ) #Add a transformation process definition mpl_graphs_procdef_id = self.create_mpl_graphs_data_process_definition( ) workflow_step_obj = IonObject( 'DataProcessWorkflowStep', data_process_definition_id=mpl_graphs_procdef_id, persist_process_output_data=False) workflow_def_obj.workflow_steps.append(workflow_step_obj) #Create it in the resource registry workflow_def_id = self.workflowclient.create_workflow_definition( workflow_def_obj) #The list of data product streams to monitor data_product_stream_ids = list() #Create the input data product ctd_stream_id, ctd_parsed_data_product_id = self.create_ctd_input_stream_and_data_product( ) data_product_stream_ids.append(ctd_stream_id) #Create and start the workflow workflow_id, workflow_product_id = self.workflowclient.create_data_process_workflow( workflow_def_id, ctd_parsed_data_product_id, persist_workflow_data_product=True, timeout=60) workflow_output_ids, _ = self.rrclient.find_subjects( RT.Workflow, PRED.hasOutputProduct, workflow_product_id, True) assertions(len(workflow_output_ids) == 1) #Walk the associations to find the appropriate output data streams to validate the messages workflow_dp_ids, _ = self.rrclient.find_objects( workflow_id, PRED.hasDataProduct, RT.DataProduct, True) assertions(len(workflow_dp_ids) == 1) for dp_id in workflow_dp_ids: stream_ids, _ = self.rrclient.find_objects(dp_id, PRED.hasStream, None, True) assertions(len(stream_ids) == 1) data_product_stream_ids.append(stream_ids[0]) #Start the output stream listener to monitor and collect messages results = self.start_output_stream_and_listen(ctd_stream_id, data_product_stream_ids) #Stop the workflow processes self.workflowclient.terminate_data_process_workflow( workflow_id=workflow_id, delete_data_products=False, timeout=60) # Should test true at some point #Validate the data from each of the messages along the way self.validate_mpl_graphs_transform_results(results) # Check to see if ingestion worked. Extract the granules from data_retrieval. # First find the dataset associated with the output dp product ds_ids, _ = self.rrclient.find_objects( workflow_dp_ids[len(workflow_dp_ids) - 1], PRED.hasDataset, RT.Dataset, True) retrieved_granule = self.data_retriever.retrieve_last_data_points( ds_ids[0], 10) #Validate the data from each of the messages along the way self.validate_mpl_graphs_transform_results(retrieved_granule) #Cleanup to make sure delete is correct. self.workflowclient.delete_workflow_definition(workflow_def_id) """ workflow_def_ids,_ = self.rrclient.find_resources(restype=RT.WorkflowDefinition) assertions(len(workflow_def_ids) == 0 ) """ @attr('LOCOINT') @unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False), 'Not integrated for CEI') def test_multiple_workflow_instances(self): assertions = self.assertTrue # Build the workflow definition workflow_def_obj = IonObject( RT.WorkflowDefinition, name='Multiple_Test_Workflow', description='Tests the workflow of converting stream data') #Add a transformation process definition highcharts_procdef_id = self.create_highcharts_data_process_definition( ) workflow_step_obj = IonObject( 'DataProcessWorkflowStep', data_process_definition_id=highcharts_procdef_id, persist_process_output_data=False) workflow_def_obj.workflow_steps.append(workflow_step_obj) #Create it in the resource registry workflow_def_id = self.workflowclient.create_workflow_definition( workflow_def_obj) #The list of data product streams to monitor data_product_stream_ids = list() #Create the first input data product ctd_stream_id1, ctd_parsed_data_product_id1 = self.create_ctd_input_stream_and_data_product( 'ctd_parsed1') data_product_stream_ids.append(ctd_stream_id1) #Create and start the first workflow workflow_id1, workflow_product_id1 = self.workflowclient.create_data_process_workflow( workflow_def_id, ctd_parsed_data_product_id1, timeout=60) #Create the second input data product ctd_stream_id2, ctd_parsed_data_product_id2 = self.create_ctd_input_stream_and_data_product( 'ctd_parsed2') data_product_stream_ids.append(ctd_stream_id2) #Create and start the second workflow workflow_id2, workflow_product_id2 = self.workflowclient.create_data_process_workflow( workflow_def_id, ctd_parsed_data_product_id2, timeout=60) #Walk the associations to find the appropriate output data streams to validate the messages workflow_ids, _ = self.rrclient.find_resources(restype=RT.Workflow) assertions(len(workflow_ids) == 2) #Start the first input stream process ctd_sim_pid1 = self.start_sinusoidal_input_stream_process( ctd_stream_id1) #Start the second input stream process ctd_sim_pid2 = self.start_simple_input_stream_process(ctd_stream_id2) #Start the output stream listener to monitor a set number of messages being sent through the workflows results = self.start_output_stream_and_listen( None, data_product_stream_ids, message_count_per_stream=5) # stop the flow of messages... self.process_dispatcher.cancel_process( ctd_sim_pid1 ) # kill the ctd simulator process - that is enough data self.process_dispatcher.cancel_process(ctd_sim_pid2) #Stop the first workflow processes self.workflowclient.terminate_data_process_workflow( workflow_id=workflow_id1, delete_data_products=False, timeout=60) # Should test true at some point #Stop the second workflow processes self.workflowclient.terminate_data_process_workflow( workflow_id=workflow_id2, delete_data_products=False, timeout=60) # Should test true at some point workflow_ids, _ = self.rrclient.find_resources(restype=RT.Workflow) assertions(len(workflow_ids) == 0) #Cleanup to make sure delete is correct. self.workflowclient.delete_workflow_definition(workflow_def_id) """
class TestWorkflowManagementIntegration(VisualizationIntegrationTestHelper): def setUp(self): # Start container self._start_container() self.container.start_rel_from_url('res/deploy/r2deploy.yml') #Instantiate a process to represent the test process=WorkflowServiceTestProcess() # Now create client to DataProductManagementService self.rrclient = ResourceRegistryServiceProcessClient(node=self.container.node, process=process) self.damsclient = DataAcquisitionManagementServiceProcessClient(node=self.container.node, process=process) self.pubsubclient = PubsubManagementServiceProcessClient(node=self.container.node, process=process) self.ingestclient = IngestionManagementServiceProcessClient(node=self.container.node, process=process) self.imsclient = InstrumentManagementServiceProcessClient(node=self.container.node, process=process) self.dataproductclient = DataProductManagementServiceProcessClient(node=self.container.node, process=process) self.dataprocessclient = DataProcessManagementServiceProcessClient(node=self.container.node, process=process) self.datasetclient = DatasetManagementServiceProcessClient(node=self.container.node, process=process) self.workflowclient = WorkflowManagementServiceProcessClient(node=self.container.node, process=process) self.process_dispatcher = ProcessDispatcherServiceProcessClient(node=self.container.node, process=process) self.data_retriever = DataRetrieverServiceProcessClient(node=self.container.node, process=process) self.ctd_stream_def = SBE37_CDM_stream_definition() @attr('LOCOINT') @unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False),'Not integrated for CEI') def test_SA_transform_components(self): assertions = self.assertTrue #The list of data product streams to monitor data_product_stream_ids = list() #Create the input data product ctd_stream_id, ctd_parsed_data_product_id = self.create_ctd_input_stream_and_data_product() data_product_stream_ids.append(ctd_stream_id) ### ### Setup the first transformation ### # Salinity: Data Process Definition ctd_L2_salinity_dprocdef_id = self.create_salinity_data_process_definition() l2_salinity_all_data_process_id, ctd_l2_salinity_output_dp_id = self.create_transform_process(ctd_L2_salinity_dprocdef_id,ctd_parsed_data_product_id, 'salinity' ) ## get the stream id for the transform outputs stream_ids, _ = self.rrclient.find_objects(ctd_l2_salinity_output_dp_id, PRED.hasStream, None, True) assertions(len(stream_ids) > 0 ) sal_stream_id = stream_ids[0] data_product_stream_ids.append(sal_stream_id) ### ### Setup the second transformation ### # Salinity Doubler: Data Process Definition salinity_doubler_dprocdef_id = self.create_salinity_doubler_data_process_definition() salinity_double_data_process_id, salinity_doubler_output_dp_id = self.create_transform_process(salinity_doubler_dprocdef_id, ctd_l2_salinity_output_dp_id, 'salinity' ) stream_ids, _ = self.rrclient.find_objects(salinity_doubler_output_dp_id, PRED.hasStream, None, True) assertions(len(stream_ids) > 0 ) sal_dbl_stream_id = stream_ids[0] data_product_stream_ids.append(sal_dbl_stream_id) #Start the output stream listener to monitor and collect messages results = self.start_output_stream_and_listen(ctd_stream_id, data_product_stream_ids) #Stop the transform processes self.dataprocessclient.deactivate_data_process(salinity_double_data_process_id) self.dataprocessclient.deactivate_data_process(l2_salinity_all_data_process_id) #Validate the data from each of the messages along the way self.validate_messages(results) @attr('LOCOINT') @attr('SMOKE') @unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False),'Not integrated for CEI') def test_transform_workflow(self): assertions = self.assertTrue log.debug("Building the workflow definition") workflow_def_obj = IonObject(RT.WorkflowDefinition, name='Salinity_Test_Workflow', description='tests a workflow of multiple transform data processes') workflow_data_product_name = 'TEST-Workflow_Output_Product' #Set a specific output product name #------------------------------------------------------------------------------------------------------------------------- log.debug( "Adding a transformation process definition for salinity") #------------------------------------------------------------------------------------------------------------------------- ctd_L2_salinity_dprocdef_id = self.create_salinity_data_process_definition() workflow_step_obj = IonObject('DataProcessWorkflowStep', data_process_definition_id=ctd_L2_salinity_dprocdef_id, persist_process_output_data=False) #Don't persist the intermediate data product workflow_def_obj.workflow_steps.append(workflow_step_obj) #------------------------------------------------------------------------------------------------------------------------- log.debug( "Adding a transformation process definition for salinity doubler") #------------------------------------------------------------------------------------------------------------------------- salinity_doubler_dprocdef_id = self.create_salinity_doubler_data_process_definition() workflow_step_obj = IonObject('DataProcessWorkflowStep', data_process_definition_id=salinity_doubler_dprocdef_id, ) workflow_def_obj.workflow_steps.append(workflow_step_obj) log.debug( "Creating workflow def in the resource registry") workflow_def_id = self.workflowclient.create_workflow_definition(workflow_def_obj) aids = self.rrclient.find_associations(workflow_def_id, PRED.hasDataProcessDefinition) assertions(len(aids) == 2 ) #The list of data product streams to monitor data_product_stream_ids = list() log.debug( "Creating the input data product") ctd_stream_id, ctd_parsed_data_product_id = self.create_ctd_input_stream_and_data_product() data_product_stream_ids.append(ctd_stream_id) log.debug( "Creating and starting the workflow") workflow_id, workflow_product_id = self.workflowclient.create_data_process_workflow(workflow_def_id, ctd_parsed_data_product_id, persist_workflow_data_product=True, output_data_product_name=workflow_data_product_name, timeout=300) workflow_output_ids,_ = self.rrclient.find_subjects(RT.Workflow, PRED.hasOutputProduct, workflow_product_id, True) assertions(len(workflow_output_ids) == 1 ) log.debug( "persisting the output product") #self.dataproductclient.activate_data_product_persistence(workflow_product_id) dataset_ids,_ = self.rrclient.find_objects(workflow_product_id, PRED.hasDataset, RT.Dataset, True) assertions(len(dataset_ids) == 1 ) dataset_id = dataset_ids[0] log.debug( "Verifying the output data product name matches what was specified in the workflow definition") workflow_product = self.rrclient.read(workflow_product_id) assertions(workflow_product.name.startswith(workflow_data_product_name), 'Nope: %s != %s' % (workflow_product.name, workflow_data_product_name)) log.debug( "Walking the associations to find the appropriate output data streams to validate the messages") workflow_dp_ids,_ = self.rrclient.find_objects(workflow_id, PRED.hasDataProduct, RT.DataProduct, True) assertions(len(workflow_dp_ids) == 2 ) for dp_id in workflow_dp_ids: stream_ids, _ = self.rrclient.find_objects(dp_id, PRED.hasStream, None, True) assertions(len(stream_ids) == 1 ) data_product_stream_ids.append(stream_ids[0]) log.debug( "data_product_stream_ids: %s" % data_product_stream_ids) log.debug( "Starting the output stream listener to monitor to collect messages") results = self.start_output_stream_and_listen(ctd_stream_id, data_product_stream_ids) log.debug( "results::: %s" % results) log.debug( "Stopping the workflow processes") self.workflowclient.terminate_data_process_workflow(workflow_id, False, timeout=250) # Should test true at some point log.debug( "Making sure the Workflow object was removed") objs, _ = self.rrclient.find_resources(restype=RT.Workflow) assertions(len(objs) == 0) log.debug( "Validating the data from each of the messages along the way") self.validate_messages(results) log.debug( "Checking to see if dataset id = %s, was persisted, and that it can be retrieved...." % dataset_id) self.validate_data_ingest_retrieve(dataset_id) log.debug( "Cleaning up to make sure delete is correct.") self.workflowclient.delete_workflow_definition(workflow_def_id) workflow_def_ids,_ = self.rrclient.find_resources(restype=RT.WorkflowDefinition) assertions(len(workflow_def_ids) == 0 ) @attr('LOCOINT') @unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False),'Not integrated for CEI') def test_google_dt_transform_workflow(self): assertions = self.assertTrue # Build the workflow definition workflow_def_obj = IonObject(RT.WorkflowDefinition, name='GoogleDT_Test_Workflow',description='Tests the workflow of converting stream data to Google DT') #Add a transformation process definition google_dt_procdef_id = self.create_google_dt_data_process_definition() workflow_step_obj = IonObject('DataProcessWorkflowStep', data_process_definition_id=google_dt_procdef_id, persist_process_output_data=False) workflow_def_obj.workflow_steps.append(workflow_step_obj) #Create it in the resource registry workflow_def_id = self.workflowclient.create_workflow_definition(workflow_def_obj) #The list of data product streams to monitor data_product_stream_ids = list() #Create the input data product ctd_stream_id, ctd_parsed_data_product_id = self.create_ctd_input_stream_and_data_product() data_product_stream_ids.append(ctd_stream_id) #Create and start the workflow workflow_id, workflow_product_id = self.workflowclient.create_data_process_workflow(workflow_def_id, ctd_parsed_data_product_id, timeout=60) workflow_output_ids,_ = self.rrclient.find_subjects(RT.Workflow, PRED.hasOutputProduct, workflow_product_id, True) assertions(len(workflow_output_ids) == 1 ) #Walk the associations to find the appropriate output data streams to validate the messages workflow_dp_ids,_ = self.rrclient.find_objects(workflow_id, PRED.hasDataProduct, RT.DataProduct, True) assertions(len(workflow_dp_ids) == 1 ) for dp_id in workflow_dp_ids: stream_ids, _ = self.rrclient.find_objects(dp_id, PRED.hasStream, None, True) assertions(len(stream_ids) == 1 ) data_product_stream_ids.append(stream_ids[0]) #Start the output stream listener to monitor and collect messages results = self.start_output_stream_and_listen(ctd_stream_id, data_product_stream_ids) #Stop the workflow processes self.workflowclient.terminate_data_process_workflow(workflow_id=workflow_id,delete_data_products=False, timeout=60) # Should test true at some point #Validate the data from each of the messages along the way self.validate_google_dt_transform_results(results) """ # Check to see if ingestion worked. Extract the granules from data_retrieval. # First find the dataset associated with the output dp product ds_ids,_ = self.rrclient.find_objects(workflow_dp_ids[len(workflow_dp_ids) - 1], PRED.hasDataset, RT.Dataset, True) retrieved_granule = self.data_retriever.retrieve(ds_ids[0]) #Validate the data from each of the messages along the way self.validate_google_dt_transform_results(retrieved_granule) """ #Cleanup to make sure delete is correct. self.workflowclient.delete_workflow_definition(workflow_def_id) workflow_def_ids,_ = self.rrclient.find_resources(restype=RT.WorkflowDefinition) assertions(len(workflow_def_ids) == 0 ) @attr('LOCOINT') @unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False),'Not integrated for CEI') def test_mpl_graphs_transform_workflow(self): assertions = self.assertTrue # Build the workflow definition workflow_def_obj = IonObject(RT.WorkflowDefinition, name='Mpl_Graphs_Test_Workflow',description='Tests the workflow of converting stream data to Matplotlib graphs') #Add a transformation process definition mpl_graphs_procdef_id = self.create_mpl_graphs_data_process_definition() workflow_step_obj = IonObject('DataProcessWorkflowStep', data_process_definition_id=mpl_graphs_procdef_id, persist_process_output_data=False) workflow_def_obj.workflow_steps.append(workflow_step_obj) #Create it in the resource registry workflow_def_id = self.workflowclient.create_workflow_definition(workflow_def_obj) #The list of data product streams to monitor data_product_stream_ids = list() #Create the input data product ctd_stream_id, ctd_parsed_data_product_id = self.create_ctd_input_stream_and_data_product() data_product_stream_ids.append(ctd_stream_id) #Create and start the workflow workflow_id, workflow_product_id = self.workflowclient.create_data_process_workflow(workflow_def_id, ctd_parsed_data_product_id, persist_workflow_data_product=True, timeout=60) workflow_output_ids,_ = self.rrclient.find_subjects(RT.Workflow, PRED.hasOutputProduct, workflow_product_id, True) assertions(len(workflow_output_ids) == 1 ) #Walk the associations to find the appropriate output data streams to validate the messages workflow_dp_ids,_ = self.rrclient.find_objects(workflow_id, PRED.hasDataProduct, RT.DataProduct, True) assertions(len(workflow_dp_ids) == 1 ) for dp_id in workflow_dp_ids: stream_ids, _ = self.rrclient.find_objects(dp_id, PRED.hasStream, None, True) assertions(len(stream_ids) == 1 ) data_product_stream_ids.append(stream_ids[0]) #Start the output stream listener to monitor and collect messages results = self.start_output_stream_and_listen(ctd_stream_id, data_product_stream_ids) #Stop the workflow processes self.workflowclient.terminate_data_process_workflow(workflow_id=workflow_id,delete_data_products=False, timeout=60) # Should test true at some point #Validate the data from each of the messages along the way self.validate_mpl_graphs_transform_results(results) # Check to see if ingestion worked. Extract the granules from data_retrieval. # First find the dataset associated with the output dp product ds_ids,_ = self.rrclient.find_objects(workflow_dp_ids[len(workflow_dp_ids) - 1], PRED.hasDataset, RT.Dataset, True) retrieved_granule = self.data_retriever.retrieve_last_data_points(ds_ids[0], 10) #Validate the data from each of the messages along the way self.validate_mpl_graphs_transform_results(retrieved_granule) #Cleanup to make sure delete is correct. self.workflowclient.delete_workflow_definition(workflow_def_id) workflow_def_ids,_ = self.rrclient.find_resources(restype=RT.WorkflowDefinition) assertions(len(workflow_def_ids) == 0 ) @attr('LOCOINT') @unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False),'Not integrated for CEI') def test_multiple_workflow_instances(self): assertions = self.assertTrue # Build the workflow definition workflow_def_obj = IonObject(RT.WorkflowDefinition, name='Multiple_Test_Workflow',description='Tests the workflow of converting stream data') #Add a transformation process definition google_dt_procdef_id = self.create_google_dt_data_process_definition() workflow_step_obj = IonObject('DataProcessWorkflowStep', data_process_definition_id=google_dt_procdef_id, persist_process_output_data=False) workflow_def_obj.workflow_steps.append(workflow_step_obj) #Create it in the resource registry workflow_def_id = self.workflowclient.create_workflow_definition(workflow_def_obj) #The list of data product streams to monitor data_product_stream_ids = list() #Create the first input data product ctd_stream_id1, ctd_parsed_data_product_id1 = self.create_ctd_input_stream_and_data_product('ctd_parsed1') data_product_stream_ids.append(ctd_stream_id1) #Create and start the first workflow workflow_id1, workflow_product_id1 = self.workflowclient.create_data_process_workflow(workflow_def_id, ctd_parsed_data_product_id1, timeout=60) #Create the second input data product ctd_stream_id2, ctd_parsed_data_product_id2 = self.create_ctd_input_stream_and_data_product('ctd_parsed2') data_product_stream_ids.append(ctd_stream_id2) #Create and start the second workflow workflow_id2, workflow_product_id2 = self.workflowclient.create_data_process_workflow(workflow_def_id, ctd_parsed_data_product_id2, timeout=60) #Walk the associations to find the appropriate output data streams to validate the messages workflow_ids,_ = self.rrclient.find_resources(restype=RT.Workflow) assertions(len(workflow_ids) == 2 ) #Start the first input stream process ctd_sim_pid1 = self.start_sinusoidal_input_stream_process(ctd_stream_id1) #Start the second input stream process ctd_sim_pid2 = self.start_simple_input_stream_process(ctd_stream_id2) #Start the output stream listener to monitor a set number of messages being sent through the workflows results = self.start_output_stream_and_listen(None, data_product_stream_ids, message_count_per_stream=5) # stop the flow of messages... self.process_dispatcher.cancel_process(ctd_sim_pid1) # kill the ctd simulator process - that is enough data self.process_dispatcher.cancel_process(ctd_sim_pid2) #Stop the first workflow processes self.workflowclient.terminate_data_process_workflow(workflow_id=workflow_id1,delete_data_products=False, timeout=60) # Should test true at some point #Stop the second workflow processes self.workflowclient.terminate_data_process_workflow(workflow_id=workflow_id2,delete_data_products=False, timeout=60) # Should test true at some point workflow_ids,_ = self.rrclient.find_resources(restype=RT.Workflow) assertions(len(workflow_ids) == 0 ) #Cleanup to make sure delete is correct. self.workflowclient.delete_workflow_definition(workflow_def_id) workflow_def_ids,_ = self.rrclient.find_resources(restype=RT.WorkflowDefinition) assertions(len(workflow_def_ids) == 0 ) aid_list = self.rrclient.find_associations(workflow_def_id, PRED.hasDataProcessDefinition) assertions(len(aid_list) == 0 )
class TestVisualizationServiceIntegration(VisualizationIntegrationTestHelper): def setUp(self): # Start container logging.disable(logging.ERROR) self._start_container() self.container.start_rel_from_url('res/deploy/r2deploy.yml') # simulate preloading preload_ion_params(self.container) logging.disable(logging.NOTSET) #Instantiate a process to represent the test process=VisualizationServiceTestProcess() # Now create client to DataProductManagementService self.rrclient = ResourceRegistryServiceProcessClient(node=self.container.node, process=process) self.damsclient = DataAcquisitionManagementServiceProcessClient(node=self.container.node, process=process) self.pubsubclient = PubsubManagementServiceProcessClient(node=self.container.node, process=process) self.ingestclient = IngestionManagementServiceProcessClient(node=self.container.node, process=process) self.imsclient = InstrumentManagementServiceProcessClient(node=self.container.node, process=process) self.dataproductclient = DataProductManagementServiceProcessClient(node=self.container.node, process=process) self.dataprocessclient = DataProcessManagementServiceProcessClient(node=self.container.node, process=process) self.datasetclient = DatasetManagementServiceProcessClient(node=self.container.node, process=process) self.workflowclient = WorkflowManagementServiceProcessClient(node=self.container.node, process=process) self.process_dispatcher = ProcessDispatcherServiceProcessClient(node=self.container.node, process=process) self.data_retriever = DataRetrieverServiceProcessClient(node=self.container.node, process=process) self.vis_client = VisualizationServiceProcessClient(node=self.container.node, process=process) self.ctd_stream_def = SBE37_CDM_stream_definition() def validate_messages(self, msgs): msg = msgs rdt = RecordDictionaryTool.load_from_granule(msg.body) vardict = {} vardict['temp'] = get_safe(rdt, 'temp') vardict['time'] = get_safe(rdt, 'time') print vardict['time'] print vardict['temp'] @attr('LOCOINT') #@patch.dict('pyon.ion.exchange.CFG', {'container':{'exchange':{'auto_register': False}}}) @unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False),'Not integrated for CEI') def test_visualization_queue(self): #The list of data product streams to monitor data_product_stream_ids = list() #Create the input data product ctd_stream_id, ctd_parsed_data_product_id = self.create_ctd_input_stream_and_data_product() data_product_stream_ids.append(ctd_stream_id) user_queue_name = USER_VISUALIZATION_QUEUE xq = self.container.ex_manager.create_xn_queue(user_queue_name) salinity_subscription_id = self.pubsubclient.create_subscription( stream_ids=data_product_stream_ids, exchange_name = user_queue_name, name = "user visualization queue" ) subscriber = Subscriber(from_name=xq) subscriber.initialize() # after the queue has been created it is safe to activate the subscription self.pubsubclient.activate_subscription(subscription_id=salinity_subscription_id) #Start the output stream listener to monitor and collect messages #results = self.start_output_stream_and_listen(None, data_product_stream_ids) #Not sure why this is needed - but it is #subscriber._chan.stop_consume() ctd_sim_pid = self.start_simple_input_stream_process(ctd_stream_id) gevent.sleep(10.0) # Send some messages - don't care how many msg_count,_ = xq.get_stats() log.info('Messages in user queue 1: %s ' % msg_count) #Validate the data from each of the messages along the way #self.validate_messages(results) # for x in range(msg_count): # mo = subscriber.get_one_msg(timeout=1) # print mo.body # mo.ack() msgs = subscriber.get_all_msgs(timeout=2) for x in range(len(msgs)): msgs[x].ack() self.validate_messages(msgs[x]) # print msgs[x].body #Should be zero after pulling all of the messages. msg_count,_ = xq.get_stats() log.info('Messages in user queue 2: %s ' % msg_count) #Trying to continue to receive messages in the queue gevent.sleep(5.0) # Send some messages - don't care how many #Turning off after everything - since it is more representative of an always on stream of data! self.process_dispatcher.cancel_process(ctd_sim_pid) # kill the ctd simulator process - that is enough data #Should see more messages in the queue msg_count,_ = xq.get_stats() log.info('Messages in user queue 3: %s ' % msg_count) msgs = subscriber.get_all_msgs(timeout=2) for x in range(len(msgs)): msgs[x].ack() self.validate_messages(msgs[x]) #Should be zero after pulling all of the messages. msg_count,_ = xq.get_stats() log.info('Messages in user queue 4: %s ' % msg_count) subscriber.close() self.container.ex_manager.delete_xn(xq) @unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False),'Not integrated for CEI') def test_multiple_visualization_queue(self): # set up a workflow with the salinity transform and the doubler. We will direct the original stream and the doubled stream to queues # and test to make sure the subscription to the queues is working correctly assertions = self.assertTrue # Build the workflow definition workflow_def_obj = IonObject(RT.WorkflowDefinition, name='Viz_Test_Workflow',description='A workflow to test collection of multiple data products in queues') workflow_data_product_name = 'TEST-Workflow_Output_Product' #Set a specific output product name #------------------------------------------------------------------------------------------------------------------------- #Add a transformation process definition for salinity #------------------------------------------------------------------------------------------------------------------------- ctd_L2_salinity_dprocdef_id = self.create_salinity_data_process_definition() workflow_step_obj = IonObject('DataProcessWorkflowStep', data_process_definition_id=ctd_L2_salinity_dprocdef_id, persist_process_output_data=False) #Don't persist the intermediate data product configuration = {'stream_name' : 'salinity'} workflow_step_obj.configuration = configuration workflow_def_obj.workflow_steps.append(workflow_step_obj) #Create it in the resource registry workflow_def_id = self.workflowclient.create_workflow_definition(workflow_def_obj) aids = self.rrclient.find_associations(workflow_def_id, PRED.hasDataProcessDefinition) assertions(len(aids) == 1 ) #The list of data product streams to monitor data_product_stream_ids = list() #Create the input data product ctd_stream_id, ctd_parsed_data_product_id = self.create_ctd_input_stream_and_data_product() data_product_stream_ids.append(ctd_stream_id) #Create and start the workflow workflow_id, workflow_product_id = self.workflowclient.create_data_process_workflow(workflow_def_id, ctd_parsed_data_product_id, timeout=30) workflow_output_ids,_ = self.rrclient.find_subjects(RT.Workflow, PRED.hasOutputProduct, workflow_product_id, True) assertions(len(workflow_output_ids) == 1 ) #Walk the associations to find the appropriate output data streams to validate the messages workflow_dp_ids,_ = self.rrclient.find_objects(workflow_id, PRED.hasDataProduct, RT.DataProduct, True) assertions(len(workflow_dp_ids) == 1 ) for dp_id in workflow_dp_ids: stream_ids, _ = self.rrclient.find_objects(dp_id, PRED.hasStream, None, True) assertions(len(stream_ids) == 1 ) data_product_stream_ids.append(stream_ids[0]) # Now for each of the data_product_stream_ids create a queue and pipe their data to the queue user_queue_name1 = USER_VISUALIZATION_QUEUE + '1' user_queue_name2 = USER_VISUALIZATION_QUEUE + '2' # use idempotency to create queues xq1 = self.container.ex_manager.create_xn_queue(user_queue_name1) self.addCleanup(xq1.delete) xq2 = self.container.ex_manager.create_xn_queue(user_queue_name2) self.addCleanup(xq2.delete) xq1.purge() xq2.purge() # the create_subscription call takes a list of stream_ids so create temp ones dp_stream_id1 = list() dp_stream_id1.append(data_product_stream_ids[0]) dp_stream_id2 = list() dp_stream_id2.append(data_product_stream_ids[1]) salinity_subscription_id1 = self.pubsubclient.create_subscription( stream_ids=dp_stream_id1, exchange_name = user_queue_name1, name = "user visualization queue1") salinity_subscription_id2 = self.pubsubclient.create_subscription( stream_ids=dp_stream_id2, exchange_name = user_queue_name2, name = "user visualization queue2") # Create subscribers for the output of the queue subscriber1 = Subscriber(from_name=xq1) subscriber1.initialize() subscriber2 = Subscriber(from_name=xq2) subscriber2.initialize() # after the queue has been created it is safe to activate the subscription self.pubsubclient.activate_subscription(subscription_id=salinity_subscription_id1) self.pubsubclient.activate_subscription(subscription_id=salinity_subscription_id2) # Start input stream and wait for some time ctd_sim_pid = self.start_simple_input_stream_process(ctd_stream_id) gevent.sleep(5.0) # Send some messages - don't care how many msg_count,_ = xq1.get_stats() log.info('Messages in user queue 1: %s ' % msg_count) msg_count,_ = xq2.get_stats() log.info('Messages in user queue 2: %s ' % msg_count) msgs1 = subscriber1.get_all_msgs(timeout=2) msgs2 = subscriber2.get_all_msgs(timeout=2) for x in range(min(len(msgs1), len(msgs2))): msgs1[x].ack() msgs2[x].ack() self.validate_multiple_vis_queue_messages(msgs1[x].body, msgs2[x].body) # kill the ctd simulator process - that is enough data self.process_dispatcher.cancel_process(ctd_sim_pid) # close the subscription and queues subscriber1.close() subscriber2.close() return @patch.dict(CFG, {'user_queue_monitor_timeout': 5}) @attr('SMOKE') def test_realtime_visualization(self): # #Start up multiple vis service workers if not a CEI launch # if not os.getenv('CEI_LAUNCH_TEST', False): # vpid1 = self.container.spawn_process('visualization_service1','ion.services.ans.visualization_service','VisualizationService', CFG ) # self.addCleanup(self.container.terminate_process, vpid1) # vpid2 = self.container.spawn_process('visualization_service2','ion.services.ans.visualization_service','VisualizationService', CFG ) # self.addCleanup(self.container.terminate_process, vpid2) # Create the Highcharts workflow definition since there is no preload for the test workflow_def_id = self.create_highcharts_workflow_def() #Create the input data product ctd_stream_id, ctd_parsed_data_product_id = self.create_ctd_input_stream_and_data_product() ctd_sim_pid = self.start_sinusoidal_input_stream_process(ctd_stream_id) vis_params ={} vis_token_resp = self.vis_client.initiate_realtime_visualization_data(data_product_id=ctd_parsed_data_product_id, visualization_parameters=simplejson.dumps(vis_params)) vis_token = ast.literal_eval(vis_token_resp)["rt_query_token"] result = gevent.event.AsyncResult() def get_vis_messages(get_data_count=7): #SHould be an odd number for round robbin processing by service workers get_cnt = 0 while get_cnt < get_data_count: vis_data = self.vis_client.get_realtime_visualization_data(vis_token) if (vis_data): self.validate_highcharts_transform_results(vis_data) get_cnt += 1 gevent.sleep(5) # simulates the polling from UI result.set(get_cnt) gevent.spawn(get_vis_messages) result.get(timeout=90) #Trying to continue to receive messages in the queue gevent.sleep(2.0) # Send some messages - don't care how many # Cleanup self.vis_client.terminate_realtime_visualization_data(vis_token) #Turning off after everything - since it is more representative of an always on stream of data! self.process_dispatcher.cancel_process(ctd_sim_pid) # kill the ctd simulator process - that is enough data @patch.dict(CFG, {'user_queue_monitor_timeout': 5}) @patch.dict(CFG, {'user_queue_monitor_size': 25}) @attr('CLEANUP') @unittest.skipIf(os.getenv('PYCC_MODE', False),'Not integrated for CEI') def test_realtime_visualization_cleanup(self): # #Start up multiple vis service workers if not a CEI launch # if not os.getenv('CEI_LAUNCH_TEST', False): # vpid1 = self.container.spawn_process('visualization_service1','ion.services.ans.visualization_service','VisualizationService', CFG ) # self.addCleanup(self.container.terminate_process, vpid1) # vpid2 = self.container.spawn_process('visualization_service2','ion.services.ans.visualization_service','VisualizationService', CFG ) # self.addCleanup(self.container.terminate_process, vpid2) #get the list of queues and message counts on the broker for the user vis queues try: queues = self.container.ex_manager.list_queues(name=USER_VISUALIZATION_QUEUE, return_columns=['name', 'messages']) q_names = [ q['name'] for q in queues if q['name']] #Get a list of only the queue names original_queue_count = len(q_names) except Exception, e: log.warn('Unable to get queue information from broker management plugin: ' + e.message) pass # Create the highcharts workflow definition since there is no preload for the test workflow_def_id = self.create_highcharts_workflow_def() #Create the input data product ctd_stream_id, ctd_parsed_data_product_id = self.create_ctd_input_stream_and_data_product() ctd_sim_pid = self.start_sinusoidal_input_stream_process(ctd_stream_id) #Start up a number of requests - and queues - to start accumulating messages. THe test will not clean them up #but instead check to see if the monitoring thread will. vis_token_resp = self.vis_client.initiate_realtime_visualization_data(data_product_id=ctd_parsed_data_product_id) bad_vis_token1 = ast.literal_eval(vis_token_resp)["rt_query_token"] vis_token_resp = self.vis_client.initiate_realtime_visualization_data(data_product_id=ctd_parsed_data_product_id) bad_vis_token2 = ast.literal_eval(vis_token_resp)["rt_query_token"] vis_token_resp = self.vis_client.initiate_realtime_visualization_data(data_product_id=ctd_parsed_data_product_id) bad_vis_token3 = ast.literal_eval(vis_token_resp)["rt_query_token"] vis_token_resp = self.vis_client.initiate_realtime_visualization_data(data_product_id=ctd_parsed_data_product_id) vis_token = ast.literal_eval(vis_token_resp)["rt_query_token"] #Get the default exchange space exchange = self.container.ex_manager.default_xs.exchange #get the list of queues and message counts on the broker for the user vis queues try: queues = self.container.ex_manager.list_queues(name=USER_VISUALIZATION_QUEUE, return_columns=['name', 'messages']) q_names = [ q['name'] for q in queues if q['name']] #Get a list of only the queue names self.assertIn(exchange + "." + bad_vis_token1, q_names) self.assertIn(exchange + "." + bad_vis_token2, q_names) self.assertIn(exchange + "." + bad_vis_token3, q_names) self.assertIn(exchange + "." + vis_token, q_names) except Exception, e: log.warn('Unable to get queue information from broker management plugin: ' + e.message) pass
class BootstrapProcessDispatcher(BootstrapPlugin): """ Bootstrap process for process dispatcher. """ def on_initial_bootstrap(self, process, config, **kwargs): self.pds_client = ProcessDispatcherServiceProcessClient( process=process) self.resource_registry = ResourceRegistryServiceProcessClient( process=process) self.ingestion_worker(process, config) self.replay_defs(process, config) self.notification_worker(process, config) self.registration_worker(process, config) self.pydap_server(process, config) self.eoi_services(process, config) def eoi_services(self, process, config): eoi_module = config.get_safe( 'bootstrap.processes.registration.module', 'ion.processes.data.registration.eoi_registration_process') eoi_class = config.get_safe('bootstrap.processes.registration.class', 'EOIRegistrationProcess') process_definition = ProcessDefinition( name='eoi_server', description='Process for eoi data sources') process_definition.executable['module'] = eoi_module process_definition.executable['class'] = eoi_class self._create_and_launch(process_definition) def pydap_server(self, process, config): pydap_module = config.get_safe( 'bootstrap.processes.pydap.module', 'ion.processes.data.externalization.lightweight_pydap') pydap_class = config.get_safe('bootstrap.processes.pydap.class', 'LightweightPyDAP') use_pydap = config.get_safe('bootstrap.launch_pydap', False) process_definition = ProcessDefinition( name='pydap_server', description='Lightweight WSGI Server for PyDAP') process_definition.executable['module'] = pydap_module process_definition.executable['class'] = pydap_class self._create_and_launch(process_definition, use_pydap) def registration_worker(self, process, config): res, meta = self.resource_registry.find_resources( name='registration_worker', restype=RT.ProcessDefinition) if len(res): return registration_module = config.get_safe( 'bootstrap.processes.registration.module', 'ion.processes.data.registration.registration_process') registration_class = config.get_safe( 'bootstrap.processes.registration.class', 'RegistrationProcess') use_pydap = True process_definition = ProcessDefinition( name='registration_worker', description='For registering datasets with ERDDAP') process_definition.executable['module'] = registration_module process_definition.executable['class'] = registration_class self._create_and_launch(process_definition, use_pydap) def _create_and_launch(self, process_definition, conditional=True): proc_def_id = self.pds_client.create_process_definition( process_definition=process_definition) if conditional: process_res_id = self.pds_client.create_process( process_definition_id=proc_def_id) self.pds_client.schedule_process(process_definition_id=proc_def_id, process_id=process_res_id) def ingestion_worker(self, process, config): # ingestion ingestion_module = config.get_safe( 'bootstrap.processes.ingestion.module', 'ion.processes.data.ingestion.science_granule_ingestion_worker') ingestion_class = config.get_safe( 'bootstrap.processes.ingestion.class', 'ScienceGranuleIngestionWorker') ingestion_datastore = config.get_safe( 'bootstrap.processes.ingestion.datastore_name', 'datasets') ingestion_queue = config.get_safe( 'bootstrap.processes.ingestion.queue', 'science_granule_ingestion') ingestion_workers = config.get_safe( 'bootstrap.processes.ingestion.workers', 1) #-------------------------------------------------------------------------------- # Create ingestion workers #-------------------------------------------------------------------------------- process_definition = ProcessDefinition( name='ingestion_worker_process', description='Worker transform process for ingestion of datasets') process_definition.executable['module'] = ingestion_module process_definition.executable['class'] = ingestion_class ingestion_procdef_id = self.pds_client.create_process_definition( process_definition=process_definition) #-------------------------------------------------------------------------------- # Simulate a HA ingestion worker by creating two of them #-------------------------------------------------------------------------------- # config = DotDict() # config.process.datastore_name = ingestion_datastore # config.process.queue_name = ingestion_queue # # for i in xrange(ingestion_workers): # self.pds_client.schedule_process(process_definition_id=ingestion_procdef_id, configuration=config) def notification_worker(self, process, config): # user notifications notification_module = config.get_safe( 'bootstrap.processes.user_notification.module', 'ion.processes.data.transforms.notification_worker') notification_class = config.get_safe( 'bootstrap.processes.user_notification.class', 'NotificationWorker') notification_workers = config.get_safe( 'bootstrap.processes.user_notification.workers', 1) #-------------------------------------------------------------------------------- # Create notification workers #-------------------------------------------------------------------------------- # set up the process definition process_definition_uns = ProcessDefinition( name='notification_worker_process', description='Worker transform process for user notifications') process_definition_uns.executable['module'] = notification_module process_definition_uns.executable['class'] = notification_class uns_procdef_id = self.pds_client.create_process_definition( process_definition=process_definition_uns) config = DotDict() config.process.type = 'simple' for i in xrange(notification_workers): config.process.name = 'notification_worker_%s' % i config.process.queue_name = 'notification_worker_queue' self.pds_client.schedule_process( process_definition_id=uns_procdef_id, configuration=config) def replay_defs(self, process, config): replay_module = config.get_safe( 'bootstrap.processes.replay.module', 'ion.processes.data.replay.replay_process') replay_class = config.get_safe('bootstrap.processes.replay.class', 'ReplayProcess') #-------------------------------------------------------------------------------- # Create replay process definition #-------------------------------------------------------------------------------- process_definition = ProcessDefinition( name=DataRetrieverService.REPLAY_PROCESS, description='Process for the replay of datasets') process_definition.executable['module'] = replay_module process_definition.executable['class'] = replay_class self.pds_client.create_process_definition( process_definition=process_definition) def on_restart(self, process, config, **kwargs): pass
class TestVisualizationServiceIntegration(VisualizationIntegrationTestHelper): def setUp(self): # Start container logging.disable(logging.ERROR) self._start_container() self.container.start_rel_from_url('res/deploy/r2deploy.yml') # simulate preloading #preload_ion_params(self.container) logging.disable(logging.NOTSET) #Instantiate a process to represent the test process = VisualizationServiceTestProcess() # Now create client to DataProductManagementService self.rrclient = ResourceRegistryServiceProcessClient( node=self.container.node, process=process) self.damsclient = DataAcquisitionManagementServiceProcessClient( node=self.container.node, process=process) self.pubsubclient = PubsubManagementServiceProcessClient( node=self.container.node, process=process) self.ingestclient = IngestionManagementServiceProcessClient( node=self.container.node, process=process) self.imsclient = InstrumentManagementServiceProcessClient( node=self.container.node, process=process) self.dataproductclient = DataProductManagementServiceProcessClient( node=self.container.node, process=process) self.dataprocessclient = DataProcessManagementServiceProcessClient( node=self.container.node, process=process) self.datasetclient = DatasetManagementServiceProcessClient( node=self.container.node, process=process) self.process_dispatcher = ProcessDispatcherServiceProcessClient( node=self.container.node, process=process) self.data_retriever = DataRetrieverServiceProcessClient( node=self.container.node, process=process) self.vis_client = VisualizationServiceProcessClient( node=self.container.node, process=process) self.ctd_stream_def = SBE37_CDM_stream_definition() def validate_messages(self, msgs): msg = msgs rdt = RecordDictionaryTool.load_from_granule(msg.body) vardict = {} vardict['temp'] = get_safe(rdt, 'temp') vardict['time'] = get_safe(rdt, 'time') print vardict['time'] print vardict['temp'] @attr('LOCOINT') #@patch.dict('pyon.ion.exchange.CFG', {'container':{'exchange':{'auto_register': False}}}) @unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False), 'Not integrated for CEI') def test_visualization_queue(self): #The list of data product streams to monitor data_product_stream_ids = list() #Create the input data product ctd_stream_id, ctd_parsed_data_product_id = self.create_ctd_input_stream_and_data_product( ) data_product_stream_ids.append(ctd_stream_id) user_queue_name = USER_VISUALIZATION_QUEUE xq = self.container.ex_manager.create_xn_queue(user_queue_name) salinity_subscription_id = self.pubsubclient.create_subscription( stream_ids=data_product_stream_ids, exchange_name=user_queue_name, name="user visualization queue") subscriber = Subscriber(from_name=xq) subscriber.initialize() # after the queue has been created it is safe to activate the subscription self.pubsubclient.activate_subscription( subscription_id=salinity_subscription_id) #Start the output stream listener to monitor and collect messages #results = self.start_output_stream_and_listen(None, data_product_stream_ids) #Not sure why this is needed - but it is #subscriber._chan.stop_consume() ctd_sim_pid = self.start_simple_input_stream_process(ctd_stream_id) gevent.sleep(10.0) # Send some messages - don't care how many msg_count, _ = xq.get_stats() log.info('Messages in user queue 1: %s ' % msg_count) #Validate the data from each of the messages along the way #self.validate_messages(results) # for x in range(msg_count): # mo = subscriber.get_one_msg(timeout=1) # print mo.body # mo.ack() msgs = subscriber.get_all_msgs(timeout=2) for x in range(len(msgs)): msgs[x].ack() self.validate_messages(msgs[x]) # print msgs[x].body #Should be zero after pulling all of the messages. msg_count, _ = xq.get_stats() log.info('Messages in user queue 2: %s ' % msg_count) #Trying to continue to receive messages in the queue gevent.sleep(5.0) # Send some messages - don't care how many #Turning off after everything - since it is more representative of an always on stream of data! self.process_dispatcher.cancel_process( ctd_sim_pid ) # kill the ctd simulator process - that is enough data #Should see more messages in the queue msg_count, _ = xq.get_stats() log.info('Messages in user queue 3: %s ' % msg_count) msgs = subscriber.get_all_msgs(timeout=2) for x in range(len(msgs)): msgs[x].ack() self.validate_messages(msgs[x]) #Should be zero after pulling all of the messages. msg_count, _ = xq.get_stats() log.info('Messages in user queue 4: %s ' % msg_count) subscriber.close() self.container.ex_manager.delete_xn(xq) @patch.dict(CFG, {'user_queue_monitor_timeout': 5}) def test_realtime_visualization(self): # #Start up multiple vis service workers if not a CEI launch # if not os.getenv('CEI_LAUNCH_TEST', False): # vpid1 = self.container.spawn_process('visualization_service1','ion.services.ans.visualization_service','VisualizationService', CFG ) # self.addCleanup(self.container.terminate_process, vpid1) # vpid2 = self.container.spawn_process('visualization_service2','ion.services.ans.visualization_service','VisualizationService', CFG ) # self.addCleanup(self.container.terminate_process, vpid2) ## Create the Highcharts workflow definition since there is no preload for the test #workflow_def_id = self.create_highcharts_workflow_def() #Create the input data product ctd_stream_id, ctd_parsed_data_product_id = self.create_ctd_input_stream_and_data_product( ) ctd_sim_pid = self.start_sinusoidal_input_stream_process(ctd_stream_id) vis_params = {} vis_token_resp = self.vis_client.initiate_realtime_visualization_data( data_product_id=ctd_parsed_data_product_id, visualization_parameters=simplejson.dumps(vis_params)) vis_token = ast.literal_eval(vis_token_resp)["rt_query_token"] result = gevent.event.AsyncResult() def get_vis_messages( get_data_count=7 ): #SHould be an odd number for round robbin processing by service workers get_cnt = 0 while get_cnt < get_data_count: vis_data = self.vis_client.get_realtime_visualization_data( vis_token) if (vis_data): self.validate_highcharts_transform_results(vis_data) get_cnt += 1 gevent.sleep(5) # simulates the polling from UI result.set(get_cnt) gevent.spawn(get_vis_messages) result.get(timeout=90) #Trying to continue to receive messages in the queue gevent.sleep(2.0) # Send some messages - don't care how many # Cleanup self.vis_client.terminate_realtime_visualization_data(vis_token) #Turning off after everything - since it is more representative of an always on stream of data! self.process_dispatcher.cancel_process( ctd_sim_pid ) # kill the ctd simulator process - that is enough data @patch.dict(CFG, {'user_queue_monitor_timeout': 5}) @patch.dict(CFG, {'user_queue_monitor_size': 25}) @attr('CLEANUP') @unittest.skipIf(os.getenv('PYCC_MODE', False), 'Not integrated for CEI') def test_realtime_visualization_cleanup(self): # #Start up multiple vis service workers if not a CEI launch # if not os.getenv('CEI_LAUNCH_TEST', False): # vpid1 = self.container.spawn_process('visualization_service1','ion.services.ans.visualization_service','VisualizationService', CFG ) # self.addCleanup(self.container.terminate_process, vpid1) # vpid2 = self.container.spawn_process('visualization_service2','ion.services.ans.visualization_service','VisualizationService', CFG ) # self.addCleanup(self.container.terminate_process, vpid2) #get the list of queues and message counts on the broker for the user vis queues try: queues = self.container.ex_manager.list_queues( name=USER_VISUALIZATION_QUEUE, return_columns=['name', 'messages']) q_names = [q['name'] for q in queues if q['name']] #Get a list of only the queue names original_queue_count = len(q_names) except Exception, e: log.warn( 'Unable to get queue information from broker management plugin: ' + e.message) pass ## Create the highcharts workflow definition since there is no preload for the test #workflow_def_id = self.create_highcharts_workflow_def() #Create the input data product ctd_stream_id, ctd_parsed_data_product_id = self.create_ctd_input_stream_and_data_product( ) ctd_sim_pid = self.start_sinusoidal_input_stream_process(ctd_stream_id) #Start up a number of requests - and queues - to start accumulating messages. THe test will not clean them up #but instead check to see if the monitoring thread will. vis_token_resp = self.vis_client.initiate_realtime_visualization_data( data_product_id=ctd_parsed_data_product_id) bad_vis_token1 = ast.literal_eval(vis_token_resp)["rt_query_token"] vis_token_resp = self.vis_client.initiate_realtime_visualization_data( data_product_id=ctd_parsed_data_product_id) bad_vis_token2 = ast.literal_eval(vis_token_resp)["rt_query_token"] vis_token_resp = self.vis_client.initiate_realtime_visualization_data( data_product_id=ctd_parsed_data_product_id) bad_vis_token3 = ast.literal_eval(vis_token_resp)["rt_query_token"] vis_token_resp = self.vis_client.initiate_realtime_visualization_data( data_product_id=ctd_parsed_data_product_id) vis_token = ast.literal_eval(vis_token_resp)["rt_query_token"] #Get the default exchange space exchange = self.container.ex_manager.default_xs.exchange #get the list of queues and message counts on the broker for the user vis queues try: queues = self.container.ex_manager.list_queues( name=USER_VISUALIZATION_QUEUE, return_columns=['name', 'messages']) q_names = [q['name'] for q in queues if q['name']] #Get a list of only the queue names self.assertIn(exchange + "." + bad_vis_token1, q_names) self.assertIn(exchange + "." + bad_vis_token2, q_names) self.assertIn(exchange + "." + bad_vis_token3, q_names) self.assertIn(exchange + "." + vis_token, q_names) except Exception, e: log.warn( 'Unable to get queue information from broker management plugin: ' + e.message) pass
class BootstrapProcessDispatcher(BootstrapPlugin): """ Bootstrap process for process dispatcher. """ def on_initial_bootstrap(self, process, config, **kwargs): self.pds_client = ProcessDispatcherServiceProcessClient(process=process) self.resource_registry = ResourceRegistryServiceProcessClient(process=process) self.ingestion_worker(process,config) self.replay_defs(process,config) self.notification_worker(process,config) self.registration_worker(process,config) self.pydap_server(process,config) def pydap_server(self, process, config): pydap_module = config.get_safe('bootstrap.processes.pydap.module', 'ion.processes.data.externalization.lightweight_pydap') pydap_class = config.get_safe('bootstrap.processes.pydap.class', 'LightweightPyDAP') use_pydap = config.get_safe('bootstrap.launch_pydap', False) process_definition = ProcessDefinition( name = 'pydap_server', description = 'Lightweight WSGI Server for PyDAP') process_definition.executable['module'] = pydap_module process_definition.executable['class'] = pydap_class self._create_and_launch(process_definition,use_pydap) def registration_worker(self, process, config): res, meta = self.resource_registry.find_resources(name='registration_worker', restype=RT.ProcessDefinition) if len(res): return registration_module = config.get_safe('bootstrap.processes.registration.module', 'ion.processes.data.registration.registration_process') registration_class = config.get_safe('bootstrap.processes.registration.class', 'RegistrationProcess') use_pydap = config.get_safe('bootstrap.use_pydap', False) process_definition = ProcessDefinition( name='registration_worker', description='For registering datasets with ERDDAP') process_definition.executable['module'] = registration_module process_definition.executable['class'] = registration_class self._create_and_launch(process_definition, use_pydap) def _create_and_launch(self, process_definition, conditional=True): proc_def_id = self.pds_client.create_process_definition(process_definition=process_definition) if conditional: process_res_id = self.pds_client.create_process(process_definition_id=proc_def_id) self.pds_client.schedule_process(process_definition_id=proc_def_id, process_id=process_res_id) def ingestion_worker(self, process, config): # ingestion ingestion_module = config.get_safe('bootstrap.processes.ingestion.module','ion.processes.data.ingestion.science_granule_ingestion_worker') ingestion_class = config.get_safe('bootstrap.processes.ingestion.class' ,'ScienceGranuleIngestionWorker') ingestion_datastore = config.get_safe('bootstrap.processes.ingestion.datastore_name', 'datasets') ingestion_queue = config.get_safe('bootstrap.processes.ingestion.queue' , 'science_granule_ingestion') ingestion_workers = config.get_safe('bootstrap.processes.ingestion.workers', 1) #-------------------------------------------------------------------------------- # Create ingestion workers #-------------------------------------------------------------------------------- process_definition = ProcessDefinition( name='ingestion_worker_process', description='Worker transform process for ingestion of datasets') process_definition.executable['module']= ingestion_module process_definition.executable['class'] = ingestion_class ingestion_procdef_id = self.pds_client.create_process_definition(process_definition=process_definition) #-------------------------------------------------------------------------------- # Simulate a HA ingestion worker by creating two of them #-------------------------------------------------------------------------------- # config = DotDict() # config.process.datastore_name = ingestion_datastore # config.process.queue_name = ingestion_queue # # for i in xrange(ingestion_workers): # self.pds_client.schedule_process(process_definition_id=ingestion_procdef_id, configuration=config) def notification_worker(self, process, config): # user notifications notification_module = config.get_safe('bootstrap.processes.user_notification.module','ion.processes.data.transforms.notification_worker') notification_class = config.get_safe('bootstrap.processes.user_notification.class' ,'NotificationWorker') notification_workers = config.get_safe('bootstrap.processes.user_notification.workers', 1) #-------------------------------------------------------------------------------- # Create notification workers #-------------------------------------------------------------------------------- # set up the process definition process_definition_uns = ProcessDefinition( name='notification_worker_process', description='Worker transform process for user notifications') process_definition_uns.executable['module']= notification_module process_definition_uns.executable['class'] = notification_class uns_procdef_id = self.pds_client.create_process_definition(process_definition=process_definition_uns) config = DotDict() config.process.type = 'simple' for i in xrange(notification_workers): config.process.name = 'notification_worker_%s' % i config.process.queue_name = 'notification_worker_queue' self.pds_client.schedule_process(process_definition_id=uns_procdef_id, configuration=config) def replay_defs(self, process, config): replay_module = config.get_safe('bootstrap.processes.replay.module', 'ion.processes.data.replay.replay_process') replay_class = config.get_safe('bootstrap.processes.replay.class' , 'ReplayProcess') #-------------------------------------------------------------------------------- # Create replay process definition #-------------------------------------------------------------------------------- process_definition = ProcessDefinition(name=DataRetrieverService.REPLAY_PROCESS, description='Process for the replay of datasets') process_definition.executable['module']= replay_module process_definition.executable['class'] = replay_class self.pds_client.create_process_definition(process_definition=process_definition) def on_restart(self, process, config, **kwargs): pass
class TestVisualizationServiceIntegration(VisualizationIntegrationTestHelper): def setUp(self): # Start container logging.disable(logging.ERROR) self._start_container() self.container.start_rel_from_url('res/deploy/r2deploy.yml') logging.disable(logging.NOTSET) #Instantiate a process to represent the test process = VisualizationServiceTestProcess() # Now create client to DataProductManagementService self.rrclient = ResourceRegistryServiceProcessClient( node=self.container.node, process=process) self.damsclient = DataAcquisitionManagementServiceProcessClient( node=self.container.node, process=process) self.pubsubclient = PubsubManagementServiceProcessClient( node=self.container.node, process=process) self.ingestclient = IngestionManagementServiceProcessClient( node=self.container.node, process=process) self.imsclient = InstrumentManagementServiceProcessClient( node=self.container.node, process=process) self.dataproductclient = DataProductManagementServiceProcessClient( node=self.container.node, process=process) self.dataprocessclient = DataProcessManagementServiceProcessClient( node=self.container.node, process=process) self.datasetclient = DatasetManagementServiceProcessClient( node=self.container.node, process=process) self.workflowclient = WorkflowManagementServiceProcessClient( node=self.container.node, process=process) self.process_dispatcher = ProcessDispatcherServiceProcessClient( node=self.container.node, process=process) self.data_retriever = DataRetrieverServiceProcessClient( node=self.container.node, process=process) self.vis_client = VisualizationServiceProcessClient( node=self.container.node, process=process) self.ctd_stream_def = SBE37_CDM_stream_definition() def validate_messages(self, msgs): msg = msgs rdt = RecordDictionaryTool.load_from_granule(msg.body) vardict = {} vardict['temp'] = get_safe(rdt, 'temp') vardict['time'] = get_safe(rdt, 'time') print vardict['time'] print vardict['temp'] @attr('LOCOINT') #@patch.dict('pyon.ion.exchange.CFG', {'container':{'exchange':{'auto_register': False}}}) @unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False), 'Not integrated for CEI') def test_visualization_queue(self): #The list of data product streams to monitor data_product_stream_ids = list() #Create the input data product ctd_stream_id, ctd_parsed_data_product_id = self.create_ctd_input_stream_and_data_product( ) data_product_stream_ids.append(ctd_stream_id) user_queue_name = 'user_queue' xq = self.container.ex_manager.create_xn_queue(user_queue_name) salinity_subscription_id = self.pubsubclient.create_subscription( stream_ids=data_product_stream_ids, exchange_name=user_queue_name, name="user visualization queue") subscriber = Subscriber(from_name=xq) subscriber.initialize() # after the queue has been created it is safe to activate the subscription self.pubsubclient.activate_subscription( subscription_id=salinity_subscription_id) #Start the output stream listener to monitor and collect messages #results = self.start_output_stream_and_listen(None, data_product_stream_ids) #Not sure why this is needed - but it is #subscriber._chan.stop_consume() ctd_sim_pid = self.start_simple_input_stream_process(ctd_stream_id) gevent.sleep(10.0) # Send some messages - don't care how many msg_count, _ = xq.get_stats() log.info('Messages in user queue 1: %s ' % msg_count) #Validate the data from each of the messages along the way #self.validate_messages(results) # for x in range(msg_count): # mo = subscriber.get_one_msg(timeout=1) # print mo.body # mo.ack() msgs = subscriber.get_all_msgs(timeout=2) for x in range(len(msgs)): msgs[x].ack() self.validate_messages(msgs[x]) # print msgs[x].body #Should be zero after pulling all of the messages. msg_count, _ = xq.get_stats() log.info('Messages in user queue 2: %s ' % msg_count) #Trying to continue to receive messages in the queue gevent.sleep(5.0) # Send some messages - don't care how many #Turning off after everything - since it is more representative of an always on stream of data! self.process_dispatcher.cancel_process( ctd_sim_pid ) # kill the ctd simulator process - that is enough data #Should see more messages in the queue msg_count, _ = xq.get_stats() log.info('Messages in user queue 3: %s ' % msg_count) msgs = subscriber.get_all_msgs(timeout=2) for x in range(len(msgs)): msgs[x].ack() self.validate_messages(msgs[x]) #Should be zero after pulling all of the messages. msg_count, _ = xq.get_stats() log.info('Messages in user queue 4: %s ' % msg_count) subscriber.close() self.container.ex_manager.delete_xn(xq) @unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False), 'Not integrated for CEI') def test_multiple_visualization_queue(self): # set up a workflow with the salinity transform and the doubler. We will direct the original stream and the doubled stream to queues # and test to make sure the subscription to the queues is working correctly assertions = self.assertTrue # Build the workflow definition workflow_def_obj = IonObject( RT.WorkflowDefinition, name='Viz_Test_Workflow', description= 'A workflow to test collection of multiple data products in queues' ) workflow_data_product_name = 'TEST-Workflow_Output_Product' #Set a specific output product name #------------------------------------------------------------------------------------------------------------------------- #Add a transformation process definition for salinity #------------------------------------------------------------------------------------------------------------------------- ctd_L2_salinity_dprocdef_id = self.create_salinity_data_process_definition( ) workflow_step_obj = IonObject( 'DataProcessWorkflowStep', data_process_definition_id=ctd_L2_salinity_dprocdef_id, persist_process_output_data=False ) #Don't persist the intermediate data product configuration = {'stream_name': 'salinity'} workflow_step_obj.configuration = configuration workflow_def_obj.workflow_steps.append(workflow_step_obj) #Create it in the resource registry workflow_def_id = self.workflowclient.create_workflow_definition( workflow_def_obj) aids = self.rrclient.find_associations(workflow_def_id, PRED.hasDataProcessDefinition) assertions(len(aids) == 1) #The list of data product streams to monitor data_product_stream_ids = list() #Create the input data product ctd_stream_id, ctd_parsed_data_product_id = self.create_ctd_input_stream_and_data_product( ) data_product_stream_ids.append(ctd_stream_id) #Create and start the workflow workflow_id, workflow_product_id = self.workflowclient.create_data_process_workflow( workflow_def_id, ctd_parsed_data_product_id, timeout=30) workflow_output_ids, _ = self.rrclient.find_subjects( RT.Workflow, PRED.hasOutputProduct, workflow_product_id, True) assertions(len(workflow_output_ids) == 1) #Walk the associations to find the appropriate output data streams to validate the messages workflow_dp_ids, _ = self.rrclient.find_objects( workflow_id, PRED.hasDataProduct, RT.DataProduct, True) assertions(len(workflow_dp_ids) == 1) for dp_id in workflow_dp_ids: stream_ids, _ = self.rrclient.find_objects(dp_id, PRED.hasStream, None, True) assertions(len(stream_ids) == 1) data_product_stream_ids.append(stream_ids[0]) # Now for each of the data_product_stream_ids create a queue and pipe their data to the queue user_queue_name1 = 'user_queue_1' user_queue_name2 = 'user_queue_2' # use idempotency to create queues xq1 = self.container.ex_manager.create_xn_queue(user_queue_name1) self.addCleanup(xq1.delete) xq2 = self.container.ex_manager.create_xn_queue(user_queue_name2) self.addCleanup(xq2.delete) xq1.purge() xq2.purge() # the create_subscription call takes a list of stream_ids so create temp ones dp_stream_id1 = list() dp_stream_id1.append(data_product_stream_ids[0]) dp_stream_id2 = list() dp_stream_id2.append(data_product_stream_ids[1]) salinity_subscription_id1 = self.pubsubclient.create_subscription( stream_ids=dp_stream_id1, exchange_name=user_queue_name1, name="user visualization queue1") salinity_subscription_id2 = self.pubsubclient.create_subscription( stream_ids=dp_stream_id2, exchange_name=user_queue_name2, name="user visualization queue2") # Create subscribers for the output of the queue subscriber1 = Subscriber(from_name=xq1) subscriber1.initialize() subscriber2 = Subscriber(from_name=xq2) subscriber2.initialize() # after the queue has been created it is safe to activate the subscription self.pubsubclient.activate_subscription( subscription_id=salinity_subscription_id1) self.pubsubclient.activate_subscription( subscription_id=salinity_subscription_id2) # Start input stream and wait for some time ctd_sim_pid = self.start_simple_input_stream_process(ctd_stream_id) gevent.sleep(5.0) # Send some messages - don't care how many msg_count, _ = xq1.get_stats() log.info('Messages in user queue 1: %s ' % msg_count) msg_count, _ = xq2.get_stats() log.info('Messages in user queue 2: %s ' % msg_count) msgs1 = subscriber1.get_all_msgs(timeout=2) msgs2 = subscriber2.get_all_msgs(timeout=2) for x in range(min(len(msgs1), len(msgs2))): msgs1[x].ack() msgs2[x].ack() self.validate_multiple_vis_queue_messages(msgs1[x].body, msgs2[x].body) # kill the ctd simulator process - that is enough data self.process_dispatcher.cancel_process(ctd_sim_pid) # close the subscription and queues subscriber1.close() subscriber2.close() return #@unittest.skip('Skipped because of broken record dictionary work-around') def test_realtime_visualization(self): #Create the input data product ctd_stream_id, ctd_parsed_data_product_id = self.create_ctd_input_stream_and_data_product( ) ctd_sim_pid = self.start_sinusoidal_input_stream_process(ctd_stream_id) #TODO - Need to add workflow creation for google data table vis_params = {} vis_token = self.vis_client.initiate_realtime_visualization( data_product_id=ctd_parsed_data_product_id, visualization_parameters=vis_params) #Trying to continue to receive messages in the queue gevent.sleep(10.0) # Send some messages - don't care how many #TODO - find out what the actual return data type should be vis_data = self.vis_client.get_realtime_visualization_data(vis_token) if (vis_data): self.validate_google_dt_transform_results(vis_data) #Trying to continue to receive messages in the queue gevent.sleep(5.0) # Send some messages - don't care how many #Turning off after everything - since it is more representative of an always on stream of data! #todo remove the try except try: self.process_dispatcher.cancel_process( ctd_sim_pid ) # kill the ctd simulator process - that is enough data except: log.warning("cancelling process did not work") vis_data = self.vis_client.get_realtime_visualization_data(vis_token) if vis_data: self.validate_google_dt_transform_results(vis_data) # Cleanup self.vis_client.terminate_realtime_visualization_data(vis_token) #@unittest.skip('Skipped because of broken record dictionary work-around') def test_google_dt_overview_visualization(self): #Create the input data product ctd_stream_id, ctd_parsed_data_product_id = self.create_ctd_input_stream_and_data_product( ) # start producing data ctd_sim_pid = self.start_sinusoidal_input_stream_process(ctd_stream_id) # Generate some data for a few seconds gevent.sleep(5.0) #Turning off after everything - since it is more representative of an always on stream of data! self.process_dispatcher.cancel_process( ctd_sim_pid ) # kill the ctd simulator process - that is enough data # Use the data product to test the data retrieval and google dt generation capability of the vis service vis_data = self.vis_client.get_visualization_data( ctd_parsed_data_product_id) # validate the returned data self.validate_vis_service_google_dt_results(vis_data) #@unittest.skip('Skipped because of broken record dictionary work-around') def test_mpl_graphs_overview_visualization(self): #Create the input data product ctd_stream_id, ctd_parsed_data_product_id = self.create_ctd_input_stream_and_data_product( ) ctd_sim_pid = self.start_sinusoidal_input_stream_process(ctd_stream_id) # Generate some data for a few seconds gevent.sleep(5.0) #Turning off after everything - since it is more representative of an always on stream of data! self.process_dispatcher.cancel_process( ctd_sim_pid ) # kill the ctd simulator process - that is enough data # Use the data product to test the data retrieval and google dt generation capability of the vis service vis_data = self.vis_client.get_visualization_image( ctd_parsed_data_product_id) # validate the returned data self.validate_vis_service_mpl_graphs_results(vis_data) return
class TestVisualizationServiceIntegration(VisualizationIntegrationTestHelper): def setUp(self): # Start container logging.disable(logging.ERROR) self._start_container() self.container.start_rel_from_url('res/deploy/r2deploy.yml') logging.disable(logging.NOTSET) #Instantiate a process to represent the test process=VisualizationServiceTestProcess() # Now create client to DataProductManagementService self.rrclient = ResourceRegistryServiceProcessClient(node=self.container.node, process=process) self.damsclient = DataAcquisitionManagementServiceProcessClient(node=self.container.node, process=process) self.pubsubclient = PubsubManagementServiceProcessClient(node=self.container.node, process=process) self.ingestclient = IngestionManagementServiceProcessClient(node=self.container.node, process=process) self.imsclient = InstrumentManagementServiceProcessClient(node=self.container.node, process=process) self.dataproductclient = DataProductManagementServiceProcessClient(node=self.container.node, process=process) self.dataprocessclient = DataProcessManagementServiceProcessClient(node=self.container.node, process=process) self.datasetclient = DatasetManagementServiceProcessClient(node=self.container.node, process=process) self.workflowclient = WorkflowManagementServiceProcessClient(node=self.container.node, process=process) self.process_dispatcher = ProcessDispatcherServiceProcessClient(node=self.container.node, process=process) self.data_retriever = DataRetrieverServiceProcessClient(node=self.container.node, process=process) self.vis_client = VisualizationServiceProcessClient(node=self.container.node, process=process) self.ctd_stream_def = SBE37_CDM_stream_definition() def validate_messages(self, msgs): msg = msgs rdt = RecordDictionaryTool.load_from_granule(msg.body) vardict = {} vardict['temp'] = get_safe(rdt, 'temp') vardict['time'] = get_safe(rdt, 'time') print vardict['time'] print vardict['temp'] @attr('LOCOINT') #@patch.dict('pyon.ion.exchange.CFG', {'container':{'exchange':{'auto_register': False}}}) @unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False),'Not integrated for CEI') def test_visualization_queue(self): #The list of data product streams to monitor data_product_stream_ids = list() #Create the input data product ctd_stream_id, ctd_parsed_data_product_id = self.create_ctd_input_stream_and_data_product() data_product_stream_ids.append(ctd_stream_id) user_queue_name = 'user_queue' xq = self.container.ex_manager.create_xn_queue(user_queue_name) salinity_subscription_id = self.pubsubclient.create_subscription( stream_ids=data_product_stream_ids, exchange_name = user_queue_name, name = "user visualization queue" ) subscriber = Subscriber(from_name=xq) subscriber.initialize() # after the queue has been created it is safe to activate the subscription self.pubsubclient.activate_subscription(subscription_id=salinity_subscription_id) #Start the output stream listener to monitor and collect messages #results = self.start_output_stream_and_listen(None, data_product_stream_ids) #Not sure why this is needed - but it is #subscriber._chan.stop_consume() ctd_sim_pid = self.start_simple_input_stream_process(ctd_stream_id) gevent.sleep(10.0) # Send some messages - don't care how many msg_count,_ = xq.get_stats() log.info('Messages in user queue 1: %s ' % msg_count) #Validate the data from each of the messages along the way #self.validate_messages(results) # for x in range(msg_count): # mo = subscriber.get_one_msg(timeout=1) # print mo.body # mo.ack() msgs = subscriber.get_all_msgs(timeout=2) for x in range(len(msgs)): msgs[x].ack() self.validate_messages(msgs[x]) # print msgs[x].body #Should be zero after pulling all of the messages. msg_count,_ = xq.get_stats() log.info('Messages in user queue 2: %s ' % msg_count) #Trying to continue to receive messages in the queue gevent.sleep(5.0) # Send some messages - don't care how many #Turning off after everything - since it is more representative of an always on stream of data! self.process_dispatcher.cancel_process(ctd_sim_pid) # kill the ctd simulator process - that is enough data #Should see more messages in the queue msg_count,_ = xq.get_stats() log.info('Messages in user queue 3: %s ' % msg_count) msgs = subscriber.get_all_msgs(timeout=2) for x in range(len(msgs)): msgs[x].ack() self.validate_messages(msgs[x]) #Should be zero after pulling all of the messages. msg_count,_ = xq.get_stats() log.info('Messages in user queue 4: %s ' % msg_count) subscriber.close() self.container.ex_manager.delete_xn(xq) @unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False),'Not integrated for CEI') def test_multiple_visualization_queue(self): # set up a workflow with the salinity transform and the doubler. We will direct the original stream and the doubled stream to queues # and test to make sure the subscription to the queues is working correctly assertions = self.assertTrue # Build the workflow definition workflow_def_obj = IonObject(RT.WorkflowDefinition, name='Viz_Test_Workflow',description='A workflow to test collection of multiple data products in queues') workflow_data_product_name = 'TEST-Workflow_Output_Product' #Set a specific output product name #------------------------------------------------------------------------------------------------------------------------- #Add a transformation process definition for salinity #------------------------------------------------------------------------------------------------------------------------- ctd_L2_salinity_dprocdef_id = self.create_salinity_data_process_definition() workflow_step_obj = IonObject('DataProcessWorkflowStep', data_process_definition_id=ctd_L2_salinity_dprocdef_id, persist_process_output_data=False) #Don't persist the intermediate data product configuration = {'stream_name' : 'salinity'} workflow_step_obj.configuration = configuration workflow_def_obj.workflow_steps.append(workflow_step_obj) #Create it in the resource registry workflow_def_id = self.workflowclient.create_workflow_definition(workflow_def_obj) aids = self.rrclient.find_associations(workflow_def_id, PRED.hasDataProcessDefinition) assertions(len(aids) == 1 ) #The list of data product streams to monitor data_product_stream_ids = list() #Create the input data product ctd_stream_id, ctd_parsed_data_product_id = self.create_ctd_input_stream_and_data_product() data_product_stream_ids.append(ctd_stream_id) #Create and start the workflow workflow_id, workflow_product_id = self.workflowclient.create_data_process_workflow(workflow_def_id, ctd_parsed_data_product_id, timeout=30) workflow_output_ids,_ = self.rrclient.find_subjects(RT.Workflow, PRED.hasOutputProduct, workflow_product_id, True) assertions(len(workflow_output_ids) == 1 ) #Walk the associations to find the appropriate output data streams to validate the messages workflow_dp_ids,_ = self.rrclient.find_objects(workflow_id, PRED.hasDataProduct, RT.DataProduct, True) assertions(len(workflow_dp_ids) == 1 ) for dp_id in workflow_dp_ids: stream_ids, _ = self.rrclient.find_objects(dp_id, PRED.hasStream, None, True) assertions(len(stream_ids) == 1 ) data_product_stream_ids.append(stream_ids[0]) # Now for each of the data_product_stream_ids create a queue and pipe their data to the queue user_queue_name1 = 'user_queue_1' user_queue_name2 = 'user_queue_2' # use idempotency to create queues xq1 = self.container.ex_manager.create_xn_queue(user_queue_name1) self.addCleanup(xq1.delete) xq2 = self.container.ex_manager.create_xn_queue(user_queue_name2) self.addCleanup(xq2.delete) xq1.purge() xq2.purge() # the create_subscription call takes a list of stream_ids so create temp ones dp_stream_id1 = list() dp_stream_id1.append(data_product_stream_ids[0]) dp_stream_id2 = list() dp_stream_id2.append(data_product_stream_ids[1]) salinity_subscription_id1 = self.pubsubclient.create_subscription( stream_ids=dp_stream_id1, exchange_name = user_queue_name1, name = "user visualization queue1") salinity_subscription_id2 = self.pubsubclient.create_subscription( stream_ids=dp_stream_id2, exchange_name = user_queue_name2, name = "user visualization queue2") # Create subscribers for the output of the queue subscriber1 = Subscriber(from_name=xq1) subscriber1.initialize() subscriber2 = Subscriber(from_name=xq2) subscriber2.initialize() # after the queue has been created it is safe to activate the subscription self.pubsubclient.activate_subscription(subscription_id=salinity_subscription_id1) self.pubsubclient.activate_subscription(subscription_id=salinity_subscription_id2) # Start input stream and wait for some time ctd_sim_pid = self.start_simple_input_stream_process(ctd_stream_id) gevent.sleep(5.0) # Send some messages - don't care how many msg_count,_ = xq1.get_stats() log.info('Messages in user queue 1: %s ' % msg_count) msg_count,_ = xq2.get_stats() log.info('Messages in user queue 2: %s ' % msg_count) msgs1 = subscriber1.get_all_msgs(timeout=2) msgs2 = subscriber2.get_all_msgs(timeout=2) for x in range(min(len(msgs1), len(msgs2))): msgs1[x].ack() msgs2[x].ack() self.validate_multiple_vis_queue_messages(msgs1[x].body, msgs2[x].body) # kill the ctd simulator process - that is enough data self.process_dispatcher.cancel_process(ctd_sim_pid) # close the subscription and queues subscriber1.close() subscriber2.close() return #@unittest.skip('Skipped because of broken record dictionary work-around') def test_realtime_visualization(self): #Create the input data product ctd_stream_id, ctd_parsed_data_product_id = self.create_ctd_input_stream_and_data_product() ctd_sim_pid = self.start_sinusoidal_input_stream_process(ctd_stream_id) #TODO - Need to add workflow creation for google data table vis_params ={} vis_token = self.vis_client.initiate_realtime_visualization(data_product_id=ctd_parsed_data_product_id, visualization_parameters=vis_params) #Trying to continue to receive messages in the queue gevent.sleep(10.0) # Send some messages - don't care how many #TODO - find out what the actual return data type should be vis_data = self.vis_client.get_realtime_visualization_data(vis_token) if (vis_data): self.validate_google_dt_transform_results(vis_data) #Trying to continue to receive messages in the queue gevent.sleep(5.0) # Send some messages - don't care how many #Turning off after everything - since it is more representative of an always on stream of data! #todo remove the try except try: self.process_dispatcher.cancel_process(ctd_sim_pid) # kill the ctd simulator process - that is enough data except: log.warning("cancelling process did not work") vis_data = self.vis_client.get_realtime_visualization_data(vis_token) if vis_data: self.validate_google_dt_transform_results(vis_data) # Cleanup self.vis_client.terminate_realtime_visualization_data(vis_token) #@unittest.skip('Skipped because of broken record dictionary work-around') def test_google_dt_overview_visualization(self): #Create the input data product ctd_stream_id, ctd_parsed_data_product_id = self.create_ctd_input_stream_and_data_product() # start producing data ctd_sim_pid = self.start_sinusoidal_input_stream_process(ctd_stream_id) # Generate some data for a few seconds gevent.sleep(5.0) #Turning off after everything - since it is more representative of an always on stream of data! self.process_dispatcher.cancel_process(ctd_sim_pid) # kill the ctd simulator process - that is enough data # Use the data product to test the data retrieval and google dt generation capability of the vis service vis_data = self.vis_client.get_visualization_data(ctd_parsed_data_product_id) # validate the returned data self.validate_vis_service_google_dt_results(vis_data) #@unittest.skip('Skipped because of broken record dictionary work-around') def test_mpl_graphs_overview_visualization(self): #Create the input data product ctd_stream_id, ctd_parsed_data_product_id = self.create_ctd_input_stream_and_data_product() ctd_sim_pid = self.start_sinusoidal_input_stream_process(ctd_stream_id) # Generate some data for a few seconds gevent.sleep(5.0) #Turning off after everything - since it is more representative of an always on stream of data! self.process_dispatcher.cancel_process(ctd_sim_pid) # kill the ctd simulator process - that is enough data # Use the data product to test the data retrieval and google dt generation capability of the vis service vis_data = self.vis_client.get_visualization_image(ctd_parsed_data_product_id) # validate the returned data self.validate_vis_service_mpl_graphs_results(vis_data) return