def on_start(self): pubsub_cli = PubsubManagementServiceProcessClient(process=self, node=self.container.node) # Get the stream(s) stream_id = self.CFG.get_safe('process.stream_id','') query = StreamQuery(stream_ids=[stream_id,]) exchange_name = 'dispatcher_%s' % self.id subscription_id = pubsub_cli.create_subscription( query = query, exchange_name = exchange_name, name = "SampleSubscription", description = "Sample Subscription Description") stream_subscriber = StreamSubscriberRegistrar(process=self, node=self.container.node) def message_received(granule, h): rdt = RecordDictionaryTool.load_from_granule(granule) log.warn('Logging Record Dictionary received in logger subscription \n%s', rdt.pretty_print()) subscriber = stream_subscriber.create_subscriber(exchange_name=exchange_name, callback=message_received) subscriber.start() pubsub_cli.activate_subscription(subscription_id)
def on_start(self): rr_cli = ResourceRegistryServiceProcessClient(process=self, node=self.container.node) pubsub_cli = PubsubManagementServiceProcessClient(process=self, node=self.container.node) # Get the stream(s) data_product_id = self.CFG.get_safe('dispatcher.data_product_id','') stream_ids,_ = rr_cli.find_objects(subject=data_product_id, predicate=PRED.hasStream, id_only=True) log.info('Got Stream Ids: "%s"', stream_ids) assert stream_ids, 'No streams found for this data product!' exchange_name = 'dispatcher_%s' % str(os.getpid()) subscription_id = pubsub_cli.create_subscription( name='SampleSubscription', exchange_name=exchange_name, stream_ids=stream_ids, description='Sample Subscription Description' ) stream_defs = {} def message_received(message, stream_route, stream_id): granule = message stream_id = granule.stream_resource_id data_stream_id = granule.data_stream_id data_stream = granule.identifiables[data_stream_id] tstamp = get_datetime(data_stream.timestamp.value) records = granule.identifiables['record_count'].value log.info('Received a message from stream %s with time stamp %s and %d records' % (stream_id, tstamp, records)) if stream_id not in stream_defs: stream_defs[stream_id] = pubsub_cli.find_stream_definition(stream_id, id_only=False).container stream_def = stream_defs.get(stream_id) sp = PointSupplementStreamParser(stream_definition=stream_def, stream_granule=granule) last_data = {} for field in sp.list_field_names(): last_data[field] = sp.get_values(field)[-1] log.info('Last values in the message: %s' % str(last_data)) subscriber = StreamSubscriber(process=self, exchange_name=exchange_name, callback=message_received) subscriber.start() pubsub_cli.activate_subscription(subscription_id)
def on_start(self): super(StreamGranuleLogger,self).on_start() pubsub_cli = PubsubManagementServiceProcessClient(process=self, node=self.container.node) # Get the stream(s) stream_id = self.CFG.get_safe('process.stream_id','') if stream_id: self.subscription_id = pubsub_cli.create_subscription('%s_sub' % self.id, stream_ids=[stream_id], exchange_name=self.queue_name) pubsub_cli.activate_subscription(self.subscription_id)
def on_start(self): super(StreamGranuleLogger, self).on_start() pubsub_cli = PubsubManagementServiceProcessClient( process=self, node=self.container.node) # Get the stream(s) stream_id = self.CFG.get_safe('process.stream_id', '') if stream_id: self.subscription_id = pubsub_cli.create_subscription( '%s_sub' % self.id, stream_ids=[stream_id], exchange_name=self.queue_name) pubsub_cli.activate_subscription(self.subscription_id)
def on_start(self): pubsub_cli = PubsubManagementServiceProcessClient( process=self, node=self.container.node) # Get the stream(s) stream_id = self.CFG.get_safe('process.stream_id', '') query = StreamQuery(stream_ids=[ stream_id, ]) exchange_name = 'dispatcher_%s' % self.id subscription_id = pubsub_cli.create_subscription( query=query, exchange_name=exchange_name, name="SampleSubscription", description="Sample Subscription Description") stream_subscriber = StreamSubscriberRegistrar(process=self, node=self.container.node) def message_received(granule, h): rdt = RecordDictionaryTool.load_from_granule(granule) log.warn( 'Logging Record Dictionary received in logger subscription \n%s', rdt.pretty_print()) subscriber = stream_subscriber.create_subscriber( exchange_name=exchange_name, callback=message_received) subscriber.start() pubsub_cli.activate_subscription(subscription_id)
class TestVisualizationServiceIntegration(VisualizationIntegrationTestHelper): def setUp(self): # Start container logging.disable(logging.ERROR) self._start_container() self.container.start_rel_from_url('res/deploy/r2deploy.yml') # simulate preloading #preload_ion_params(self.container) logging.disable(logging.NOTSET) #Instantiate a process to represent the test process=VisualizationServiceTestProcess() # Now create client to DataProductManagementService self.rrclient = ResourceRegistryServiceProcessClient(node=self.container.node, process=process) self.damsclient = DataAcquisitionManagementServiceProcessClient(node=self.container.node, process=process) self.pubsubclient = PubsubManagementServiceProcessClient(node=self.container.node, process=process) self.ingestclient = IngestionManagementServiceProcessClient(node=self.container.node, process=process) self.imsclient = InstrumentManagementServiceProcessClient(node=self.container.node, process=process) self.dataproductclient = DataProductManagementServiceProcessClient(node=self.container.node, process=process) self.dataprocessclient = DataProcessManagementServiceProcessClient(node=self.container.node, process=process) self.datasetclient = DatasetManagementServiceProcessClient(node=self.container.node, process=process) self.process_dispatcher = ProcessDispatcherServiceProcessClient(node=self.container.node, process=process) self.data_retriever = DataRetrieverServiceProcessClient(node=self.container.node, process=process) self.vis_client = VisualizationServiceProcessClient(node=self.container.node, process=process) self.ctd_stream_def = SBE37_CDM_stream_definition() def validate_messages(self, msgs): msg = msgs rdt = RecordDictionaryTool.load_from_granule(msg.body) vardict = {} vardict['temp'] = get_safe(rdt, 'temp') vardict['time'] = get_safe(rdt, 'time') print vardict['time'] print vardict['temp'] @attr('LOCOINT') #@patch.dict('pyon.ion.exchange.CFG', {'container':{'exchange':{'auto_register': False}}}) @unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False),'Not integrated for CEI') def test_visualization_queue(self): #The list of data product streams to monitor data_product_stream_ids = list() #Create the input data product ctd_stream_id, ctd_parsed_data_product_id = self.create_ctd_input_stream_and_data_product() data_product_stream_ids.append(ctd_stream_id) user_queue_name = USER_VISUALIZATION_QUEUE xq = self.container.ex_manager.create_xn_queue(user_queue_name) salinity_subscription_id = self.pubsubclient.create_subscription( stream_ids=data_product_stream_ids, exchange_name = user_queue_name, name = "user visualization queue" ) subscriber = Subscriber(from_name=xq) subscriber.initialize() # after the queue has been created it is safe to activate the subscription self.pubsubclient.activate_subscription(subscription_id=salinity_subscription_id) #Start the output stream listener to monitor and collect messages #results = self.start_output_stream_and_listen(None, data_product_stream_ids) #Not sure why this is needed - but it is #subscriber._chan.stop_consume() ctd_sim_pid = self.start_simple_input_stream_process(ctd_stream_id) gevent.sleep(10.0) # Send some messages - don't care how many msg_count,_ = xq.get_stats() log.info('Messages in user queue 1: %s ' % msg_count) #Validate the data from each of the messages along the way #self.validate_messages(results) # for x in range(msg_count): # mo = subscriber.get_one_msg(timeout=1) # print mo.body # mo.ack() msgs = subscriber.get_all_msgs(timeout=2) for x in range(len(msgs)): msgs[x].ack() self.validate_messages(msgs[x]) # print msgs[x].body #Should be zero after pulling all of the messages. msg_count,_ = xq.get_stats() log.info('Messages in user queue 2: %s ' % msg_count) #Trying to continue to receive messages in the queue gevent.sleep(5.0) # Send some messages - don't care how many #Turning off after everything - since it is more representative of an always on stream of data! self.process_dispatcher.cancel_process(ctd_sim_pid) # kill the ctd simulator process - that is enough data #Should see more messages in the queue msg_count,_ = xq.get_stats() log.info('Messages in user queue 3: %s ' % msg_count) msgs = subscriber.get_all_msgs(timeout=2) for x in range(len(msgs)): msgs[x].ack() self.validate_messages(msgs[x]) #Should be zero after pulling all of the messages. msg_count,_ = xq.get_stats() log.info('Messages in user queue 4: %s ' % msg_count) subscriber.close() self.container.ex_manager.delete_xn(xq) @patch.dict(CFG, {'user_queue_monitor_timeout': 5}) def test_realtime_visualization(self): # #Start up multiple vis service workers if not a CEI launch # if not os.getenv('CEI_LAUNCH_TEST', False): # vpid1 = self.container.spawn_process('visualization_service1','ion.services.ans.visualization_service','VisualizationService', CFG ) # self.addCleanup(self.container.terminate_process, vpid1) # vpid2 = self.container.spawn_process('visualization_service2','ion.services.ans.visualization_service','VisualizationService', CFG ) # self.addCleanup(self.container.terminate_process, vpid2) ## Create the Highcharts workflow definition since there is no preload for the test #workflow_def_id = self.create_highcharts_workflow_def() #Create the input data product ctd_stream_id, ctd_parsed_data_product_id = self.create_ctd_input_stream_and_data_product() ctd_sim_pid = self.start_sinusoidal_input_stream_process(ctd_stream_id) vis_params ={} vis_token_resp = self.vis_client.initiate_realtime_visualization_data(data_product_id=ctd_parsed_data_product_id, visualization_parameters=simplejson.dumps(vis_params)) vis_token = ast.literal_eval(vis_token_resp)["rt_query_token"] result = gevent.event.AsyncResult() def get_vis_messages(get_data_count=7): #SHould be an odd number for round robbin processing by service workers get_cnt = 0 while get_cnt < get_data_count: vis_data = self.vis_client.get_realtime_visualization_data(vis_token) if (vis_data): self.validate_highcharts_transform_results(vis_data) get_cnt += 1 gevent.sleep(5) # simulates the polling from UI result.set(get_cnt) gevent.spawn(get_vis_messages) result.get(timeout=90) #Trying to continue to receive messages in the queue gevent.sleep(2.0) # Send some messages - don't care how many # Cleanup self.vis_client.terminate_realtime_visualization_data(vis_token) #Turning off after everything - since it is more representative of an always on stream of data! self.process_dispatcher.cancel_process(ctd_sim_pid) # kill the ctd simulator process - that is enough data @patch.dict(CFG, {'user_queue_monitor_timeout': 5}) @patch.dict(CFG, {'user_queue_monitor_size': 25}) @attr('CLEANUP') @unittest.skipIf(os.getenv('PYCC_MODE', False),'Not integrated for CEI') def test_realtime_visualization_cleanup(self): # #Start up multiple vis service workers if not a CEI launch # if not os.getenv('CEI_LAUNCH_TEST', False): # vpid1 = self.container.spawn_process('visualization_service1','ion.services.ans.visualization_service','VisualizationService', CFG ) # self.addCleanup(self.container.terminate_process, vpid1) # vpid2 = self.container.spawn_process('visualization_service2','ion.services.ans.visualization_service','VisualizationService', CFG ) # self.addCleanup(self.container.terminate_process, vpid2) #get the list of queues and message counts on the broker for the user vis queues try: queues = self.container.ex_manager.list_queues(name=USER_VISUALIZATION_QUEUE, return_columns=['name', 'messages']) q_names = [ q['name'] for q in queues if q['name']] #Get a list of only the queue names original_queue_count = len(q_names) except Exception, e: log.warn('Unable to get queue information from broker management plugin: ' + e.message) pass ## Create the highcharts workflow definition since there is no preload for the test #workflow_def_id = self.create_highcharts_workflow_def() #Create the input data product ctd_stream_id, ctd_parsed_data_product_id = self.create_ctd_input_stream_and_data_product() ctd_sim_pid = self.start_sinusoidal_input_stream_process(ctd_stream_id) #Start up a number of requests - and queues - to start accumulating messages. THe test will not clean them up #but instead check to see if the monitoring thread will. vis_token_resp = self.vis_client.initiate_realtime_visualization_data(data_product_id=ctd_parsed_data_product_id) bad_vis_token1 = ast.literal_eval(vis_token_resp)["rt_query_token"] vis_token_resp = self.vis_client.initiate_realtime_visualization_data(data_product_id=ctd_parsed_data_product_id) bad_vis_token2 = ast.literal_eval(vis_token_resp)["rt_query_token"] vis_token_resp = self.vis_client.initiate_realtime_visualization_data(data_product_id=ctd_parsed_data_product_id) bad_vis_token3 = ast.literal_eval(vis_token_resp)["rt_query_token"] vis_token_resp = self.vis_client.initiate_realtime_visualization_data(data_product_id=ctd_parsed_data_product_id) vis_token = ast.literal_eval(vis_token_resp)["rt_query_token"] #Get the default exchange space exchange = self.container.ex_manager.default_xs.exchange #get the list of queues and message counts on the broker for the user vis queues try: queues = self.container.ex_manager.list_queues(name=USER_VISUALIZATION_QUEUE, return_columns=['name', 'messages']) q_names = [ q['name'] for q in queues if q['name']] #Get a list of only the queue names self.assertIn(exchange + "." + bad_vis_token1, q_names) self.assertIn(exchange + "." + bad_vis_token2, q_names) self.assertIn(exchange + "." + bad_vis_token3, q_names) self.assertIn(exchange + "." + vis_token, q_names) except Exception, e: log.warn('Unable to get queue information from broker management plugin: ' + e.message) pass
class TestVisualizationServiceIntegration(VisualizationIntegrationTestHelper): def setUp(self): # Start container logging.disable(logging.ERROR) self._start_container() self.container.start_rel_from_url('res/deploy/r2deploy.yml') # simulate preloading preload_ion_params(self.container) logging.disable(logging.NOTSET) #Instantiate a process to represent the test process=VisualizationServiceTestProcess() # Now create client to DataProductManagementService self.rrclient = ResourceRegistryServiceProcessClient(node=self.container.node, process=process) self.damsclient = DataAcquisitionManagementServiceProcessClient(node=self.container.node, process=process) self.pubsubclient = PubsubManagementServiceProcessClient(node=self.container.node, process=process) self.ingestclient = IngestionManagementServiceProcessClient(node=self.container.node, process=process) self.imsclient = InstrumentManagementServiceProcessClient(node=self.container.node, process=process) self.dataproductclient = DataProductManagementServiceProcessClient(node=self.container.node, process=process) self.dataprocessclient = DataProcessManagementServiceProcessClient(node=self.container.node, process=process) self.datasetclient = DatasetManagementServiceProcessClient(node=self.container.node, process=process) self.workflowclient = WorkflowManagementServiceProcessClient(node=self.container.node, process=process) self.process_dispatcher = ProcessDispatcherServiceProcessClient(node=self.container.node, process=process) self.data_retriever = DataRetrieverServiceProcessClient(node=self.container.node, process=process) self.vis_client = VisualizationServiceProcessClient(node=self.container.node, process=process) self.ctd_stream_def = SBE37_CDM_stream_definition() def validate_messages(self, msgs): msg = msgs rdt = RecordDictionaryTool.load_from_granule(msg.body) vardict = {} vardict['temp'] = get_safe(rdt, 'temp') vardict['time'] = get_safe(rdt, 'time') print vardict['time'] print vardict['temp'] @attr('LOCOINT') #@patch.dict('pyon.ion.exchange.CFG', {'container':{'exchange':{'auto_register': False}}}) @unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False),'Not integrated for CEI') def test_visualization_queue(self): #The list of data product streams to monitor data_product_stream_ids = list() #Create the input data product ctd_stream_id, ctd_parsed_data_product_id = self.create_ctd_input_stream_and_data_product() data_product_stream_ids.append(ctd_stream_id) user_queue_name = USER_VISUALIZATION_QUEUE xq = self.container.ex_manager.create_xn_queue(user_queue_name) salinity_subscription_id = self.pubsubclient.create_subscription( stream_ids=data_product_stream_ids, exchange_name = user_queue_name, name = "user visualization queue" ) subscriber = Subscriber(from_name=xq) subscriber.initialize() # after the queue has been created it is safe to activate the subscription self.pubsubclient.activate_subscription(subscription_id=salinity_subscription_id) #Start the output stream listener to monitor and collect messages #results = self.start_output_stream_and_listen(None, data_product_stream_ids) #Not sure why this is needed - but it is #subscriber._chan.stop_consume() ctd_sim_pid = self.start_simple_input_stream_process(ctd_stream_id) gevent.sleep(10.0) # Send some messages - don't care how many msg_count,_ = xq.get_stats() log.info('Messages in user queue 1: %s ' % msg_count) #Validate the data from each of the messages along the way #self.validate_messages(results) # for x in range(msg_count): # mo = subscriber.get_one_msg(timeout=1) # print mo.body # mo.ack() msgs = subscriber.get_all_msgs(timeout=2) for x in range(len(msgs)): msgs[x].ack() self.validate_messages(msgs[x]) # print msgs[x].body #Should be zero after pulling all of the messages. msg_count,_ = xq.get_stats() log.info('Messages in user queue 2: %s ' % msg_count) #Trying to continue to receive messages in the queue gevent.sleep(5.0) # Send some messages - don't care how many #Turning off after everything - since it is more representative of an always on stream of data! self.process_dispatcher.cancel_process(ctd_sim_pid) # kill the ctd simulator process - that is enough data #Should see more messages in the queue msg_count,_ = xq.get_stats() log.info('Messages in user queue 3: %s ' % msg_count) msgs = subscriber.get_all_msgs(timeout=2) for x in range(len(msgs)): msgs[x].ack() self.validate_messages(msgs[x]) #Should be zero after pulling all of the messages. msg_count,_ = xq.get_stats() log.info('Messages in user queue 4: %s ' % msg_count) subscriber.close() self.container.ex_manager.delete_xn(xq) @unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False),'Not integrated for CEI') def test_multiple_visualization_queue(self): # set up a workflow with the salinity transform and the doubler. We will direct the original stream and the doubled stream to queues # and test to make sure the subscription to the queues is working correctly assertions = self.assertTrue # Build the workflow definition workflow_def_obj = IonObject(RT.WorkflowDefinition, name='Viz_Test_Workflow',description='A workflow to test collection of multiple data products in queues') workflow_data_product_name = 'TEST-Workflow_Output_Product' #Set a specific output product name #------------------------------------------------------------------------------------------------------------------------- #Add a transformation process definition for salinity #------------------------------------------------------------------------------------------------------------------------- ctd_L2_salinity_dprocdef_id = self.create_salinity_data_process_definition() workflow_step_obj = IonObject('DataProcessWorkflowStep', data_process_definition_id=ctd_L2_salinity_dprocdef_id, persist_process_output_data=False) #Don't persist the intermediate data product configuration = {'stream_name' : 'salinity'} workflow_step_obj.configuration = configuration workflow_def_obj.workflow_steps.append(workflow_step_obj) #Create it in the resource registry workflow_def_id = self.workflowclient.create_workflow_definition(workflow_def_obj) aids = self.rrclient.find_associations(workflow_def_id, PRED.hasDataProcessDefinition) assertions(len(aids) == 1 ) #The list of data product streams to monitor data_product_stream_ids = list() #Create the input data product ctd_stream_id, ctd_parsed_data_product_id = self.create_ctd_input_stream_and_data_product() data_product_stream_ids.append(ctd_stream_id) #Create and start the workflow workflow_id, workflow_product_id = self.workflowclient.create_data_process_workflow(workflow_def_id, ctd_parsed_data_product_id, timeout=30) workflow_output_ids,_ = self.rrclient.find_subjects(RT.Workflow, PRED.hasOutputProduct, workflow_product_id, True) assertions(len(workflow_output_ids) == 1 ) #Walk the associations to find the appropriate output data streams to validate the messages workflow_dp_ids,_ = self.rrclient.find_objects(workflow_id, PRED.hasDataProduct, RT.DataProduct, True) assertions(len(workflow_dp_ids) == 1 ) for dp_id in workflow_dp_ids: stream_ids, _ = self.rrclient.find_objects(dp_id, PRED.hasStream, None, True) assertions(len(stream_ids) == 1 ) data_product_stream_ids.append(stream_ids[0]) # Now for each of the data_product_stream_ids create a queue and pipe their data to the queue user_queue_name1 = USER_VISUALIZATION_QUEUE + '1' user_queue_name2 = USER_VISUALIZATION_QUEUE + '2' # use idempotency to create queues xq1 = self.container.ex_manager.create_xn_queue(user_queue_name1) self.addCleanup(xq1.delete) xq2 = self.container.ex_manager.create_xn_queue(user_queue_name2) self.addCleanup(xq2.delete) xq1.purge() xq2.purge() # the create_subscription call takes a list of stream_ids so create temp ones dp_stream_id1 = list() dp_stream_id1.append(data_product_stream_ids[0]) dp_stream_id2 = list() dp_stream_id2.append(data_product_stream_ids[1]) salinity_subscription_id1 = self.pubsubclient.create_subscription( stream_ids=dp_stream_id1, exchange_name = user_queue_name1, name = "user visualization queue1") salinity_subscription_id2 = self.pubsubclient.create_subscription( stream_ids=dp_stream_id2, exchange_name = user_queue_name2, name = "user visualization queue2") # Create subscribers for the output of the queue subscriber1 = Subscriber(from_name=xq1) subscriber1.initialize() subscriber2 = Subscriber(from_name=xq2) subscriber2.initialize() # after the queue has been created it is safe to activate the subscription self.pubsubclient.activate_subscription(subscription_id=salinity_subscription_id1) self.pubsubclient.activate_subscription(subscription_id=salinity_subscription_id2) # Start input stream and wait for some time ctd_sim_pid = self.start_simple_input_stream_process(ctd_stream_id) gevent.sleep(5.0) # Send some messages - don't care how many msg_count,_ = xq1.get_stats() log.info('Messages in user queue 1: %s ' % msg_count) msg_count,_ = xq2.get_stats() log.info('Messages in user queue 2: %s ' % msg_count) msgs1 = subscriber1.get_all_msgs(timeout=2) msgs2 = subscriber2.get_all_msgs(timeout=2) for x in range(min(len(msgs1), len(msgs2))): msgs1[x].ack() msgs2[x].ack() self.validate_multiple_vis_queue_messages(msgs1[x].body, msgs2[x].body) # kill the ctd simulator process - that is enough data self.process_dispatcher.cancel_process(ctd_sim_pid) # close the subscription and queues subscriber1.close() subscriber2.close() return @patch.dict(CFG, {'user_queue_monitor_timeout': 5}) @attr('SMOKE') def test_realtime_visualization(self): # #Start up multiple vis service workers if not a CEI launch # if not os.getenv('CEI_LAUNCH_TEST', False): # vpid1 = self.container.spawn_process('visualization_service1','ion.services.ans.visualization_service','VisualizationService', CFG ) # self.addCleanup(self.container.terminate_process, vpid1) # vpid2 = self.container.spawn_process('visualization_service2','ion.services.ans.visualization_service','VisualizationService', CFG ) # self.addCleanup(self.container.terminate_process, vpid2) # Create the Highcharts workflow definition since there is no preload for the test workflow_def_id = self.create_highcharts_workflow_def() #Create the input data product ctd_stream_id, ctd_parsed_data_product_id = self.create_ctd_input_stream_and_data_product() ctd_sim_pid = self.start_sinusoidal_input_stream_process(ctd_stream_id) vis_params ={} vis_token_resp = self.vis_client.initiate_realtime_visualization_data(data_product_id=ctd_parsed_data_product_id, visualization_parameters=simplejson.dumps(vis_params)) vis_token = ast.literal_eval(vis_token_resp)["rt_query_token"] result = gevent.event.AsyncResult() def get_vis_messages(get_data_count=7): #SHould be an odd number for round robbin processing by service workers get_cnt = 0 while get_cnt < get_data_count: vis_data = self.vis_client.get_realtime_visualization_data(vis_token) if (vis_data): self.validate_highcharts_transform_results(vis_data) get_cnt += 1 gevent.sleep(5) # simulates the polling from UI result.set(get_cnt) gevent.spawn(get_vis_messages) result.get(timeout=90) #Trying to continue to receive messages in the queue gevent.sleep(2.0) # Send some messages - don't care how many # Cleanup self.vis_client.terminate_realtime_visualization_data(vis_token) #Turning off after everything - since it is more representative of an always on stream of data! self.process_dispatcher.cancel_process(ctd_sim_pid) # kill the ctd simulator process - that is enough data @patch.dict(CFG, {'user_queue_monitor_timeout': 5}) @patch.dict(CFG, {'user_queue_monitor_size': 25}) @attr('CLEANUP') @unittest.skipIf(os.getenv('PYCC_MODE', False),'Not integrated for CEI') def test_realtime_visualization_cleanup(self): # #Start up multiple vis service workers if not a CEI launch # if not os.getenv('CEI_LAUNCH_TEST', False): # vpid1 = self.container.spawn_process('visualization_service1','ion.services.ans.visualization_service','VisualizationService', CFG ) # self.addCleanup(self.container.terminate_process, vpid1) # vpid2 = self.container.spawn_process('visualization_service2','ion.services.ans.visualization_service','VisualizationService', CFG ) # self.addCleanup(self.container.terminate_process, vpid2) #get the list of queues and message counts on the broker for the user vis queues try: queues = self.container.ex_manager.list_queues(name=USER_VISUALIZATION_QUEUE, return_columns=['name', 'messages']) q_names = [ q['name'] for q in queues if q['name']] #Get a list of only the queue names original_queue_count = len(q_names) except Exception, e: log.warn('Unable to get queue information from broker management plugin: ' + e.message) pass # Create the highcharts workflow definition since there is no preload for the test workflow_def_id = self.create_highcharts_workflow_def() #Create the input data product ctd_stream_id, ctd_parsed_data_product_id = self.create_ctd_input_stream_and_data_product() ctd_sim_pid = self.start_sinusoidal_input_stream_process(ctd_stream_id) #Start up a number of requests - and queues - to start accumulating messages. THe test will not clean them up #but instead check to see if the monitoring thread will. vis_token_resp = self.vis_client.initiate_realtime_visualization_data(data_product_id=ctd_parsed_data_product_id) bad_vis_token1 = ast.literal_eval(vis_token_resp)["rt_query_token"] vis_token_resp = self.vis_client.initiate_realtime_visualization_data(data_product_id=ctd_parsed_data_product_id) bad_vis_token2 = ast.literal_eval(vis_token_resp)["rt_query_token"] vis_token_resp = self.vis_client.initiate_realtime_visualization_data(data_product_id=ctd_parsed_data_product_id) bad_vis_token3 = ast.literal_eval(vis_token_resp)["rt_query_token"] vis_token_resp = self.vis_client.initiate_realtime_visualization_data(data_product_id=ctd_parsed_data_product_id) vis_token = ast.literal_eval(vis_token_resp)["rt_query_token"] #Get the default exchange space exchange = self.container.ex_manager.default_xs.exchange #get the list of queues and message counts on the broker for the user vis queues try: queues = self.container.ex_manager.list_queues(name=USER_VISUALIZATION_QUEUE, return_columns=['name', 'messages']) q_names = [ q['name'] for q in queues if q['name']] #Get a list of only the queue names self.assertIn(exchange + "." + bad_vis_token1, q_names) self.assertIn(exchange + "." + bad_vis_token2, q_names) self.assertIn(exchange + "." + bad_vis_token3, q_names) self.assertIn(exchange + "." + vis_token, q_names) except Exception, e: log.warn('Unable to get queue information from broker management plugin: ' + e.message) pass
class TestVisualizationServiceIntegration(VisualizationIntegrationTestHelper): def setUp(self): # Start container logging.disable(logging.ERROR) self._start_container() self.container.start_rel_from_url('res/deploy/r2deploy.yml') # simulate preloading #preload_ion_params(self.container) logging.disable(logging.NOTSET) #Instantiate a process to represent the test process = VisualizationServiceTestProcess() # Now create client to DataProductManagementService self.rrclient = ResourceRegistryServiceProcessClient( node=self.container.node, process=process) self.damsclient = DataAcquisitionManagementServiceProcessClient( node=self.container.node, process=process) self.pubsubclient = PubsubManagementServiceProcessClient( node=self.container.node, process=process) self.ingestclient = IngestionManagementServiceProcessClient( node=self.container.node, process=process) self.imsclient = InstrumentManagementServiceProcessClient( node=self.container.node, process=process) self.dataproductclient = DataProductManagementServiceProcessClient( node=self.container.node, process=process) self.dataprocessclient = DataProcessManagementServiceProcessClient( node=self.container.node, process=process) self.datasetclient = DatasetManagementServiceProcessClient( node=self.container.node, process=process) self.process_dispatcher = ProcessDispatcherServiceProcessClient( node=self.container.node, process=process) self.data_retriever = DataRetrieverServiceProcessClient( node=self.container.node, process=process) self.vis_client = VisualizationServiceProcessClient( node=self.container.node, process=process) self.ctd_stream_def = SBE37_CDM_stream_definition() def validate_messages(self, msgs): msg = msgs rdt = RecordDictionaryTool.load_from_granule(msg.body) vardict = {} vardict['temp'] = get_safe(rdt, 'temp') vardict['time'] = get_safe(rdt, 'time') print vardict['time'] print vardict['temp'] @attr('LOCOINT') #@patch.dict('pyon.ion.exchange.CFG', {'container':{'exchange':{'auto_register': False}}}) @unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False), 'Not integrated for CEI') def test_visualization_queue(self): #The list of data product streams to monitor data_product_stream_ids = list() #Create the input data product ctd_stream_id, ctd_parsed_data_product_id = self.create_ctd_input_stream_and_data_product( ) data_product_stream_ids.append(ctd_stream_id) user_queue_name = USER_VISUALIZATION_QUEUE xq = self.container.ex_manager.create_xn_queue(user_queue_name) salinity_subscription_id = self.pubsubclient.create_subscription( stream_ids=data_product_stream_ids, exchange_name=user_queue_name, name="user visualization queue") subscriber = Subscriber(from_name=xq) subscriber.initialize() # after the queue has been created it is safe to activate the subscription self.pubsubclient.activate_subscription( subscription_id=salinity_subscription_id) #Start the output stream listener to monitor and collect messages #results = self.start_output_stream_and_listen(None, data_product_stream_ids) #Not sure why this is needed - but it is #subscriber._chan.stop_consume() ctd_sim_pid = self.start_simple_input_stream_process(ctd_stream_id) gevent.sleep(10.0) # Send some messages - don't care how many msg_count, _ = xq.get_stats() log.info('Messages in user queue 1: %s ' % msg_count) #Validate the data from each of the messages along the way #self.validate_messages(results) # for x in range(msg_count): # mo = subscriber.get_one_msg(timeout=1) # print mo.body # mo.ack() msgs = subscriber.get_all_msgs(timeout=2) for x in range(len(msgs)): msgs[x].ack() self.validate_messages(msgs[x]) # print msgs[x].body #Should be zero after pulling all of the messages. msg_count, _ = xq.get_stats() log.info('Messages in user queue 2: %s ' % msg_count) #Trying to continue to receive messages in the queue gevent.sleep(5.0) # Send some messages - don't care how many #Turning off after everything - since it is more representative of an always on stream of data! self.process_dispatcher.cancel_process( ctd_sim_pid ) # kill the ctd simulator process - that is enough data #Should see more messages in the queue msg_count, _ = xq.get_stats() log.info('Messages in user queue 3: %s ' % msg_count) msgs = subscriber.get_all_msgs(timeout=2) for x in range(len(msgs)): msgs[x].ack() self.validate_messages(msgs[x]) #Should be zero after pulling all of the messages. msg_count, _ = xq.get_stats() log.info('Messages in user queue 4: %s ' % msg_count) subscriber.close() self.container.ex_manager.delete_xn(xq) @patch.dict(CFG, {'user_queue_monitor_timeout': 5}) def test_realtime_visualization(self): # #Start up multiple vis service workers if not a CEI launch # if not os.getenv('CEI_LAUNCH_TEST', False): # vpid1 = self.container.spawn_process('visualization_service1','ion.services.ans.visualization_service','VisualizationService', CFG ) # self.addCleanup(self.container.terminate_process, vpid1) # vpid2 = self.container.spawn_process('visualization_service2','ion.services.ans.visualization_service','VisualizationService', CFG ) # self.addCleanup(self.container.terminate_process, vpid2) ## Create the Highcharts workflow definition since there is no preload for the test #workflow_def_id = self.create_highcharts_workflow_def() #Create the input data product ctd_stream_id, ctd_parsed_data_product_id = self.create_ctd_input_stream_and_data_product( ) ctd_sim_pid = self.start_sinusoidal_input_stream_process(ctd_stream_id) vis_params = {} vis_token_resp = self.vis_client.initiate_realtime_visualization_data( data_product_id=ctd_parsed_data_product_id, visualization_parameters=simplejson.dumps(vis_params)) vis_token = ast.literal_eval(vis_token_resp)["rt_query_token"] result = gevent.event.AsyncResult() def get_vis_messages( get_data_count=7 ): #SHould be an odd number for round robbin processing by service workers get_cnt = 0 while get_cnt < get_data_count: vis_data = self.vis_client.get_realtime_visualization_data( vis_token) if (vis_data): self.validate_highcharts_transform_results(vis_data) get_cnt += 1 gevent.sleep(5) # simulates the polling from UI result.set(get_cnt) gevent.spawn(get_vis_messages) result.get(timeout=90) #Trying to continue to receive messages in the queue gevent.sleep(2.0) # Send some messages - don't care how many # Cleanup self.vis_client.terminate_realtime_visualization_data(vis_token) #Turning off after everything - since it is more representative of an always on stream of data! self.process_dispatcher.cancel_process( ctd_sim_pid ) # kill the ctd simulator process - that is enough data @patch.dict(CFG, {'user_queue_monitor_timeout': 5}) @patch.dict(CFG, {'user_queue_monitor_size': 25}) @attr('CLEANUP') @unittest.skipIf(os.getenv('PYCC_MODE', False), 'Not integrated for CEI') def test_realtime_visualization_cleanup(self): # #Start up multiple vis service workers if not a CEI launch # if not os.getenv('CEI_LAUNCH_TEST', False): # vpid1 = self.container.spawn_process('visualization_service1','ion.services.ans.visualization_service','VisualizationService', CFG ) # self.addCleanup(self.container.terminate_process, vpid1) # vpid2 = self.container.spawn_process('visualization_service2','ion.services.ans.visualization_service','VisualizationService', CFG ) # self.addCleanup(self.container.terminate_process, vpid2) #get the list of queues and message counts on the broker for the user vis queues try: queues = self.container.ex_manager.list_queues( name=USER_VISUALIZATION_QUEUE, return_columns=['name', 'messages']) q_names = [q['name'] for q in queues if q['name']] #Get a list of only the queue names original_queue_count = len(q_names) except Exception, e: log.warn( 'Unable to get queue information from broker management plugin: ' + e.message) pass ## Create the highcharts workflow definition since there is no preload for the test #workflow_def_id = self.create_highcharts_workflow_def() #Create the input data product ctd_stream_id, ctd_parsed_data_product_id = self.create_ctd_input_stream_and_data_product( ) ctd_sim_pid = self.start_sinusoidal_input_stream_process(ctd_stream_id) #Start up a number of requests - and queues - to start accumulating messages. THe test will not clean them up #but instead check to see if the monitoring thread will. vis_token_resp = self.vis_client.initiate_realtime_visualization_data( data_product_id=ctd_parsed_data_product_id) bad_vis_token1 = ast.literal_eval(vis_token_resp)["rt_query_token"] vis_token_resp = self.vis_client.initiate_realtime_visualization_data( data_product_id=ctd_parsed_data_product_id) bad_vis_token2 = ast.literal_eval(vis_token_resp)["rt_query_token"] vis_token_resp = self.vis_client.initiate_realtime_visualization_data( data_product_id=ctd_parsed_data_product_id) bad_vis_token3 = ast.literal_eval(vis_token_resp)["rt_query_token"] vis_token_resp = self.vis_client.initiate_realtime_visualization_data( data_product_id=ctd_parsed_data_product_id) vis_token = ast.literal_eval(vis_token_resp)["rt_query_token"] #Get the default exchange space exchange = self.container.ex_manager.default_xs.exchange #get the list of queues and message counts on the broker for the user vis queues try: queues = self.container.ex_manager.list_queues( name=USER_VISUALIZATION_QUEUE, return_columns=['name', 'messages']) q_names = [q['name'] for q in queues if q['name']] #Get a list of only the queue names self.assertIn(exchange + "." + bad_vis_token1, q_names) self.assertIn(exchange + "." + bad_vis_token2, q_names) self.assertIn(exchange + "." + bad_vis_token3, q_names) self.assertIn(exchange + "." + vis_token, q_names) except Exception, e: log.warn( 'Unable to get queue information from broker management plugin: ' + e.message) pass
class TestVisualizationServiceIntegration(VisualizationIntegrationTestHelper): def setUp(self): # Start container logging.disable(logging.ERROR) self._start_container() self.container.start_rel_from_url('res/deploy/r2deploy.yml') logging.disable(logging.NOTSET) #Instantiate a process to represent the test process=VisualizationServiceTestProcess() # Now create client to DataProductManagementService self.rrclient = ResourceRegistryServiceProcessClient(node=self.container.node, process=process) self.damsclient = DataAcquisitionManagementServiceProcessClient(node=self.container.node, process=process) self.pubsubclient = PubsubManagementServiceProcessClient(node=self.container.node, process=process) self.ingestclient = IngestionManagementServiceProcessClient(node=self.container.node, process=process) self.imsclient = InstrumentManagementServiceProcessClient(node=self.container.node, process=process) self.dataproductclient = DataProductManagementServiceProcessClient(node=self.container.node, process=process) self.dataprocessclient = DataProcessManagementServiceProcessClient(node=self.container.node, process=process) self.datasetclient = DatasetManagementServiceProcessClient(node=self.container.node, process=process) self.workflowclient = WorkflowManagementServiceProcessClient(node=self.container.node, process=process) self.process_dispatcher = ProcessDispatcherServiceProcessClient(node=self.container.node, process=process) self.data_retriever = DataRetrieverServiceProcessClient(node=self.container.node, process=process) self.vis_client = VisualizationServiceProcessClient(node=self.container.node, process=process) self.ctd_stream_def = SBE37_CDM_stream_definition() def validate_messages(self, msgs): msg = msgs rdt = RecordDictionaryTool.load_from_granule(msg.body) vardict = {} vardict['temp'] = get_safe(rdt, 'temp') vardict['time'] = get_safe(rdt, 'time') print vardict['time'] print vardict['temp'] @attr('LOCOINT') #@patch.dict('pyon.ion.exchange.CFG', {'container':{'exchange':{'auto_register': False}}}) @unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False),'Not integrated for CEI') def test_visualization_queue(self): #The list of data product streams to monitor data_product_stream_ids = list() #Create the input data product ctd_stream_id, ctd_parsed_data_product_id = self.create_ctd_input_stream_and_data_product() data_product_stream_ids.append(ctd_stream_id) user_queue_name = 'user_queue' xq = self.container.ex_manager.create_xn_queue(user_queue_name) salinity_subscription_id = self.pubsubclient.create_subscription( stream_ids=data_product_stream_ids, exchange_name = user_queue_name, name = "user visualization queue" ) subscriber = Subscriber(from_name=xq) subscriber.initialize() # after the queue has been created it is safe to activate the subscription self.pubsubclient.activate_subscription(subscription_id=salinity_subscription_id) #Start the output stream listener to monitor and collect messages #results = self.start_output_stream_and_listen(None, data_product_stream_ids) #Not sure why this is needed - but it is #subscriber._chan.stop_consume() ctd_sim_pid = self.start_simple_input_stream_process(ctd_stream_id) gevent.sleep(10.0) # Send some messages - don't care how many msg_count,_ = xq.get_stats() log.info('Messages in user queue 1: %s ' % msg_count) #Validate the data from each of the messages along the way #self.validate_messages(results) # for x in range(msg_count): # mo = subscriber.get_one_msg(timeout=1) # print mo.body # mo.ack() msgs = subscriber.get_all_msgs(timeout=2) for x in range(len(msgs)): msgs[x].ack() self.validate_messages(msgs[x]) # print msgs[x].body #Should be zero after pulling all of the messages. msg_count,_ = xq.get_stats() log.info('Messages in user queue 2: %s ' % msg_count) #Trying to continue to receive messages in the queue gevent.sleep(5.0) # Send some messages - don't care how many #Turning off after everything - since it is more representative of an always on stream of data! self.process_dispatcher.cancel_process(ctd_sim_pid) # kill the ctd simulator process - that is enough data #Should see more messages in the queue msg_count,_ = xq.get_stats() log.info('Messages in user queue 3: %s ' % msg_count) msgs = subscriber.get_all_msgs(timeout=2) for x in range(len(msgs)): msgs[x].ack() self.validate_messages(msgs[x]) #Should be zero after pulling all of the messages. msg_count,_ = xq.get_stats() log.info('Messages in user queue 4: %s ' % msg_count) subscriber.close() self.container.ex_manager.delete_xn(xq) @unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False),'Not integrated for CEI') def test_multiple_visualization_queue(self): # set up a workflow with the salinity transform and the doubler. We will direct the original stream and the doubled stream to queues # and test to make sure the subscription to the queues is working correctly assertions = self.assertTrue # Build the workflow definition workflow_def_obj = IonObject(RT.WorkflowDefinition, name='Viz_Test_Workflow',description='A workflow to test collection of multiple data products in queues') workflow_data_product_name = 'TEST-Workflow_Output_Product' #Set a specific output product name #------------------------------------------------------------------------------------------------------------------------- #Add a transformation process definition for salinity #------------------------------------------------------------------------------------------------------------------------- ctd_L2_salinity_dprocdef_id = self.create_salinity_data_process_definition() workflow_step_obj = IonObject('DataProcessWorkflowStep', data_process_definition_id=ctd_L2_salinity_dprocdef_id, persist_process_output_data=False) #Don't persist the intermediate data product configuration = {'stream_name' : 'salinity'} workflow_step_obj.configuration = configuration workflow_def_obj.workflow_steps.append(workflow_step_obj) #Create it in the resource registry workflow_def_id = self.workflowclient.create_workflow_definition(workflow_def_obj) aids = self.rrclient.find_associations(workflow_def_id, PRED.hasDataProcessDefinition) assertions(len(aids) == 1 ) #The list of data product streams to monitor data_product_stream_ids = list() #Create the input data product ctd_stream_id, ctd_parsed_data_product_id = self.create_ctd_input_stream_and_data_product() data_product_stream_ids.append(ctd_stream_id) #Create and start the workflow workflow_id, workflow_product_id = self.workflowclient.create_data_process_workflow(workflow_def_id, ctd_parsed_data_product_id, timeout=30) workflow_output_ids,_ = self.rrclient.find_subjects(RT.Workflow, PRED.hasOutputProduct, workflow_product_id, True) assertions(len(workflow_output_ids) == 1 ) #Walk the associations to find the appropriate output data streams to validate the messages workflow_dp_ids,_ = self.rrclient.find_objects(workflow_id, PRED.hasDataProduct, RT.DataProduct, True) assertions(len(workflow_dp_ids) == 1 ) for dp_id in workflow_dp_ids: stream_ids, _ = self.rrclient.find_objects(dp_id, PRED.hasStream, None, True) assertions(len(stream_ids) == 1 ) data_product_stream_ids.append(stream_ids[0]) # Now for each of the data_product_stream_ids create a queue and pipe their data to the queue user_queue_name1 = 'user_queue_1' user_queue_name2 = 'user_queue_2' # use idempotency to create queues xq1 = self.container.ex_manager.create_xn_queue(user_queue_name1) self.addCleanup(xq1.delete) xq2 = self.container.ex_manager.create_xn_queue(user_queue_name2) self.addCleanup(xq2.delete) xq1.purge() xq2.purge() # the create_subscription call takes a list of stream_ids so create temp ones dp_stream_id1 = list() dp_stream_id1.append(data_product_stream_ids[0]) dp_stream_id2 = list() dp_stream_id2.append(data_product_stream_ids[1]) salinity_subscription_id1 = self.pubsubclient.create_subscription( stream_ids=dp_stream_id1, exchange_name = user_queue_name1, name = "user visualization queue1") salinity_subscription_id2 = self.pubsubclient.create_subscription( stream_ids=dp_stream_id2, exchange_name = user_queue_name2, name = "user visualization queue2") # Create subscribers for the output of the queue subscriber1 = Subscriber(from_name=xq1) subscriber1.initialize() subscriber2 = Subscriber(from_name=xq2) subscriber2.initialize() # after the queue has been created it is safe to activate the subscription self.pubsubclient.activate_subscription(subscription_id=salinity_subscription_id1) self.pubsubclient.activate_subscription(subscription_id=salinity_subscription_id2) # Start input stream and wait for some time ctd_sim_pid = self.start_simple_input_stream_process(ctd_stream_id) gevent.sleep(5.0) # Send some messages - don't care how many msg_count,_ = xq1.get_stats() log.info('Messages in user queue 1: %s ' % msg_count) msg_count,_ = xq2.get_stats() log.info('Messages in user queue 2: %s ' % msg_count) msgs1 = subscriber1.get_all_msgs(timeout=2) msgs2 = subscriber2.get_all_msgs(timeout=2) for x in range(min(len(msgs1), len(msgs2))): msgs1[x].ack() msgs2[x].ack() self.validate_multiple_vis_queue_messages(msgs1[x].body, msgs2[x].body) # kill the ctd simulator process - that is enough data self.process_dispatcher.cancel_process(ctd_sim_pid) # close the subscription and queues subscriber1.close() subscriber2.close() return #@unittest.skip('Skipped because of broken record dictionary work-around') def test_realtime_visualization(self): #Create the input data product ctd_stream_id, ctd_parsed_data_product_id = self.create_ctd_input_stream_and_data_product() ctd_sim_pid = self.start_sinusoidal_input_stream_process(ctd_stream_id) #TODO - Need to add workflow creation for google data table vis_params ={} vis_token = self.vis_client.initiate_realtime_visualization(data_product_id=ctd_parsed_data_product_id, visualization_parameters=vis_params) #Trying to continue to receive messages in the queue gevent.sleep(10.0) # Send some messages - don't care how many #TODO - find out what the actual return data type should be vis_data = self.vis_client.get_realtime_visualization_data(vis_token) if (vis_data): self.validate_google_dt_transform_results(vis_data) #Trying to continue to receive messages in the queue gevent.sleep(5.0) # Send some messages - don't care how many #Turning off after everything - since it is more representative of an always on stream of data! #todo remove the try except try: self.process_dispatcher.cancel_process(ctd_sim_pid) # kill the ctd simulator process - that is enough data except: log.warning("cancelling process did not work") vis_data = self.vis_client.get_realtime_visualization_data(vis_token) if vis_data: self.validate_google_dt_transform_results(vis_data) # Cleanup self.vis_client.terminate_realtime_visualization_data(vis_token) #@unittest.skip('Skipped because of broken record dictionary work-around') def test_google_dt_overview_visualization(self): #Create the input data product ctd_stream_id, ctd_parsed_data_product_id = self.create_ctd_input_stream_and_data_product() # start producing data ctd_sim_pid = self.start_sinusoidal_input_stream_process(ctd_stream_id) # Generate some data for a few seconds gevent.sleep(5.0) #Turning off after everything - since it is more representative of an always on stream of data! self.process_dispatcher.cancel_process(ctd_sim_pid) # kill the ctd simulator process - that is enough data # Use the data product to test the data retrieval and google dt generation capability of the vis service vis_data = self.vis_client.get_visualization_data(ctd_parsed_data_product_id) # validate the returned data self.validate_vis_service_google_dt_results(vis_data) #@unittest.skip('Skipped because of broken record dictionary work-around') def test_mpl_graphs_overview_visualization(self): #Create the input data product ctd_stream_id, ctd_parsed_data_product_id = self.create_ctd_input_stream_and_data_product() ctd_sim_pid = self.start_sinusoidal_input_stream_process(ctd_stream_id) # Generate some data for a few seconds gevent.sleep(5.0) #Turning off after everything - since it is more representative of an always on stream of data! self.process_dispatcher.cancel_process(ctd_sim_pid) # kill the ctd simulator process - that is enough data # Use the data product to test the data retrieval and google dt generation capability of the vis service vis_data = self.vis_client.get_visualization_image(ctd_parsed_data_product_id) # validate the returned data self.validate_vis_service_mpl_graphs_results(vis_data) return
class TestVisualizationServiceIntegration(VisualizationIntegrationTestHelper): def setUp(self): # Start container logging.disable(logging.ERROR) self._start_container() self.container.start_rel_from_url('res/deploy/r2deploy.yml') logging.disable(logging.NOTSET) #Instantiate a process to represent the test process = VisualizationServiceTestProcess() # Now create client to DataProductManagementService self.rrclient = ResourceRegistryServiceProcessClient( node=self.container.node, process=process) self.damsclient = DataAcquisitionManagementServiceProcessClient( node=self.container.node, process=process) self.pubsubclient = PubsubManagementServiceProcessClient( node=self.container.node, process=process) self.ingestclient = IngestionManagementServiceProcessClient( node=self.container.node, process=process) self.imsclient = InstrumentManagementServiceProcessClient( node=self.container.node, process=process) self.dataproductclient = DataProductManagementServiceProcessClient( node=self.container.node, process=process) self.dataprocessclient = DataProcessManagementServiceProcessClient( node=self.container.node, process=process) self.datasetclient = DatasetManagementServiceProcessClient( node=self.container.node, process=process) self.workflowclient = WorkflowManagementServiceProcessClient( node=self.container.node, process=process) self.process_dispatcher = ProcessDispatcherServiceProcessClient( node=self.container.node, process=process) self.data_retriever = DataRetrieverServiceProcessClient( node=self.container.node, process=process) self.vis_client = VisualizationServiceProcessClient( node=self.container.node, process=process) self.ctd_stream_def = SBE37_CDM_stream_definition() def validate_messages(self, msgs): msg = msgs rdt = RecordDictionaryTool.load_from_granule(msg.body) vardict = {} vardict['temp'] = get_safe(rdt, 'temp') vardict['time'] = get_safe(rdt, 'time') print vardict['time'] print vardict['temp'] @attr('LOCOINT') #@patch.dict('pyon.ion.exchange.CFG', {'container':{'exchange':{'auto_register': False}}}) @unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False), 'Not integrated for CEI') def test_visualization_queue(self): #The list of data product streams to monitor data_product_stream_ids = list() #Create the input data product ctd_stream_id, ctd_parsed_data_product_id = self.create_ctd_input_stream_and_data_product( ) data_product_stream_ids.append(ctd_stream_id) user_queue_name = 'user_queue' xq = self.container.ex_manager.create_xn_queue(user_queue_name) salinity_subscription_id = self.pubsubclient.create_subscription( stream_ids=data_product_stream_ids, exchange_name=user_queue_name, name="user visualization queue") subscriber = Subscriber(from_name=xq) subscriber.initialize() # after the queue has been created it is safe to activate the subscription self.pubsubclient.activate_subscription( subscription_id=salinity_subscription_id) #Start the output stream listener to monitor and collect messages #results = self.start_output_stream_and_listen(None, data_product_stream_ids) #Not sure why this is needed - but it is #subscriber._chan.stop_consume() ctd_sim_pid = self.start_simple_input_stream_process(ctd_stream_id) gevent.sleep(10.0) # Send some messages - don't care how many msg_count, _ = xq.get_stats() log.info('Messages in user queue 1: %s ' % msg_count) #Validate the data from each of the messages along the way #self.validate_messages(results) # for x in range(msg_count): # mo = subscriber.get_one_msg(timeout=1) # print mo.body # mo.ack() msgs = subscriber.get_all_msgs(timeout=2) for x in range(len(msgs)): msgs[x].ack() self.validate_messages(msgs[x]) # print msgs[x].body #Should be zero after pulling all of the messages. msg_count, _ = xq.get_stats() log.info('Messages in user queue 2: %s ' % msg_count) #Trying to continue to receive messages in the queue gevent.sleep(5.0) # Send some messages - don't care how many #Turning off after everything - since it is more representative of an always on stream of data! self.process_dispatcher.cancel_process( ctd_sim_pid ) # kill the ctd simulator process - that is enough data #Should see more messages in the queue msg_count, _ = xq.get_stats() log.info('Messages in user queue 3: %s ' % msg_count) msgs = subscriber.get_all_msgs(timeout=2) for x in range(len(msgs)): msgs[x].ack() self.validate_messages(msgs[x]) #Should be zero after pulling all of the messages. msg_count, _ = xq.get_stats() log.info('Messages in user queue 4: %s ' % msg_count) subscriber.close() self.container.ex_manager.delete_xn(xq) @unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False), 'Not integrated for CEI') def test_multiple_visualization_queue(self): # set up a workflow with the salinity transform and the doubler. We will direct the original stream and the doubled stream to queues # and test to make sure the subscription to the queues is working correctly assertions = self.assertTrue # Build the workflow definition workflow_def_obj = IonObject( RT.WorkflowDefinition, name='Viz_Test_Workflow', description= 'A workflow to test collection of multiple data products in queues' ) workflow_data_product_name = 'TEST-Workflow_Output_Product' #Set a specific output product name #------------------------------------------------------------------------------------------------------------------------- #Add a transformation process definition for salinity #------------------------------------------------------------------------------------------------------------------------- ctd_L2_salinity_dprocdef_id = self.create_salinity_data_process_definition( ) workflow_step_obj = IonObject( 'DataProcessWorkflowStep', data_process_definition_id=ctd_L2_salinity_dprocdef_id, persist_process_output_data=False ) #Don't persist the intermediate data product configuration = {'stream_name': 'salinity'} workflow_step_obj.configuration = configuration workflow_def_obj.workflow_steps.append(workflow_step_obj) #Create it in the resource registry workflow_def_id = self.workflowclient.create_workflow_definition( workflow_def_obj) aids = self.rrclient.find_associations(workflow_def_id, PRED.hasDataProcessDefinition) assertions(len(aids) == 1) #The list of data product streams to monitor data_product_stream_ids = list() #Create the input data product ctd_stream_id, ctd_parsed_data_product_id = self.create_ctd_input_stream_and_data_product( ) data_product_stream_ids.append(ctd_stream_id) #Create and start the workflow workflow_id, workflow_product_id = self.workflowclient.create_data_process_workflow( workflow_def_id, ctd_parsed_data_product_id, timeout=30) workflow_output_ids, _ = self.rrclient.find_subjects( RT.Workflow, PRED.hasOutputProduct, workflow_product_id, True) assertions(len(workflow_output_ids) == 1) #Walk the associations to find the appropriate output data streams to validate the messages workflow_dp_ids, _ = self.rrclient.find_objects( workflow_id, PRED.hasDataProduct, RT.DataProduct, True) assertions(len(workflow_dp_ids) == 1) for dp_id in workflow_dp_ids: stream_ids, _ = self.rrclient.find_objects(dp_id, PRED.hasStream, None, True) assertions(len(stream_ids) == 1) data_product_stream_ids.append(stream_ids[0]) # Now for each of the data_product_stream_ids create a queue and pipe their data to the queue user_queue_name1 = 'user_queue_1' user_queue_name2 = 'user_queue_2' # use idempotency to create queues xq1 = self.container.ex_manager.create_xn_queue(user_queue_name1) self.addCleanup(xq1.delete) xq2 = self.container.ex_manager.create_xn_queue(user_queue_name2) self.addCleanup(xq2.delete) xq1.purge() xq2.purge() # the create_subscription call takes a list of stream_ids so create temp ones dp_stream_id1 = list() dp_stream_id1.append(data_product_stream_ids[0]) dp_stream_id2 = list() dp_stream_id2.append(data_product_stream_ids[1]) salinity_subscription_id1 = self.pubsubclient.create_subscription( stream_ids=dp_stream_id1, exchange_name=user_queue_name1, name="user visualization queue1") salinity_subscription_id2 = self.pubsubclient.create_subscription( stream_ids=dp_stream_id2, exchange_name=user_queue_name2, name="user visualization queue2") # Create subscribers for the output of the queue subscriber1 = Subscriber(from_name=xq1) subscriber1.initialize() subscriber2 = Subscriber(from_name=xq2) subscriber2.initialize() # after the queue has been created it is safe to activate the subscription self.pubsubclient.activate_subscription( subscription_id=salinity_subscription_id1) self.pubsubclient.activate_subscription( subscription_id=salinity_subscription_id2) # Start input stream and wait for some time ctd_sim_pid = self.start_simple_input_stream_process(ctd_stream_id) gevent.sleep(5.0) # Send some messages - don't care how many msg_count, _ = xq1.get_stats() log.info('Messages in user queue 1: %s ' % msg_count) msg_count, _ = xq2.get_stats() log.info('Messages in user queue 2: %s ' % msg_count) msgs1 = subscriber1.get_all_msgs(timeout=2) msgs2 = subscriber2.get_all_msgs(timeout=2) for x in range(min(len(msgs1), len(msgs2))): msgs1[x].ack() msgs2[x].ack() self.validate_multiple_vis_queue_messages(msgs1[x].body, msgs2[x].body) # kill the ctd simulator process - that is enough data self.process_dispatcher.cancel_process(ctd_sim_pid) # close the subscription and queues subscriber1.close() subscriber2.close() return #@unittest.skip('Skipped because of broken record dictionary work-around') def test_realtime_visualization(self): #Create the input data product ctd_stream_id, ctd_parsed_data_product_id = self.create_ctd_input_stream_and_data_product( ) ctd_sim_pid = self.start_sinusoidal_input_stream_process(ctd_stream_id) #TODO - Need to add workflow creation for google data table vis_params = {} vis_token = self.vis_client.initiate_realtime_visualization( data_product_id=ctd_parsed_data_product_id, visualization_parameters=vis_params) #Trying to continue to receive messages in the queue gevent.sleep(10.0) # Send some messages - don't care how many #TODO - find out what the actual return data type should be vis_data = self.vis_client.get_realtime_visualization_data(vis_token) if (vis_data): self.validate_google_dt_transform_results(vis_data) #Trying to continue to receive messages in the queue gevent.sleep(5.0) # Send some messages - don't care how many #Turning off after everything - since it is more representative of an always on stream of data! #todo remove the try except try: self.process_dispatcher.cancel_process( ctd_sim_pid ) # kill the ctd simulator process - that is enough data except: log.warning("cancelling process did not work") vis_data = self.vis_client.get_realtime_visualization_data(vis_token) if vis_data: self.validate_google_dt_transform_results(vis_data) # Cleanup self.vis_client.terminate_realtime_visualization_data(vis_token) #@unittest.skip('Skipped because of broken record dictionary work-around') def test_google_dt_overview_visualization(self): #Create the input data product ctd_stream_id, ctd_parsed_data_product_id = self.create_ctd_input_stream_and_data_product( ) # start producing data ctd_sim_pid = self.start_sinusoidal_input_stream_process(ctd_stream_id) # Generate some data for a few seconds gevent.sleep(5.0) #Turning off after everything - since it is more representative of an always on stream of data! self.process_dispatcher.cancel_process( ctd_sim_pid ) # kill the ctd simulator process - that is enough data # Use the data product to test the data retrieval and google dt generation capability of the vis service vis_data = self.vis_client.get_visualization_data( ctd_parsed_data_product_id) # validate the returned data self.validate_vis_service_google_dt_results(vis_data) #@unittest.skip('Skipped because of broken record dictionary work-around') def test_mpl_graphs_overview_visualization(self): #Create the input data product ctd_stream_id, ctd_parsed_data_product_id = self.create_ctd_input_stream_and_data_product( ) ctd_sim_pid = self.start_sinusoidal_input_stream_process(ctd_stream_id) # Generate some data for a few seconds gevent.sleep(5.0) #Turning off after everything - since it is more representative of an always on stream of data! self.process_dispatcher.cancel_process( ctd_sim_pid ) # kill the ctd simulator process - that is enough data # Use the data product to test the data retrieval and google dt generation capability of the vis service vis_data = self.vis_client.get_visualization_image( ctd_parsed_data_product_id) # validate the returned data self.validate_vis_service_mpl_graphs_results(vis_data) return