def on_start(self):
try:
SimpleProcess.on_start(self)
self.viz_service = VisualizationServiceClient()
# Get config params
self.monitor_timeout = get_safe(self.CFG,
'user_queue_monitor_timeout', 300)
self.monitor_queue_size = get_safe(self.CFG,
'user_queue_monitor_size', 100)
self.monitor_event = gevent.event.Event()
self.monitor_event.clear()
self._process.thread_manager.spawn(self.user_vis_queue_monitor)
except:
log.exception('Unable to start VisUserQueueMonitor')
raise
def on_start(self):
try:
SimpleProcess.on_start(self)
self.viz_service = VisualizationServiceClient()
# Get config params
self.monitor_timeout = get_safe(self.CFG, 'user_queue_monitor_timeout', 300)
self.monitor_queue_size = get_safe(self.CFG, 'user_queue_monitor_size', 100)
self.monitor_event = gevent.event.Event()
self.monitor_event.clear()
self._process.thread_manager.spawn(self.user_vis_queue_monitor)
except:
log.exception('Unable to start VisUserQueueMonitor')
raise
def setUp(self):
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
self.resource_registry = self.container.resource_registry
self.RR2 = EnhancedResourceRegistryClient(self.resource_registry)
self.data_acquisition_management = DataAcquisitionManagementServiceClient()
self.pubsub_management = PubsubManagementServiceClient()
self.instrument_management = InstrumentManagementServiceClient()
self.data_product_management = DataProductManagementServiceClient()
self.dataset_management = DatasetManagementServiceClient()
self.process_dispatcher = ProcessDispatcherServiceClient()
self.data_process_management = DataProcessManagementServiceClient()
self.data_product_management = DataProductManagementServiceClient()
self.data_retriever = DataRetrieverServiceClient()
self.dataset_management = DatasetManagementServiceClient()
self.user_notification = UserNotificationServiceClient()
self.workflow_management = WorkflowManagementServiceClient()
self.visualization = VisualizationServiceClient()
def setUp(self):
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
self.resource_registry = self.container.resource_registry
self.RR2 = EnhancedResourceRegistryClient(self.resource_registry)
self.data_acquisition_management = DataAcquisitionManagementServiceClient()
self.pubsub_management = PubsubManagementServiceClient()
self.instrument_management = InstrumentManagementServiceClient()
self.discovery = DiscoveryServiceClient()
self.dataset_management = DatasetManagementServiceClient()
self.process_dispatcher = ProcessDispatcherServiceClient()
self.data_process_management = DataProcessManagementServiceClient()
self.data_product_management = DataProductManagementServiceClient()
self.data_retriever = DataRetrieverServiceClient()
self.dataset_management = DatasetManagementServiceClient()
self.user_notification = UserNotificationServiceClient()
self.observatory_management = ObservatoryManagementServiceClient()
self.visualization = VisualizationServiceClient()
self.ph = ParameterHelper(self.dataset_management, self.addCleanup)
self.ctd_count = 0
def setUp(self):
# Start container
logging.disable(logging.ERROR)
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
logging.disable(logging.NOTSET)
# Now create client to DataProductManagementService
self.rrclient = ResourceRegistryServiceClient(node=self.container.node)
self.damsclient = DataAcquisitionManagementServiceClient(node=self.container.node)
self.pubsubclient = PubsubManagementServiceClient(node=self.container.node)
self.ingestclient = IngestionManagementServiceClient(node=self.container.node)
self.imsclient = InstrumentManagementServiceClient(node=self.container.node)
self.dataproductclient = DataProductManagementServiceClient(node=self.container.node)
self.dataprocessclient = DataProcessManagementServiceClient(node=self.container.node)
self.datasetclient = DatasetManagementServiceClient(node=self.container.node)
self.workflowclient = WorkflowManagementServiceClient(node=self.container.node)
self.process_dispatcher = ProcessDispatcherServiceClient(node=self.container.node)
self.vis_client = VisualizationServiceClient(node=self.container.node)
self.ctd_stream_def = SBE37_CDM_stream_definition()
class TestVisualizationServiceIntegration(VisualizationIntegrationTestHelper):
def setUp(self):
# Start container
logging.disable(logging.ERROR)
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
logging.disable(logging.NOTSET)
# Now create client to DataProductManagementService
self.rrclient = ResourceRegistryServiceClient(node=self.container.node)
self.damsclient = DataAcquisitionManagementServiceClient(node=self.container.node)
self.pubsubclient = PubsubManagementServiceClient(node=self.container.node)
self.ingestclient = IngestionManagementServiceClient(node=self.container.node)
self.imsclient = InstrumentManagementServiceClient(node=self.container.node)
self.dataproductclient = DataProductManagementServiceClient(node=self.container.node)
self.dataprocessclient = DataProcessManagementServiceClient(node=self.container.node)
self.datasetclient = DatasetManagementServiceClient(node=self.container.node)
self.workflowclient = WorkflowManagementServiceClient(node=self.container.node)
self.process_dispatcher = ProcessDispatcherServiceClient(node=self.container.node)
self.vis_client = VisualizationServiceClient(node=self.container.node)
self.ctd_stream_def = SBE37_CDM_stream_definition()
def validate_messages(self, msgs):
cc = self.container
assertions = self.assertTrue
rdt = RecordDictionaryTool.load_from_granule(msgs.body)
vardict = {}
vardict['temp'] = get_safe(rdt, 'temp')
vardict['time'] = get_safe(rdt, 'time')
print vardict['time']
print vardict['temp']
@attr('LOCOINT')
#@patch.dict('pyon.ion.exchange.CFG', {'container':{'exchange':{'auto_register': False}}})
@unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False),'Not integrated for CEI')
@unittest.skip("in progress")
def test_visualization_queue(self):
assertions = self.assertTrue
#The list of data product streams to monitor
data_product_stream_ids = list()
#Create the input data product
ctd_stream_id, ctd_parsed_data_product_id = self.create_ctd_input_stream_and_data_product()
data_product_stream_ids.append(ctd_stream_id)
user_queue_name = 'user_queue'
xq = self.container.ex_manager.create_xn_queue(user_queue_name)
salinity_subscription_id = self.pubsubclient.create_subscription(
query=StreamQuery(data_product_stream_ids),
exchange_name = user_queue_name,
exchange_point = 'science_data',
name = "user visualization queue"
)
subscriber_registrar = StreamSubscriberRegistrar(container=self.container)
#subscriber = subscriber_registrar.create_subscriber(exchange_name=user_queue_name)
#subscriber.start()
#Using endpoint Subscriber directly; but should be a Stream-based subscriber that does nto require a process
#subscriber = Subscriber(from_name=(subscriber_registrar.XP, user_queue_name), callback=cb)
subscriber = Subscriber(from_name=xq)
subscriber.initialize()
# after the queue has been created it is safe to activate the subscription
self.pubsubclient.activate_subscription(subscription_id=salinity_subscription_id)
#Start the output stream listener to monitor and collect messages
#results = self.start_output_stream_and_listen(None, data_product_stream_ids)
#Not sure why this is needed - but it is
#subscriber._chan.stop_consume()
ctd_sim_pid = self.start_simple_input_stream_process(ctd_stream_id)
gevent.sleep(10.0) # Send some messages - don't care how many
msg_count,_ = xq.get_stats()
log.info('Messages in user queue 1: %s ' % msg_count)
#Validate the data from each of the messages along the way
#self.validate_messages(results)
# for x in range(msg_count):
# mo = subscriber.get_one_msg(timeout=1)
# print mo.body
# mo.ack()
msgs = subscriber.get_all_msgs(timeout=2)
for x in range(len(msgs)):
msgs[x].ack()
self.validate_messages(msgs[x])
# print msgs[x].body
#Should be zero after pulling all of the messages.
msg_count,_ = xq.get_stats()
log.info('Messages in user queue 2: %s ' % msg_count)
#Trying to continue to receive messages in the queue
gevent.sleep(5.0) # Send some messages - don't care how many
#Turning off after everything - since it is more representative of an always on stream of data!
self.process_dispatcher.cancel_process(ctd_sim_pid) # kill the ctd simulator process - that is enough data
#Should see more messages in the queue
msg_count,_ = xq.get_stats()
log.info('Messages in user queue 3: %s ' % msg_count)
msgs = subscriber.get_all_msgs(timeout=2)
for x in range(len(msgs)):
msgs[x].ack()
self.validate_messages(msgs[x])
#Should be zero after pulling all of the messages.
msg_count,_ = xq.get_stats()
log.info('Messages in user queue 4: %s ' % msg_count)
subscriber.close()
#@unittest.skip("in progress")
def test_realtime_visualization(self):
assertions = self.assertTrue
# Build the workflow definition
workflow_def_obj = IonObject(RT.WorkflowDefinition, name='GoogleDT_Test_Workflow',description='Tests the workflow of converting stream data to Google DT')
#Add a transformation process definition
google_dt_procdef_id = self.create_google_dt_data_process_definition()
workflow_step_obj = IonObject('DataProcessWorkflowStep', data_process_definition_id=google_dt_procdef_id, persist_process_output_data=True)
workflow_def_obj.workflow_steps.append(workflow_step_obj)
#Create it in the resource registry
workflow_def_id = self.workflowclient.create_workflow_definition(workflow_def_obj)
#Create the input data product
ctd_stream_id, ctd_parsed_data_product_id = self.create_ctd_input_stream_and_data_product()
#Create and start the workflow
workflow_id, workflow_product_id = self.workflowclient.create_data_process_workflow(workflow_def_id, ctd_parsed_data_product_id, timeout=20)
ctd_sim_pid = self.start_sinusoidal_input_stream_process(ctd_stream_id)
#TODO - Need to add workflow creation for google data table
vis_params ={}
vis_params['in_product_type'] = 'google_dt'
vis_token = self.vis_client.initiate_realtime_visualization(data_product_id=workflow_product_id, visualization_parameters=vis_params)
#Trying to continue to receive messages in the queue
gevent.sleep(10.0) # Send some messages - don't care how many
#TODO - find out what the actual return data type should be
vis_data = self.vis_client.get_realtime_visualization_data(vis_token)
print vis_data
#Trying to continue to receive messages in the queue
gevent.sleep(5.0) # Send some messages - don't care how many
#Turning off after everything - since it is more representative of an always on stream of data!
#todo remove the try except
try:
self.process_dispatcher.cancel_process(ctd_sim_pid) # kill the ctd simulator process - that is enough data
except:
log.warning("cancelling process did not work")
vis_data = self.vis_client.get_realtime_visualization_data(vis_token)
print vis_data
self.vis_client.terminate_realtime_visualization_data(vis_token)
#Stop the workflow processes
self.workflowclient.terminate_data_process_workflow(workflow_id, False) # Should test true at some point
#Cleanup to make sure delete is correct.
self.workflowclient.delete_workflow_definition(workflow_def_id)
def test_google_dt_overview_visualization(self):
#Create the input data product
ctd_stream_id, ctd_parsed_data_product_id = self.create_ctd_input_stream_and_data_product()
ctd_sim_pid = self.start_sinusoidal_input_stream_process(ctd_stream_id)
# Generate some data for a few seconds
gevent.sleep(5.0)
#Turning off after everything - since it is more representative of an always on stream of data!
self.process_dispatcher.cancel_process(ctd_sim_pid) # kill the ctd simulator process - that is enough data
# Use the data product to test the data retrieval and google dt generation capability of the vis service
vis_data = self.vis_client.get_visualization_data(ctd_parsed_data_product_id)
# validate the returned data
self.validate_vis_service_google_dt_results(vis_data)
def test_mpl_graphs_overview_visualization(self):
#Create the input data product
ctd_stream_id, ctd_parsed_data_product_id = self.create_ctd_input_stream_and_data_product()
ctd_sim_pid = self.start_sinusoidal_input_stream_process(ctd_stream_id)
# Generate some data for a few seconds
gevent.sleep(5.0)
#Turning off after everything - since it is more representative of an always on stream of data!
self.process_dispatcher.cancel_process(ctd_sim_pid) # kill the ctd simulator process - that is enough data
# Use the data product to test the data retrieval and google dt generation capability of the vis service
vis_data = self.vis_client.get_visualization_image(ctd_parsed_data_product_id)
# validate the returned data
self.validate_vis_service_mpl_graphs_results(vis_data)
return
class VisUserQueueMonitor(SimpleProcess):
def on_start(self):
try:
SimpleProcess.on_start(self)
self.viz_service = VisualizationServiceClient()
# Get config params
self.monitor_timeout = get_safe(self.CFG,
'user_queue_monitor_timeout', 300)
self.monitor_queue_size = get_safe(self.CFG,
'user_queue_monitor_size', 100)
self.monitor_event = gevent.event.Event()
self.monitor_event.clear()
self._process.thread_manager.spawn(self.user_vis_queue_monitor)
except:
log.exception('Unable to start VisUserQueueMonitor')
raise
def on_quit(self):
self.monitor_event.set()
super(VisUserQueueMonitor, self).on_quit()
def user_vis_queue_monitor(self, **kwargs):
log.debug("Starting Monitor Loop worker: %s timeout=%s", self.id,
self.monitor_timeout)
while not self.monitor_event.wait(timeout=self.monitor_timeout):
if self.container.is_terminating():
break
#get the list of queues and message counts on the broker for the user vis queues
queues = []
try:
queues = self.container.ex_manager.list_queues(
name=USER_VISUALIZATION_QUEUE,
return_columns=['name', 'messages'],
use_ems=False)
except Exception, e:
log.warn(
'Unable to get queue information from broker management plugin: '
+ e.message)
pass
log.debug("In Monitor Loop worker: %s", self.id)
for queue in queues:
log.debug('queue name: %s, messages: %d', queue['name'],
queue['messages'])
#Check for queues which are getting too large and clean them up if need be.
if queue['messages'] > self.monitor_queue_size:
vis_token = queue['name'][queue['name'].
index('UserVisQueue'):]
try:
log.warn(
"Real-time visualization queue %s had too many messages %d, so terminating this queue and associated resources.",
queue['name'], queue['messages'])
#Clear out the queue
#msgs = self.viz_service.get_realtime_visualization_data(query_token=vis_token)
#Now terminate it
self.viz_service.terminate_realtime_visualization_data(
query_token=vis_token)
except NotFound, e:
log.warn(
"The token %s could not not be found by the terminate_realtime_visualization_data operation; another worked may have cleaned it up already",
vis_token)
except Exception, e1:
#Log errors and keep going!
log.exception(e1)
class VisUserQueueMonitor(SimpleProcess):
def on_start(self):
try:
SimpleProcess.on_start(self)
self.viz_service = VisualizationServiceClient()
# Get config params
self.monitor_timeout = get_safe(self.CFG, 'user_queue_monitor_timeout', 300)
self.monitor_queue_size = get_safe(self.CFG, 'user_queue_monitor_size', 100)
self.monitor_event = gevent.event.Event()
self.monitor_event.clear()
self._process.thread_manager.spawn(self.user_vis_queue_monitor)
except:
log.exception('Unable to start VisUserQueueMonitor')
raise
def on_quit(self):
self.monitor_event.set()
super(VisUserQueueMonitor,self).on_quit()
def user_vis_queue_monitor(self, **kwargs):
log.debug("Starting Monitor Loop worker: %s timeout=%s" , self.id, self.monitor_timeout)
while not self.monitor_event.wait(timeout=self.monitor_timeout):
if self.container.is_terminating():
break
#get the list of queues and message counts on the broker for the user vis queues
queues = []
try:
queues = self.container.ex_manager.list_queues(name=USER_VISUALIZATION_QUEUE, return_columns=['name', 'messages'], use_ems=False)
except Exception, e:
log.warn('Unable to get queue information from broker management plugin: ' + e.message)
pass
log.debug( "In Monitor Loop worker: %s", self.id)
for queue in queues:
log.debug('queue name: %s, messages: %d', queue['name'], queue['messages'])
#Check for queues which are getting too large and clean them up if need be.
if queue['messages'] > self.monitor_queue_size:
vis_token = queue['name'][queue['name'].index('UserVisQueue'):]
try:
log.warn("Real-time visualization queue %s had too many messages %d, so terminating this queue and associated resources.", queue['name'], queue['messages'] )
#Clear out the queue
#msgs = self.viz_service.get_realtime_visualization_data(query_token=vis_token)
#Now terminate it
self.viz_service.terminate_realtime_visualization_data(query_token=vis_token)
except NotFound, e:
log.warn("The token %s could not not be found by the terminate_realtime_visualization_data operation; another worked may have cleaned it up already", vis_token)
except Exception, e1:
#Log errors and keep going!
log.exception(e1)
