def resolve_org_negotiation(): try: payload = request.form['payload'] json_params = simplejson.loads(str(payload)) ion_actor_id, expiry = get_governance_info_from_request( 'serviceRequest', json_params) ion_actor_id, expiry = validate_request(ion_actor_id, expiry) headers = build_message_headers(ion_actor_id, expiry) # extract negotiation-specific data (convert from unicode just in case - these are machine generated and unicode specific # chars are unexpected) verb = str(json_params['verb']) originator = str(json_params['originator']) negotiation_id = str(json_params['negotiation_id']) reason = str(json_params.get('reason', '')) proposal_status = None if verb.lower() == "accept": proposal_status = ProposalStatusEnum.ACCEPTED elif verb.lower() == "reject": proposal_status = ProposalStatusEnum.REJECTED proposal_originator = None if originator.lower() == "consumer": proposal_originator = ProposalOriginatorEnum.CONSUMER elif originator.lower() == "provider": proposal_originator = ProposalOriginatorEnum.PROVIDER rr_client = ResourceRegistryServiceProcessClient( node=Container.instance.node, process=service_gateway_instance) negotiation = rr_client.read(negotiation_id, headers=headers) new_negotiation_sap = Negotiation.create_counter_proposal( negotiation, proposal_status, proposal_originator) org_client = OrgManagementServiceProcessClient( node=Container.instance.node, process=service_gateway_instance) resp = org_client.negotiate(new_negotiation_sap, headers=headers) # update reason if it exists if reason: # reload negotiation because it has changed negotiation = rr_client.read(negotiation_id, headers=headers) negotiation.reason = reason rr_client.update(negotiation) return gateway_json_response(resp) except Exception, e: return build_error_response(e)
def on_start(self): self.query = self.CFG.get_safe('process.query',{}) self.delivery_format = self.CFG.get_safe('process.delivery_format',{}) self.datastore_name = self.CFG.get_safe('process.datastore_name','dm_datastore') definition_id = self.delivery_format.get('definition_id') rrsc = ResourceRegistryServiceProcessClient(process=self, node=self.container.node) definition = rrsc.read(definition_id) self.definition = definition.container self.fields = self.delivery_format.get('fields',None) self.view_name = self.CFG.get_safe('process.view_name','datasets/dataset_by_id') self.key_id = self.CFG.get_safe('process.key_id') self.stream_id = self.CFG.get_safe('process.publish_streams.output') if not self.stream_id: raise Inconsistent('The replay process requires a stream id. Invalid configuration!') self.data_stream_id = self.definition.data_stream_id self.encoding_id = self.definition.identifiables[self.data_stream_id].encoding_id self.element_type_id = self.definition.identifiables[self.data_stream_id].element_type_id self.element_count_id = self.definition.identifiables[self.data_stream_id].element_count_id self.data_record_id = self.definition.identifiables[self.element_type_id].data_record_id self.field_ids = self.definition.identifiables[self.data_record_id].field_ids self.domain_ids = self.definition.identifiables[self.data_record_id].domain_ids self.time_id = self.definition.identifiables[self.domain_ids[0]].temporal_coordinate_vector_id
def resolve_org_negotiation(): try: payload = request.form['payload'] json_params = json_loads(str(payload)) ion_actor_id, expiry = get_governance_info_from_request('serviceRequest', json_params) ion_actor_id, expiry = validate_request(ion_actor_id, expiry) headers = build_message_headers(ion_actor_id, expiry) # extract negotiation-specific data (convert from unicode just in case - these are machine generated and unicode specific # chars are unexpected) verb = str(json_params['verb']) originator = str(json_params['originator']) negotiation_id = str(json_params['negotiation_id']) reason = str(json_params.get('reason', '')) proposal_status = None if verb.lower() == "accept": proposal_status = ProposalStatusEnum.ACCEPTED elif verb.lower() == "reject": proposal_status = ProposalStatusEnum.REJECTED proposal_originator = None if originator.lower() == "consumer": proposal_originator = ProposalOriginatorEnum.CONSUMER elif originator.lower() == "provider": proposal_originator = ProposalOriginatorEnum.PROVIDER rr_client = ResourceRegistryServiceProcessClient(process=service_gateway_instance) negotiation = rr_client.read(negotiation_id, headers=headers) new_negotiation_sap = Negotiation.create_counter_proposal(negotiation, proposal_status, proposal_originator) org_client = OrgManagementServiceProcessClient(process=service_gateway_instance) resp = org_client.negotiate(new_negotiation_sap, headers=headers) # update reason if it exists if reason: # reload negotiation because it has changed negotiation = rr_client.read(negotiation_id, headers=headers) negotiation.reason = reason rr_client.update(negotiation) return gateway_json_response(resp) except Exception as e: return build_error_response(e)
def get_resource(resource_id): ret = None client = ResourceRegistryServiceProcessClient(node=Container.instance.node, process=service_gateway_instance) if resource_id != '': try: result = client.read(resource_id) if not result: raise NotFound("No resource found for id: %s " % resource_id) ret = simplejson.dumps(result, default=ion_object_encoder) except Exception, e: ret = "Error: %s" % e
def get_resource(resource_id): result = None client = ResourceRegistryServiceProcessClient(node=Container.instance.node, process=service_gateway_instance) if resource_id != '': try: #Database object IDs are not unicode result = client.read(convert_unicode(resource_id)) if not result: raise NotFound("No resource found for id: %s " % resource_id) return json_response({ GATEWAY_RESPONSE :result } ) except Exception, e: return build_error_response(e)
def get_resource(resource_id): try: client = ResourceRegistryServiceProcessClient(node=Container.instance.node, process=service_gateway_instance) # Validate requesting user and expiry and add governance headers ion_actor_id, expiry = get_governance_info_from_request() ion_actor_id, expiry = validate_request(ion_actor_id, expiry) # Database object IDs are not unicode result = client.read(convert_unicode(resource_id)) if not result: raise NotFound("No resource found for id: %s " % resource_id) return gateway_json_response(result) except Exception, e: return build_error_response(e)
def get_resource(resource_id): try: client = ResourceRegistryServiceProcessClient( node=Container.instance.node, process=service_gateway_instance) #Validate requesting user and expiry and add governance headers ion_actor_id, expiry = get_governance_info_from_request() ion_actor_id, expiry = validate_request(ion_actor_id, expiry) #Database object IDs are not unicode result = client.read(str(resource_id)) if not result: raise NotFound("No resource found for id: %s " % resource_id) return gateway_json_response(result) except Exception, e: return build_error_response(e)
def on_start(self): self.query = self.CFG.get_safe('process.query', {}) self.delivery_format = self.CFG.get_safe('process.delivery_format', {}) self.datastore_name = self.CFG.get_safe('process.datastore_name', 'dm_datastore') definition_id = self.delivery_format.get('definition_id') rrsc = ResourceRegistryServiceProcessClient(process=self, node=self.container.node) definition = rrsc.read(definition_id) self.definition = definition.container self.fields = self.delivery_format.get('fields', None) self.view_name = self.CFG.get_safe('process.view_name', 'datasets/dataset_by_id') self.key_id = self.CFG.get_safe('process.key_id') self.stream_id = self.CFG.get_safe('process.publish_streams.output') if not self.stream_id: raise Inconsistent( 'The replay process requires a stream id. Invalid configuration!' ) self.data_stream_id = self.definition.data_stream_id self.encoding_id = self.definition.identifiables[ self.data_stream_id].encoding_id self.element_type_id = self.definition.identifiables[ self.data_stream_id].element_type_id self.element_count_id = self.definition.identifiables[ self.data_stream_id].element_count_id self.data_record_id = self.definition.identifiables[ self.element_type_id].data_record_id self.field_ids = self.definition.identifiables[ self.data_record_id].field_ids self.domain_ids = self.definition.identifiables[ self.data_record_id].domain_ids self.time_id = self.definition.identifiables[ self.domain_ids[0]].temporal_coordinate_vector_id
class TestSchedulerService(IonIntegrationTestCase): def setUp(self): self.interval_timer_count = 0 self.interval_timer_sent_time = 0 self.interval_timer_received_time = 0 self.interval_timer_interval = 3 # Start container self._start_container() self.container.start_rel_from_url('res/deploy/r2deploy.yml') process = FakeProcess() self.ssclient = SchedulerServiceProcessClient(node=self.container.node, process=process) self.rrclient = ResourceRegistryServiceProcessClient(node=self.container.node, process=process) def tearDown(self): pass def now_utc(self): return time.mktime(datetime.datetime.utcnow().timetuple()) def test_create_interval_timer(self): # create the interval timer resource # create the event listener # call scheduler to set the timer # receive a few intervals, validate that arrival time is as expected # cancel the timer # wait until after next interval to verify that timer was correctly cancelled self.interval_timer_count = 0 self.interval_timer_sent_time = 0 self.interval_timer_received_time = 0 self.interval_timer_interval = 3 event_origin = "Interval_Timer_233" sub = EventSubscriber(event_type="TimerEvent", callback=self.interval_timer_callback, origin=event_origin) sub.start() self.addCleanup(sub.stop) start_time = self.now_utc() self.interval_timer_end_time = start_time + 10 id = self.ssclient.create_interval_timer(start_time="now" , interval=self.interval_timer_interval, end_time=self.interval_timer_end_time, event_origin=event_origin, event_subtype="") self.interval_timer_sent_time = datetime.datetime.utcnow() self.assertEqual(type(id), str) # Validate the timer is stored in RR ss = self.rrclient.read(id) self.assertEqual(ss.entry.event_origin, event_origin) # Wait until two events are published gevent.sleep((self.interval_timer_interval * 2) + 1) time_diff = (datetime.datetime.utcnow() - self.interval_timer_sent_time).seconds timer_counts = math.floor(time_diff/self.interval_timer_interval) #Cancle the timer ss = self.ssclient.cancel_timer(id) # wait until after next interval to verify that timer was correctly cancelled gevent.sleep(self.interval_timer_interval) # Validate the timer correctly cancelled with self.assertRaises(BadRequest): self.ssclient.cancel_timer(id) # Validate the timer is removed from resource regsitry with self.assertRaises(NotFound): self.rrclient.read(id) # Validate the number of timer counts self.assertEqual(self.interval_timer_count, timer_counts, "Invalid number of timeouts generated. Number of timeout: %d Expected timeout: %d Timer id: %s " %(self.interval_timer_count, timer_counts, id)) def test_system_restart(self): # create the interval timer resource # create the event listener # call scheduler to set the timer # receive a few intervals, validate that arrival time is as expected # cancel the timer # wait until after next interval to verify that timer was correctly cancelled self.interval_timer_count = 0 self.interval_timer_sent_time = 0 self.interval_timer_received_time = 0 self.interval_timer_interval = 3 event_origin = "Interval_Timer_4444" sub = EventSubscriber(event_type="TimerEvent", callback=self.on_restart_callback, origin=event_origin) sub.start() self.addCleanup(sub.stop) start_time = self.now_utc() self.interval_timer_end_time = start_time + 20 id = self.ssclient.create_interval_timer(start_time="now" , interval=self.interval_timer_interval, end_time=self.interval_timer_end_time, event_origin=event_origin, event_subtype="") self.interval_timer_sent_time = datetime.datetime.utcnow() self.assertEqual(type(id), str) # Validate the timer is stored in RR ss = self.rrclient.read(id) self.assertEqual(ss.entry.event_origin, event_origin) # Wait until 1 event is published gevent.sleep((self.interval_timer_interval) + 1) time_diff = (datetime.datetime.utcnow() - self.interval_timer_sent_time).seconds timer_counts = math.floor(time_diff/self.interval_timer_interval) # Validate the number of events generated self.assertEqual(self.interval_timer_count, timer_counts, "Invalid number of timeouts generated. Number of timeout: %d Expected timeout: %d Timer id: %s " %(self.interval_timer_count, timer_counts, id)) self.ssclient.on_system_restart() # after system restart, validate the timer is restored ss = self.rrclient.read(id) self.assertEqual(ss.entry.event_origin, event_origin) # Wait until another event is published start_time = datetime.datetime.utcnow() gevent.sleep((self.interval_timer_interval * 2) + 1) time_diff = (datetime.datetime.utcnow() - start_time).seconds timer_counts = math.floor(time_diff/self.interval_timer_interval) # Validate the number of events generated self.assertGreater(self.interval_timer_count, timer_counts) #Cancle the timer ss = self.ssclient.cancel_timer(id) # wait until after next interval to verify that timer was correctly cancelled gevent.sleep(self.interval_timer_interval) # Validate the timer correctly cancelled with self.assertRaises(BadRequest): self.ssclient.cancel_timer(id) # Validate the timer is removed from resource regsitry with self.assertRaises(NotFound): self.rrclient.read(id) def on_restart_callback(self, *args, **kwargs): self.interval_timer_count += 1 log.debug("test_scheduler: on_restart_callback: time: " + str(self.now_utc()) + " count: " + str(self.interval_timer_count)) def test_create_interval_timer_with_end_time(self): # create the interval timer resource # create the event listener # call scheduler to set the timer # receive a few intervals, validate that arrival time is as expected # Validate no more events are published after end_time expires # Validate the timer was canceled after the end_time expires self.interval_timer_count_2 = 0 self.interval_timer_sent_time_2 = 0 self.interval_timer_received_time_2 = 0 self.interval_timer_interval_2 = 3 event_origin = "Interval_Timer_2" sub = EventSubscriber(event_type="TimerEvent", callback=self.interval_timer_callback_with_end_time, origin=event_origin) sub.start() self.addCleanup(sub.stop) start_time = self.now_utc() self.interval_timer_end_time_2 = start_time + 7 id = self.ssclient.create_interval_timer(start_time="now" , interval=self.interval_timer_interval_2, end_time=self.interval_timer_end_time_2, event_origin=event_origin, event_subtype="") self.interval_timer_sent_time_2 = datetime.datetime.utcnow() self.assertEqual(type(id), str) # Wait until all events are published gevent.sleep((self.interval_timer_end_time_2 - start_time) + self.interval_timer_interval_2 + 1) # Validate the number of events generated self.assertEqual(self.interval_timer_count_2, 2, "Invalid number of timeouts generated. Number of event: %d Expected: 2 Timer id: %s " %(self.interval_timer_count_2, id)) # Validate the timer was canceled after the end_time is expired with self.assertRaises(BadRequest): self.ssclient.cancel_timer(id) def interval_timer_callback_with_end_time(self, *args, **kwargs): self.interval_timer_received_time_2 = datetime.datetime.utcnow() self.interval_timer_count_2 += 1 time_diff = math.fabs( ((self.interval_timer_received_time_2 - self.interval_timer_sent_time_2).total_seconds()) - (self.interval_timer_interval_2 * self.interval_timer_count_2) ) # Assert expire time is within +-10 seconds self.assertTrue(time_diff <= 10) log.debug("test_scheduler: interval_timer_callback_with_end_time: time:" + str(self.interval_timer_received_time_2) + " count:" + str(self.interval_timer_count_2)) def interval_timer_callback(self, *args, **kwargs): self.interval_timer_received_time = datetime.datetime.utcnow() self.interval_timer_count += 1 time_diff = math.fabs( ((self.interval_timer_received_time - self.interval_timer_sent_time).total_seconds()) - (self.interval_timer_interval * self.interval_timer_count) ) # Assert expire time is within +-10 seconds self.assertTrue(time_diff <= 10) log.debug("test_scheduler: interval_timer_callback: time:" + str(self.interval_timer_received_time) + " count:" + str(self.interval_timer_count)) def test_cancel_single_timer(self): # test creating a new timer that is one-time-only # create the timer resource # create the event listener # call scheduler to set the timer # create then cancel the timer, verify that event is not received # create the timer resource # create the event listener # call scheduler to set the timer # call scheduler to cancel the timer # wait until after expiry to verify that event is not sent self.single_timer_count = 0 event_origin = "Time_of_Day" sub = EventSubscriber(event_type="TimerEvent", callback=self.single_timer_callback, origin=event_origin) sub.start() self.addCleanup(sub.stop) now = datetime.datetime.utcnow() + timedelta(seconds=3) times_of_day =[{'hour': str(now.hour),'minute' : str(now.minute), 'second':str(now.second) }] id = self.ssclient.create_time_of_day_timer(times_of_day=times_of_day, expires=self.now_utc()+3, event_origin=event_origin, event_subtype="test") self.assertEqual(type(id), str) self.ssclient.cancel_timer(id) gevent.sleep(3) # Validate the event is not generated self.assertEqual(self.single_timer_count, 0, "Invalid number of timeouts generated. Number of timeout: %d Expected timeout: 0 Timer id: %s " %(self.single_timer_count, id)) def single_timer_callback (self, *args, **kwargs): self.single_timer_count =+ 1 log.debug("test_scheduler: single_timer_call_back: time:" + str(self.now_utc()) + " count:" + str(self.single_timer_count)) def test_create_forever_interval_timer(self): # Test creating interval timer that runs forever self.interval_timer_count = 0 self.interval_timer_sent_time = 0 self.interval_timer_received_time = 0 self.interval_timer_interval = 3 event_origin = "Interval Timer Forever" sub = EventSubscriber(event_type="TimerEvent", callback=self.interval_timer_callback, origin=event_origin) sub.start() self.addCleanup(sub.stop) id = self.ssclient.create_interval_timer(start_time= self.now_utc(), interval=self.interval_timer_interval, end_time=-1, event_origin=event_origin, event_subtype=event_origin) self.interval_timer_sent_time = datetime.datetime.utcnow() self.assertEqual(type(id), str) # Wait for 4 events to be published gevent.sleep((self.interval_timer_interval * 4) + 1) self.ssclient.cancel_timer(id) time_diff = (datetime.datetime.utcnow() - self.interval_timer_sent_time).seconds timer_counts = math.floor(time_diff/self.interval_timer_interval) # Validate the timer id is invalid once it has been canceled with self.assertRaises(BadRequest): self.ssclient.cancel_timer(id) # Validate events are not generated after canceling the timer self.assertEqual(self.interval_timer_count, timer_counts, "Invalid number of timeouts generated. Number of timeout: %d Expected timeout: %d Timer id: %s " %(self.interval_timer_count, timer_counts, id)) def test_timeoffday_timer(self): # test creating a new timer that is one-time-only # create the timer resource # get the current time, set the timer to several seconds from current time # create the event listener # call scheduler to set the timer # verify that event arrival is within one/two seconds of current time event_origin = "Time Of Day2" self.expire_sec_1 = 4 self.expire_sec_2 = 5 self.tod_count = 0 expire1 = datetime.datetime.utcnow() + timedelta(seconds=self.expire_sec_1) expire2 = datetime.datetime.utcnow() + timedelta(seconds=self.expire_sec_2) # Create two timers times_of_day =[{'hour': str(expire1.hour),'minute' : str(expire1.minute), 'second':str(expire1.second) }, {'hour': str(expire2.hour),'minute' : str(expire2.minute), 'second':str(expire2.second)}] sub = EventSubscriber(event_type="TimerEvent", callback=self.tod_callback, origin=event_origin) sub.start() self.addCleanup(sub.stop) # Expires in one days expires = time.mktime((datetime.datetime.utcnow() + timedelta(days=2)).timetuple()) self.tod_sent_time = datetime.datetime.utcnow() id = self.ssclient.create_time_of_day_timer(times_of_day=times_of_day, expires=expires, event_origin=event_origin, event_subtype="") self.interval_timer_sent_time = datetime.datetime.utcnow() self.assertEqual(type(id), str) # Wait until all events are generated gevent.sleep(9) time_diff = (datetime.datetime.utcnow() - self.interval_timer_sent_time).seconds timer_counts = math.floor(time_diff/self.expire_sec_1) + math.floor(time_diff/self.expire_sec_2) # After waiting, validate only 2 events are generated. self.assertEqual(self.tod_count, 2, "Invalid number of timeouts generated. Number of timeout: %d Expected timeout: %d Timer id: %s " %(self.tod_count, timer_counts, id)) # Cancel the timer self.ssclient.cancel_timer(id) def tod_callback(self, *args, **kwargs): tod_receive_time = datetime.datetime.utcnow() self.tod_count += 1 if self.tod_count == 1: time_diff = math.fabs((tod_receive_time - self.tod_sent_time).total_seconds() - self.expire_sec_1) self.assertTrue(time_diff <= 2) elif self.tod_count == 2: time_diff = math.fabs((tod_receive_time - self.tod_sent_time).total_seconds() - self.expire_sec_2) self.assertTrue(time_diff <= 2) log.debug("test_scheduler: tod_callback: time:" + str(tod_receive_time) + " count:" + str(self.tod_count)) @unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False), 'Skip test while in CEI LAUNCH mode') def test_quit_stops_timers(self): ar = AsyncResult() def cb(*args, **kwargs): ar.set(args) self.interval_timer_count += 1 event_origin = "test_quitter" sub = EventSubscriber(event_type="TimerEvent", callback=cb, origin=event_origin) sub.start() self.addCleanup(sub.stop) tid = self.ssclient.create_interval_timer(start_time="now", interval=1, event_origin=event_origin) # wait until at least one scheduled message ar.get(timeout=5) # shut it down! p = self.container.proc_manager.procs_by_name['scheduler'] self.container.terminate_process(p.id) # assert empty self.assertEquals(p.schedule_entries, {})
class TestWorkflowManagementIntegration(VisualizationIntegrationTestHelper): def setUp(self): # Start container self._start_container() self.container.start_rel_from_url('res/deploy/r2deploy.yml') #Instantiate a process to represent the test process=WorkflowServiceTestProcess() # Now create client to DataProductManagementService self.rrclient = ResourceRegistryServiceProcessClient(node=self.container.node, process=process) self.damsclient = DataAcquisitionManagementServiceProcessClient(node=self.container.node, process=process) self.pubsubclient = PubsubManagementServiceProcessClient(node=self.container.node, process=process) self.ingestclient = IngestionManagementServiceProcessClient(node=self.container.node, process=process) self.imsclient = InstrumentManagementServiceProcessClient(node=self.container.node, process=process) self.dataproductclient = DataProductManagementServiceProcessClient(node=self.container.node, process=process) self.dataprocessclient = DataProcessManagementServiceProcessClient(node=self.container.node, process=process) self.datasetclient = DatasetManagementServiceProcessClient(node=self.container.node, process=process) self.workflowclient = WorkflowManagementServiceProcessClient(node=self.container.node, process=process) self.process_dispatcher = ProcessDispatcherServiceProcessClient(node=self.container.node, process=process) self.data_retriever = DataRetrieverServiceProcessClient(node=self.container.node, process=process) self.ctd_stream_def = SBE37_CDM_stream_definition() @attr('LOCOINT') @unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False),'Not integrated for CEI') def test_SA_transform_components(self): assertions = self.assertTrue #The list of data product streams to monitor data_product_stream_ids = list() #Create the input data product ctd_stream_id, ctd_parsed_data_product_id = self.create_ctd_input_stream_and_data_product() data_product_stream_ids.append(ctd_stream_id) ### ### Setup the first transformation ### # Salinity: Data Process Definition ctd_L2_salinity_dprocdef_id = self.create_salinity_data_process_definition() l2_salinity_all_data_process_id, ctd_l2_salinity_output_dp_id = self.create_transform_process(ctd_L2_salinity_dprocdef_id,ctd_parsed_data_product_id, 'salinity' ) ## get the stream id for the transform outputs stream_ids, _ = self.rrclient.find_objects(ctd_l2_salinity_output_dp_id, PRED.hasStream, None, True) assertions(len(stream_ids) > 0 ) sal_stream_id = stream_ids[0] data_product_stream_ids.append(sal_stream_id) ### ### Setup the second transformation ### # Salinity Doubler: Data Process Definition salinity_doubler_dprocdef_id = self.create_salinity_doubler_data_process_definition() salinity_double_data_process_id, salinity_doubler_output_dp_id = self.create_transform_process(salinity_doubler_dprocdef_id, ctd_l2_salinity_output_dp_id, 'salinity' ) stream_ids, _ = self.rrclient.find_objects(salinity_doubler_output_dp_id, PRED.hasStream, None, True) assertions(len(stream_ids) > 0 ) sal_dbl_stream_id = stream_ids[0] data_product_stream_ids.append(sal_dbl_stream_id) #Start the output stream listener to monitor and collect messages results = self.start_output_stream_and_listen(ctd_stream_id, data_product_stream_ids) #Stop the transform processes self.dataprocessclient.deactivate_data_process(salinity_double_data_process_id) self.dataprocessclient.deactivate_data_process(l2_salinity_all_data_process_id) #Validate the data from each of the messages along the way self.validate_messages(results) @attr('LOCOINT') @attr('SMOKE') @unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False),'Not integrated for CEI') def test_transform_workflow(self): assertions = self.assertTrue log.debug("Building the workflow definition") workflow_def_obj = IonObject(RT.WorkflowDefinition, name='Salinity_Test_Workflow', description='tests a workflow of multiple transform data processes') workflow_data_product_name = 'TEST-Workflow_Output_Product' #Set a specific output product name #------------------------------------------------------------------------------------------------------------------------- log.debug( "Adding a transformation process definition for salinity") #------------------------------------------------------------------------------------------------------------------------- ctd_L2_salinity_dprocdef_id = self.create_salinity_data_process_definition() workflow_step_obj = IonObject('DataProcessWorkflowStep', data_process_definition_id=ctd_L2_salinity_dprocdef_id, persist_process_output_data=False) #Don't persist the intermediate data product workflow_def_obj.workflow_steps.append(workflow_step_obj) #------------------------------------------------------------------------------------------------------------------------- log.debug( "Adding a transformation process definition for salinity doubler") #------------------------------------------------------------------------------------------------------------------------- salinity_doubler_dprocdef_id = self.create_salinity_doubler_data_process_definition() workflow_step_obj = IonObject('DataProcessWorkflowStep', data_process_definition_id=salinity_doubler_dprocdef_id, ) workflow_def_obj.workflow_steps.append(workflow_step_obj) log.debug( "Creating workflow def in the resource registry") workflow_def_id = self.workflowclient.create_workflow_definition(workflow_def_obj) aids = self.rrclient.find_associations(workflow_def_id, PRED.hasDataProcessDefinition) assertions(len(aids) == 2 ) #The list of data product streams to monitor data_product_stream_ids = list() log.debug( "Creating the input data product") ctd_stream_id, ctd_parsed_data_product_id = self.create_ctd_input_stream_and_data_product() data_product_stream_ids.append(ctd_stream_id) log.debug( "Creating and starting the workflow") workflow_id, workflow_product_id = self.workflowclient.create_data_process_workflow(workflow_def_id, ctd_parsed_data_product_id, persist_workflow_data_product=True, output_data_product_name=workflow_data_product_name, timeout=300) workflow_output_ids,_ = self.rrclient.find_subjects(RT.Workflow, PRED.hasOutputProduct, workflow_product_id, True) assertions(len(workflow_output_ids) == 1 ) log.debug( "persisting the output product") #self.dataproductclient.activate_data_product_persistence(workflow_product_id) dataset_ids,_ = self.rrclient.find_objects(workflow_product_id, PRED.hasDataset, RT.Dataset, True) assertions(len(dataset_ids) == 1 ) dataset_id = dataset_ids[0] log.debug( "Verifying the output data product name matches what was specified in the workflow definition") workflow_product = self.rrclient.read(workflow_product_id) assertions(workflow_product.name.startswith(workflow_data_product_name), 'Nope: %s != %s' % (workflow_product.name, workflow_data_product_name)) log.debug( "Walking the associations to find the appropriate output data streams to validate the messages") workflow_dp_ids,_ = self.rrclient.find_objects(workflow_id, PRED.hasDataProduct, RT.DataProduct, True) assertions(len(workflow_dp_ids) == 2 ) for dp_id in workflow_dp_ids: stream_ids, _ = self.rrclient.find_objects(dp_id, PRED.hasStream, None, True) assertions(len(stream_ids) == 1 ) data_product_stream_ids.append(stream_ids[0]) log.debug( "data_product_stream_ids: %s" % data_product_stream_ids) log.debug( "Starting the output stream listener to monitor to collect messages") results = self.start_output_stream_and_listen(ctd_stream_id, data_product_stream_ids) log.debug( "results::: %s" % results) log.debug( "Stopping the workflow processes") self.workflowclient.terminate_data_process_workflow(workflow_id, False, timeout=250) # Should test true at some point log.debug( "Making sure the Workflow object was removed") objs, _ = self.rrclient.find_resources(restype=RT.Workflow) assertions(len(objs) == 0) log.debug( "Validating the data from each of the messages along the way") self.validate_messages(results) log.debug( "Checking to see if dataset id = %s, was persisted, and that it can be retrieved...." % dataset_id) self.validate_data_ingest_retrieve(dataset_id) log.debug( "Cleaning up to make sure delete is correct.") self.workflowclient.delete_workflow_definition(workflow_def_id) workflow_def_ids,_ = self.rrclient.find_resources(restype=RT.WorkflowDefinition) assertions(len(workflow_def_ids) == 0 ) @attr('LOCOINT') @unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False),'Not integrated for CEI') def test_google_dt_transform_workflow(self): assertions = self.assertTrue # Build the workflow definition workflow_def_obj = IonObject(RT.WorkflowDefinition, name='GoogleDT_Test_Workflow',description='Tests the workflow of converting stream data to Google DT') #Add a transformation process definition google_dt_procdef_id = self.create_google_dt_data_process_definition() workflow_step_obj = IonObject('DataProcessWorkflowStep', data_process_definition_id=google_dt_procdef_id, persist_process_output_data=False) workflow_def_obj.workflow_steps.append(workflow_step_obj) #Create it in the resource registry workflow_def_id = self.workflowclient.create_workflow_definition(workflow_def_obj) #The list of data product streams to monitor data_product_stream_ids = list() #Create the input data product ctd_stream_id, ctd_parsed_data_product_id = self.create_ctd_input_stream_and_data_product() data_product_stream_ids.append(ctd_stream_id) #Create and start the workflow workflow_id, workflow_product_id = self.workflowclient.create_data_process_workflow(workflow_def_id, ctd_parsed_data_product_id, timeout=60) workflow_output_ids,_ = self.rrclient.find_subjects(RT.Workflow, PRED.hasOutputProduct, workflow_product_id, True) assertions(len(workflow_output_ids) == 1 ) #Walk the associations to find the appropriate output data streams to validate the messages workflow_dp_ids,_ = self.rrclient.find_objects(workflow_id, PRED.hasDataProduct, RT.DataProduct, True) assertions(len(workflow_dp_ids) == 1 ) for dp_id in workflow_dp_ids: stream_ids, _ = self.rrclient.find_objects(dp_id, PRED.hasStream, None, True) assertions(len(stream_ids) == 1 ) data_product_stream_ids.append(stream_ids[0]) #Start the output stream listener to monitor and collect messages results = self.start_output_stream_and_listen(ctd_stream_id, data_product_stream_ids) #Stop the workflow processes self.workflowclient.terminate_data_process_workflow(workflow_id=workflow_id,delete_data_products=False, timeout=60) # Should test true at some point #Validate the data from each of the messages along the way self.validate_google_dt_transform_results(results) """ # Check to see if ingestion worked. Extract the granules from data_retrieval. # First find the dataset associated with the output dp product ds_ids,_ = self.rrclient.find_objects(workflow_dp_ids[len(workflow_dp_ids) - 1], PRED.hasDataset, RT.Dataset, True) retrieved_granule = self.data_retriever.retrieve(ds_ids[0]) #Validate the data from each of the messages along the way self.validate_google_dt_transform_results(retrieved_granule) """ #Cleanup to make sure delete is correct. self.workflowclient.delete_workflow_definition(workflow_def_id) workflow_def_ids,_ = self.rrclient.find_resources(restype=RT.WorkflowDefinition) assertions(len(workflow_def_ids) == 0 ) @attr('LOCOINT') @unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False),'Not integrated for CEI') def test_mpl_graphs_transform_workflow(self): assertions = self.assertTrue # Build the workflow definition workflow_def_obj = IonObject(RT.WorkflowDefinition, name='Mpl_Graphs_Test_Workflow',description='Tests the workflow of converting stream data to Matplotlib graphs') #Add a transformation process definition mpl_graphs_procdef_id = self.create_mpl_graphs_data_process_definition() workflow_step_obj = IonObject('DataProcessWorkflowStep', data_process_definition_id=mpl_graphs_procdef_id, persist_process_output_data=False) workflow_def_obj.workflow_steps.append(workflow_step_obj) #Create it in the resource registry workflow_def_id = self.workflowclient.create_workflow_definition(workflow_def_obj) #The list of data product streams to monitor data_product_stream_ids = list() #Create the input data product ctd_stream_id, ctd_parsed_data_product_id = self.create_ctd_input_stream_and_data_product() data_product_stream_ids.append(ctd_stream_id) #Create and start the workflow workflow_id, workflow_product_id = self.workflowclient.create_data_process_workflow(workflow_def_id, ctd_parsed_data_product_id, persist_workflow_data_product=True, timeout=60) workflow_output_ids,_ = self.rrclient.find_subjects(RT.Workflow, PRED.hasOutputProduct, workflow_product_id, True) assertions(len(workflow_output_ids) == 1 ) #Walk the associations to find the appropriate output data streams to validate the messages workflow_dp_ids,_ = self.rrclient.find_objects(workflow_id, PRED.hasDataProduct, RT.DataProduct, True) assertions(len(workflow_dp_ids) == 1 ) for dp_id in workflow_dp_ids: stream_ids, _ = self.rrclient.find_objects(dp_id, PRED.hasStream, None, True) assertions(len(stream_ids) == 1 ) data_product_stream_ids.append(stream_ids[0]) #Start the output stream listener to monitor and collect messages results = self.start_output_stream_and_listen(ctd_stream_id, data_product_stream_ids) #Stop the workflow processes self.workflowclient.terminate_data_process_workflow(workflow_id=workflow_id,delete_data_products=False, timeout=60) # Should test true at some point #Validate the data from each of the messages along the way self.validate_mpl_graphs_transform_results(results) # Check to see if ingestion worked. Extract the granules from data_retrieval. # First find the dataset associated with the output dp product ds_ids,_ = self.rrclient.find_objects(workflow_dp_ids[len(workflow_dp_ids) - 1], PRED.hasDataset, RT.Dataset, True) retrieved_granule = self.data_retriever.retrieve_last_data_points(ds_ids[0], 10) #Validate the data from each of the messages along the way self.validate_mpl_graphs_transform_results(retrieved_granule) #Cleanup to make sure delete is correct. self.workflowclient.delete_workflow_definition(workflow_def_id) workflow_def_ids,_ = self.rrclient.find_resources(restype=RT.WorkflowDefinition) assertions(len(workflow_def_ids) == 0 ) @attr('LOCOINT') @unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False),'Not integrated for CEI') def test_multiple_workflow_instances(self): assertions = self.assertTrue # Build the workflow definition workflow_def_obj = IonObject(RT.WorkflowDefinition, name='Multiple_Test_Workflow',description='Tests the workflow of converting stream data') #Add a transformation process definition google_dt_procdef_id = self.create_google_dt_data_process_definition() workflow_step_obj = IonObject('DataProcessWorkflowStep', data_process_definition_id=google_dt_procdef_id, persist_process_output_data=False) workflow_def_obj.workflow_steps.append(workflow_step_obj) #Create it in the resource registry workflow_def_id = self.workflowclient.create_workflow_definition(workflow_def_obj) #The list of data product streams to monitor data_product_stream_ids = list() #Create the first input data product ctd_stream_id1, ctd_parsed_data_product_id1 = self.create_ctd_input_stream_and_data_product('ctd_parsed1') data_product_stream_ids.append(ctd_stream_id1) #Create and start the first workflow workflow_id1, workflow_product_id1 = self.workflowclient.create_data_process_workflow(workflow_def_id, ctd_parsed_data_product_id1, timeout=60) #Create the second input data product ctd_stream_id2, ctd_parsed_data_product_id2 = self.create_ctd_input_stream_and_data_product('ctd_parsed2') data_product_stream_ids.append(ctd_stream_id2) #Create and start the second workflow workflow_id2, workflow_product_id2 = self.workflowclient.create_data_process_workflow(workflow_def_id, ctd_parsed_data_product_id2, timeout=60) #Walk the associations to find the appropriate output data streams to validate the messages workflow_ids,_ = self.rrclient.find_resources(restype=RT.Workflow) assertions(len(workflow_ids) == 2 ) #Start the first input stream process ctd_sim_pid1 = self.start_sinusoidal_input_stream_process(ctd_stream_id1) #Start the second input stream process ctd_sim_pid2 = self.start_simple_input_stream_process(ctd_stream_id2) #Start the output stream listener to monitor a set number of messages being sent through the workflows results = self.start_output_stream_and_listen(None, data_product_stream_ids, message_count_per_stream=5) # stop the flow of messages... self.process_dispatcher.cancel_process(ctd_sim_pid1) # kill the ctd simulator process - that is enough data self.process_dispatcher.cancel_process(ctd_sim_pid2) #Stop the first workflow processes self.workflowclient.terminate_data_process_workflow(workflow_id=workflow_id1,delete_data_products=False, timeout=60) # Should test true at some point #Stop the second workflow processes self.workflowclient.terminate_data_process_workflow(workflow_id=workflow_id2,delete_data_products=False, timeout=60) # Should test true at some point workflow_ids,_ = self.rrclient.find_resources(restype=RT.Workflow) assertions(len(workflow_ids) == 0 ) #Cleanup to make sure delete is correct. self.workflowclient.delete_workflow_definition(workflow_def_id) workflow_def_ids,_ = self.rrclient.find_resources(restype=RT.WorkflowDefinition) assertions(len(workflow_def_ids) == 0 ) aid_list = self.rrclient.find_associations(workflow_def_id, PRED.hasDataProcessDefinition) assertions(len(aid_list) == 0 )
class VizTransformProcForMatplotlibGraphs(TransformDataProcess): """ This class is used for instantiating worker processes that have subscriptions to data streams and convert incoming data from CDM format to Matplotlib graphs """ def on_start(self): super(VizTransformProcForMatplotlibGraphs, self).on_start() #assert len(self.streams)==1 self.initDataFlag = True self.graph_data = { } # Stores a dictionary of variables : [List of values] # Need some clients self.rr_cli = ResourceRegistryServiceProcessClient( process=self, node=self.container.node) self.pubsub_cli = PubsubManagementServiceClient( node=self.container.node) # extract the various parameters passed to the transform process self.out_stream_id = self.CFG.get('process').get( 'publish_streams').get('visualization_service_submit_stream_id') # Create a publisher on the output stream #stream_route = self.pubsub_cli.register_producer(stream_id=self.out_stream_id) out_stream_pub_registrar = StreamPublisherRegistrar( process=self.container, node=self.container.node) self.out_stream_pub = out_stream_pub_registrar.create_publisher( stream_id=self.out_stream_id) self.data_product_id = self.CFG.get('data_product_id') self.stream_def_id = self.CFG.get("stream_def_id") self.stream_def = self.rr_cli.read(self.stream_def_id) # Start the thread responsible for keeping track of time and generating graphs # Mutex for ensuring proper concurrent communications between threads self.lock = RLock() self.rendering_proc = Greenlet(self.rendering_thread) self.rendering_proc.start() def process(self, packet): log.debug('(%s): Received Viz Data Packet' % self.name) #log.debug('(%s): - Processing: %s' % (self.name,packet)) # parse the incoming data psd = PointSupplementStreamParser( stream_definition=self.stream_def.container, stream_granule=packet) # re-arrange incoming data into an easy to parse dictionary vardict = {} arrLen = None for varname in psd.list_field_names(): vardict[varname] = psd.get_values(varname) arrLen = len(vardict[varname]) if self.initDataFlag: # look at the incoming packet and store for varname in psd.list_field_names(): self.lock.acquire() self.graph_data[varname] = [] self.lock.release() self.initDataFlag = False # If code reached here, the graph data storage has been initialized. Just add values # to the list with self.lock: for varname in psd.list_field_names(): self.graph_data[varname].extend(vardict[varname]) def rendering_thread(self): from copy import deepcopy # Service Client # init Matplotlib fig = Figure() ax = fig.add_subplot(111) canvas = FigureCanvas(fig) imgInMem = StringIO.StringIO() while True: # Sleep for a pre-decided interval. Should be specifiable in a YAML file gevent.sleep(20) # If there's no data, wait # Lock is used here to make sure the entire vector exists start to finish, this assures that the data won working_set = None with self.lock: if len(self.graph_data) == 0: continue else: working_set = deepcopy(self.graph_data) # For the simple case of testing, lets plot all time variant variables one at a time xAxisVar = 'time' xAxisFloatData = working_set[xAxisVar] for varName, varData in working_set.iteritems(): if varName == 'time' or varName == 'height' or varName == 'longitude' or varName == 'latitude': continue yAxisVar = varName yAxisFloatData = working_set[varName] # Generate the plot ax.plot(xAxisFloatData, yAxisFloatData, 'ro') ax.set_xlabel(xAxisVar) ax.set_ylabel(yAxisVar) ax.set_title(yAxisVar + ' vs ' + xAxisVar) ax.set_autoscale_on(False) # generate filename for the output image fileName = yAxisVar + '_vs_' + xAxisVar + '.png' # Save the figure to the in memory file canvas.print_figure(imgInMem, format="png") imgInMem.seek(0) # submit resulting table back using the out stream publisher msg = { "viz_product_type": "matplotlib_graphs", "data_product_id": self.data_product_id, "image_obj": imgInMem.getvalue(), "image_name": fileName } self.out_stream_pub.publish(msg) #clear the canvas for the next image ax.clear()
class VizTransformProcForGoogleDT(TransformDataProcess): """ This class is used for instantiating worker processes that have subscriptions to data streams and convert incoming data from CDM format to JSON style Google DataTables """ def on_start(self): super(VizTransformProcForGoogleDT, self).on_start() self.initDataTableFlag = True # need some clients self.rr_cli = ResourceRegistryServiceProcessClient( process=self, node=self.container.node) self.pubsub_cli = PubsubManagementServiceClient( node=self.container.node) # extract the various parameters passed self.out_stream_id = self.CFG.get('process').get( 'publish_streams').get('visualization_service_submit_stream_id') # Create a publisher on the output stream out_stream_pub_registrar = StreamPublisherRegistrar( process=self.container, node=self.container.node) self.out_stream_pub = out_stream_pub_registrar.create_publisher( stream_id=self.out_stream_id) self.data_product_id = self.CFG.get('data_product_id') self.stream_def_id = self.CFG.get("stream_def_id") stream_def_resource = self.rr_cli.read(self.stream_def_id) self.stream_def = stream_def_resource.container self.realtime_flag = False if self.CFG.get("realtime_flag") == "True": self.realtime_flag = True else: self.data_product_id_token = self.CFG.get('data_product_id_token') # extract the stream_id associated with the DP. Needed later stream_ids, _ = self.rr_cli.find_objects(self.data_product_id, PRED.hasStream, None, True) self.stream_id = stream_ids[0] self.dataDescription = [] self.dataTableContent = [] self.varTuple = [] self.total_num_of_records_recvd = 0 def process(self, packet): log.debug('(%s): Received Viz Data Packet' % (self.name)) element_count_id = 0 expected_range = [] psd = PointSupplementStreamParser(stream_definition=self.stream_def, stream_granule=packet) vardict = {} arrLen = None for varname in psd.list_field_names(): vardict[varname] = psd.get_values(varname) arrLen = len(vardict[varname]) #if its the first time, init the dataTable if self.initDataTableFlag: # create data description from the variables in the message self.dataDescription = [('time', 'datetime', 'time')] # split the data string to extract variable names for varname in psd.list_field_names(): if varname == 'time': continue self.dataDescription.append((varname, 'number', varname)) self.initDataTableFlag = False # Add the records to the datatable for i in xrange(arrLen): varTuple = [] for varname, _, _ in self.dataDescription: val = float(vardict[varname][i]) if varname == 'time': varTuple.append(datetime.fromtimestamp(val)) else: varTuple.append(val) # Append the tuples to the data table self.dataTableContent.append(varTuple) if self.realtime_flag: # Maintain a sliding window for realtime transform processes realtime_window_size = 100 if len(self.dataTableContent) > realtime_window_size: # always pop the first element till window size is what we want while len(self.dataTableContent) > realtime_window_size: self.dataTableContent.pop(0) if not self.realtime_flag: # This is the historical view part. Make a note of now many records were received data_stream_id = self.stream_def.data_stream_id element_count_id = self.stream_def.identifiables[ data_stream_id].element_count_id # From each granule you can check the constraint on the number of records expected_range = packet.identifiables[ element_count_id].constraint.intervals[0] # The number of records in a given packet is: self.total_num_of_records_recvd += packet.identifiables[ element_count_id].value # submit the Json version of the datatable to the viz service if self.realtime_flag: # create the google viz data table data_table = gviz_api.DataTable(self.dataDescription) data_table.LoadData(self.dataTableContent) # submit resulting table back using the out stream publisher msg = { "viz_product_type": "google_realtime_dt", "data_product_id": self.data_product_id, "data_table": data_table.ToJSonResponse() } self.out_stream_pub.publish(msg) else: # Submit table back to the service if we received all the replay data if self.total_num_of_records_recvd == (expected_range[1] + 1): # If the datatable received was too big, decimate on the fly to a fixed size max_google_dt_len = 1024 if len(self.dataTableContent) > max_google_dt_len: decimation_factor = int( math.ceil( len(self.dataTableContent) / (max_google_dt_len))) tempDataTableContent = [] for i in xrange(0, len(self.dataTableContent), decimation_factor): # check limits if i >= len(self.dataTableContent): break tempDataTableContent.append(self.dataTableContent[i]) self.dataTableContent = tempDataTableContent data_table = gviz_api.DataTable(self.dataDescription) data_table.LoadData(self.dataTableContent) # submit resulting table back using the out stream publisher msg = { "viz_product_type": "google_dt", "data_product_id_token": self.data_product_id_token, "data_table": data_table.ToJSonResponse() } self.out_stream_pub.publish(msg) return # clear the tuple for future use self.varTuple[:] = []
class TestSchedulerService(IonIntegrationTestCase): def setUp(self): self.interval_timer_count = 0 self.interval_timer_sent_time = 0 self.interval_timer_received_time = 0 self.interval_timer_interval = 3 # Start container self._start_container() self.container.start_rel_from_url('res/deploy/r2deploy.yml') process = FakeProcess() self.ssclient = SchedulerServiceProcessClient(node=self.container.node, process=process) self.rrclient = ResourceRegistryServiceProcessClient(node=self.container.node, process=process) def tearDown(self): pass def now_utc(self): return time.mktime(datetime.datetime.utcnow().timetuple()) def test_create_interval_timer(self): # create the interval timer resource # create the event listener # call scheduler to set the timer # receive a few intervals, validate that arrival time is as expected # cancel the timer # wait until after next interval to verify that timer was correctly cancelled self.interval_timer_count = 0 self.interval_timer_sent_time = 0 self.interval_timer_received_time = 0 self.interval_timer_interval = 3 event_origin = "Interval_Timer_233" sub = EventSubscriber(event_type="ResourceEvent", callback=self.interval_timer_callback, origin=event_origin) sub.start() start_time = self.now_utc() self.interval_timer_end_time = start_time + 10 id = self.ssclient.create_interval_timer(start_time="now" , interval=self.interval_timer_interval, end_time=self.interval_timer_end_time, event_origin=event_origin, event_subtype="") self.interval_timer_sent_time = datetime.datetime.utcnow() self.assertEqual(type(id), str) # Validate the timer is stored in RR ss = self.rrclient.read(id) self.assertEqual(ss.entry.event_origin, event_origin) # Wait until two events are published gevent.sleep((self.interval_timer_interval * 2) + 1) #Cancle the timer ss = self.ssclient.cancel_timer(id) # wait until after next interval to verify that timer was correctly cancelled gevent.sleep(self.interval_timer_interval) # Validate the timer correctly cancelled with self.assertRaises(BadRequest): self.ssclient.cancel_timer(id) # Validate the timer is removed from resource regsitry with self.assertRaises(NotFound): self.rrclient.read(id) # Validate only 2 events are published self.assertEqual(self.interval_timer_count, 2) def test_system_restart(self): # create the interval timer resource # create the event listener # call scheduler to set the timer # receive a few intervals, validate that arrival time is as expected # cancel the timer # wait until after next interval to verify that timer was correctly cancelled self.interval_timer_count = 0 self.interval_timer_sent_time = 0 self.interval_timer_received_time = 0 self.interval_timer_interval = 3 event_origin = "Interval_Timer_4444" sub = EventSubscriber(event_type="ResourceEvent", callback=self.on_restart_callback, origin=event_origin) sub.start() start_time = self.now_utc() self.interval_timer_end_time = start_time + 20 id = self.ssclient.create_interval_timer(start_time="now" , interval=self.interval_timer_interval, end_time=self.interval_timer_end_time, event_origin=event_origin, event_subtype="") self.interval_timer_sent_time = datetime.datetime.utcnow() self.assertEqual(type(id), str) # Validate the timer is stored in RR ss = self.rrclient.read(id) self.assertEqual(ss.entry.event_origin, event_origin) # Wait until 1 event is published gevent.sleep((self.interval_timer_interval) + 1) # Validate 1 event is published self.assertEqual(self.interval_timer_count, 1) self.ssclient.on_system_restart() # after system restart, validate the timer is restored ss = self.rrclient.read(id) self.assertEqual(ss.entry.event_origin, event_origin) # Wait until another event is published gevent.sleep((self.interval_timer_interval * 2) + 1) # Validate 1 event is published self.assertGreater(self.interval_timer_count, 2) #Cancle the timer ss = self.ssclient.cancel_timer(id) # wait until after next interval to verify that timer was correctly cancelled gevent.sleep(self.interval_timer_interval) # Validate the timer correctly cancelled with self.assertRaises(BadRequest): self.ssclient.cancel_timer(id) # Validate the timer is removed from resource regsitry with self.assertRaises(NotFound): self.rrclient.read(id) def on_restart_callback(self, *args, **kwargs): self.interval_timer_count += 1 def test_create_interval_timer_with_end_time(self): # create the interval timer resource # create the event listener # call scheduler to set the timer # receive a few intervals, validate that arrival time is as expected # Validate no more events are published after end_time expires # Validate the timer was canceled after the end_time expires self.interval_timer_count = 0 self.interval_timer_sent_time = 0 self.interval_timer_received_time = 0 self.interval_timer_interval = 2 event_origin = "Interval Timer" sub = EventSubscriber(event_type="ResourceEvent", callback=self.interval_timer_callback, origin=event_origin) sub.start() start_time = self.now_utc() self.interval_timer_end_time = start_time + 5 id = self.ssclient.create_interval_timer(start_time="now" , interval=self.interval_timer_interval, end_time=self.interval_timer_end_time, event_origin=event_origin, event_subtype="") self.interval_timer_sent_time = datetime.datetime.utcnow() self.assertEqual(type(id), str) # Wait until all events are published gevent.sleep((self.interval_timer_end_time - start_time) + self.interval_timer_interval + 1) # Validate only 2 events are published self.assertEqual(self.interval_timer_count, 2) # Validate the timer was canceled after the end_time is expired with self.assertRaises(BadRequest): self.ssclient.cancel_timer(id) def interval_timer_callback(self, *args, **kwargs): self.interval_timer_received_time = datetime.datetime.utcnow() self.interval_timer_count += 1 time_diff = math.fabs( ((self.interval_timer_received_time - self.interval_timer_sent_time).total_seconds()) - (self.interval_timer_interval * self.interval_timer_count) ) # Assert expire time is within +-10 seconds self.assertTrue(time_diff <= 10) def test_cancel_single_timer(self): # test creating a new timer that is one-time-only # create the timer resource # create the event listener # call scheduler to set the timer # create then cancel the timer, verify that event is not received # create the timer resource # create the event listener # call scheduler to set the timer # call scheduler to cancel the timer # wait until after expiry to verify that event is not sent self.single_timer_count = 0 event_origin = "Time_of_Day" sub = EventSubscriber(event_type="ResourceEvent", callback=self.single_timer_call_back, origin=event_origin) sub.start() now = datetime.datetime.utcnow() + timedelta(seconds=3) times_of_day =[{'hour': str(now.hour),'minute' : str(now.minute), 'second':str(now.second) }] id = self.ssclient.create_time_of_day_timer(times_of_day=times_of_day, expires=self.now_utc()+3, event_origin=event_origin, event_subtype="test") self.assertEqual(type(id), str) self.ssclient.cancel_timer(id) gevent.sleep(3) # Validate the event is not generated self.assertEqual(self.single_timer_count, 0) def single_timer_call_back (self, *args, **kwargs): self.single_timer_count =+ 1 def test_create_forever_interval_timer(self): # Test creating interval timer that runs forever self.interval_timer_count = 0 self.interval_timer_sent_time = 0 self.interval_timer_received_time = 0 self.interval_timer_interval = 3 event_origin = "Interval Timer Forever" sub = EventSubscriber(event_type="ResourceEvent", callback=self.interval_timer_callback, origin=event_origin) sub.start() id = self.ssclient.create_interval_timer(start_time= self.now_utc(), interval=self.interval_timer_interval, end_time=-1, event_origin=event_origin, event_subtype=event_origin) self.interval_timer_sent_time = datetime.datetime.utcnow() self.assertEqual(type(id), str) # Wait for 4 events to be published gevent.sleep((self.interval_timer_interval * 4) + 1) self.ssclient.cancel_timer(id) # Validate the timer id is invalid once it has been canceled with self.assertRaises(BadRequest): self.ssclient.cancel_timer(id) # Validate events are not generated after canceling the timer self.assertEqual(self.interval_timer_count, 4) def test_timeoffday_timer(self): # test creating a new timer that is one-time-only # create the timer resource # get the current time, set the timer to several seconds from current time # create the event listener # call scheduler to set the timer # verify that event arrival is within one/two seconds of current time event_origin = "Time Of Day2" self.expire_sec_1 = 4 self.expire_sec_2 = 5 self.tod_count = 0 expire1 = datetime.datetime.utcnow() + timedelta(seconds=self.expire_sec_1) expire2 = datetime.datetime.utcnow() + timedelta(seconds=self.expire_sec_2) # Create two timers times_of_day =[{'hour': str(expire1.hour),'minute' : str(expire1.minute), 'second':str(expire1.second) }, {'hour': str(expire2.hour),'minute' : str(expire2.minute), 'second':str(expire2.second)}] sub = EventSubscriber(event_type="ResourceEvent", callback=self.tod_callback, origin=event_origin) sub.start() # Expires in one days expires = time.mktime((datetime.datetime.utcnow() + timedelta(days=2)).timetuple()) self.tod_sent_time = datetime.datetime.utcnow() id = self.ssclient.create_time_of_day_timer(times_of_day=times_of_day, expires=expires, event_origin=event_origin, event_subtype="") self.assertEqual(type(id), str) # Wait until all events are generated gevent.sleep(9) # After waiting for 15 seconds, validate only 2 events are generated. self.assertTrue(self.tod_count == 2) # Cancel the timer self.ssclient.cancel_timer(id) def tod_callback(self, *args, **kwargs): tod_receive_time = datetime.datetime.utcnow() self.tod_count += 1 if self.tod_count == 1: time_diff = math.fabs((tod_receive_time - self.tod_sent_time).total_seconds() - self.expire_sec_1) self.assertTrue(time_diff <= 2) elif self.tod_count == 2: time_diff = math.fabs((tod_receive_time - self.tod_sent_time).total_seconds() - self.expire_sec_2) self.assertTrue(time_diff <= 2)
class VizTransformProcForGoogleDT(TransformDataProcess): """ This class is used for instantiating worker processes that have subscriptions to data streams and convert incoming data from CDM format to JSON style Google DataTables """ def on_start(self): super(VizTransformProcForGoogleDT,self).on_start() self.initDataTableFlag = True # need some clients self.rr_cli = ResourceRegistryServiceProcessClient(process = self, node = self.container.node) self.pubsub_cli = PubsubManagementServiceClient(node=self.container.node) # extract the various parameters passed self.out_stream_id = self.CFG.get('process').get('publish_streams').get('visualization_service_submit_stream_id') # Create a publisher on the output stream out_stream_pub_registrar = StreamPublisherRegistrar(process=self.container, node=self.container.node) self.out_stream_pub = out_stream_pub_registrar.create_publisher(stream_id=self.out_stream_id) self.data_product_id = self.CFG.get('data_product_id') self.stream_def_id = self.CFG.get("stream_def_id") stream_def_resource = self.rr_cli.read(self.stream_def_id) self.stream_def = stream_def_resource.container self.realtime_flag = False if self.CFG.get("realtime_flag") == "True": self.realtime_flag = True else: self.data_product_id_token = self.CFG.get('data_product_id_token') # extract the stream_id associated with the DP. Needed later stream_ids,_ = self.rr_cli.find_objects(self.data_product_id, PRED.hasStream, None, True) self.stream_id = stream_ids[0] self.dataDescription = [] self.dataTableContent = [] self.varTuple = [] self.total_num_of_records_recvd = 0 def process(self, packet): log.debug('(%s): Received Viz Data Packet' % (self.name) ) element_count_id = 0 expected_range = [] psd = PointSupplementStreamParser(stream_definition=self.stream_def, stream_granule=packet) vardict = {} arrLen = None for varname in psd.list_field_names(): vardict[varname] = psd.get_values(varname) arrLen = len(vardict[varname]) #if its the first time, init the dataTable if self.initDataTableFlag: # create data description from the variables in the message self.dataDescription = [('time', 'datetime', 'time')] # split the data string to extract variable names for varname in psd.list_field_names(): if varname == 'time': continue self.dataDescription.append((varname, 'number', varname)) self.initDataTableFlag = False # Add the records to the datatable for i in xrange(arrLen): varTuple = [] for varname,_,_ in self.dataDescription: val = float(vardict[varname][i]) if varname == 'time': varTuple.append(datetime.fromtimestamp(val)) else: varTuple.append(val) # Append the tuples to the data table self.dataTableContent.append (varTuple) if self.realtime_flag: # Maintain a sliding window for realtime transform processes realtime_window_size = 100 if len(self.dataTableContent) > realtime_window_size: # always pop the first element till window size is what we want while len(self.dataTableContent) > realtime_window_size: self.dataTableContent.pop(0) if not self.realtime_flag: # This is the historical view part. Make a note of now many records were received data_stream_id = self.stream_def.data_stream_id element_count_id = self.stream_def.identifiables[data_stream_id].element_count_id # From each granule you can check the constraint on the number of records expected_range = packet.identifiables[element_count_id].constraint.intervals[0] # The number of records in a given packet is: self.total_num_of_records_recvd += packet.identifiables[element_count_id].value # submit the Json version of the datatable to the viz service if self.realtime_flag: # create the google viz data table data_table = gviz_api.DataTable(self.dataDescription) data_table.LoadData(self.dataTableContent) # submit resulting table back using the out stream publisher msg = {"viz_product_type": "google_realtime_dt", "data_product_id": self.data_product_id, "data_table": data_table.ToJSonResponse() } self.out_stream_pub.publish(msg) else: # Submit table back to the service if we received all the replay data if self.total_num_of_records_recvd == (expected_range[1] + 1): # If the datatable received was too big, decimate on the fly to a fixed size max_google_dt_len = 1024 if len(self.dataTableContent) > max_google_dt_len: decimation_factor = int(math.ceil(len(self.dataTableContent) / (max_google_dt_len))) for i in xrange(len(self.dataTableContent) - 1, 0, -1): if(i % decimation_factor == 0): continue self.dataTableContent.pop(i) data_table = gviz_api.DataTable(self.dataDescription) data_table.LoadData(self.dataTableContent) # submit resulting table back using the out stream publisher msg = {"viz_product_type": "google_dt", "data_product_id_token": self.data_product_id_token, "data_table": data_table.ToJSonResponse() } self.out_stream_pub.publish(msg) return # clear the tuple for future use self.varTuple[:] = []
class VizTransformProcForMatplotlibGraphs(TransformDataProcess): """ This class is used for instantiating worker processes that have subscriptions to data streams and convert incoming data from CDM format to Matplotlib graphs """ def on_start(self): super(VizTransformProcForMatplotlibGraphs,self).on_start() #assert len(self.streams)==1 self.initDataFlag = True self.graph_data = {} # Stores a dictionary of variables : [List of values] # Need some clients self.rr_cli = ResourceRegistryServiceProcessClient(process = self, node = self.container.node) self.pubsub_cli = PubsubManagementServiceClient(node=self.container.node) # extract the various parameters passed to the transform process self.out_stream_id = self.CFG.get('process').get('publish_streams').get('visualization_service_submit_stream_id') # Create a publisher on the output stream #stream_route = self.pubsub_cli.register_producer(stream_id=self.out_stream_id) out_stream_pub_registrar = StreamPublisherRegistrar(process=self.container, node=self.container.node) self.out_stream_pub = out_stream_pub_registrar.create_publisher(stream_id=self.out_stream_id) self.data_product_id = self.CFG.get('data_product_id') self.stream_def_id = self.CFG.get("stream_def_id") self.stream_def = self.rr_cli.read(self.stream_def_id) # Start the thread responsible for keeping track of time and generating graphs # Mutex for ensuring proper concurrent communications between threads self.lock = RLock() self.rendering_proc = Greenlet(self.rendering_thread) self.rendering_proc.start() def process(self, packet): log.debug('(%s): Received Viz Data Packet' % self.name ) #log.debug('(%s): - Processing: %s' % (self.name,packet)) # parse the incoming data psd = PointSupplementStreamParser(stream_definition=self.stream_def.container, stream_granule=packet) # re-arrange incoming data into an easy to parse dictionary vardict = {} arrLen = None for varname in psd.list_field_names(): vardict[varname] = psd.get_values(varname) arrLen = len(vardict[varname]) if self.initDataFlag: # look at the incoming packet and store for varname in psd.list_field_names(): self.lock.acquire() self.graph_data[varname] = [] self.lock.release() self.initDataFlag = False # If code reached here, the graph data storage has been initialized. Just add values # to the list with self.lock: for varname in psd.list_field_names(): self.graph_data[varname].extend(vardict[varname]) def rendering_thread(self): from copy import deepcopy # Service Client # init Matplotlib fig = Figure() ax = fig.add_subplot(111) canvas = FigureCanvas(fig) imgInMem = StringIO.StringIO() while True: # Sleep for a pre-decided interval. Should be specifiable in a YAML file gevent.sleep(20) # If there's no data, wait # Lock is used here to make sure the entire vector exists start to finish, this assures that the data won working_set=None with self.lock: if len(self.graph_data) == 0: continue else: working_set = deepcopy(self.graph_data) # For the simple case of testing, lets plot all time variant variables one at a time xAxisVar = 'time' xAxisFloatData = working_set[xAxisVar] for varName, varData in working_set.iteritems(): if varName == 'time' or varName == 'height' or varName == 'longitude' or varName == 'latitude': continue yAxisVar = varName yAxisFloatData = working_set[varName] # Generate the plot ax.plot(xAxisFloatData, yAxisFloatData, 'ro') ax.set_xlabel(xAxisVar) ax.set_ylabel(yAxisVar) ax.set_title(yAxisVar + ' vs ' + xAxisVar) ax.set_autoscale_on(False) # generate filename for the output image fileName = yAxisVar + '_vs_' + xAxisVar + '.png' # Save the figure to the in memory file canvas.print_figure(imgInMem, format="png") imgInMem.seek(0) # submit resulting table back using the out stream publisher msg = {"viz_product_type": "matplotlib_graphs", "data_product_id": self.data_product_id, "image_obj": imgInMem.getvalue(), "image_name": fileName} self.out_stream_pub.publish(msg) #clear the canvas for the next image ax.clear()
def upload_data(dataproduct_id): upload_folder = FileSystem.get_url(FS.TEMP, 'uploads') try: rr_client = ResourceRegistryServiceProcessClient( node=Container.instance.node, process=service_gateway_instance) object_store = Container.instance.object_store try: rr_client.read(str(dataproduct_id)) except BadRequest: raise BadRequest('Unknown DataProduct ID %s' % dataproduct_id) # required fields upload = request.files['file'] # <input type=file name="file"> # determine filetype filetype = _check_magic(upload) upload.seek(0) # return to beginning for save if upload and filetype is not None: # upload file - run filename through werkzeug.secure_filename filename = secure_filename(upload.filename) path = os.path.join(upload_folder, filename) upload_time = time.time() upload.save(path) # register upload file_upload_context = { # TODO add dataproduct_id 'name': 'User uploaded file %s' % filename, 'filename': filename, 'filetype': filetype, 'path': path, 'upload_time': upload_time, 'status': 'File uploaded to server' } fuc_id, _ = object_store.create_doc(file_upload_context) # client to process dispatch pd_client = ProcessDispatcherServiceClient() # create process definition process_definition = ProcessDefinition( name='upload_data_processor', executable={ 'module': 'ion.processes.data.upload.upload_data_processing', 'class': 'UploadDataProcessing' }) process_definition_id = pd_client.create_process_definition( process_definition) # create process process_id = pd_client.create_process(process_definition_id) #schedule process config = DotDict() config.process.fuc_id = fuc_id config.process.dp_id = dataproduct_id pid = pd_client.schedule_process(process_definition_id, process_id=process_id, configuration=config) log.info('UploadDataProcessing process created %s' % pid) # response - only FileUploadContext ID and determined filetype for UX display resp = {'fuc_id': fuc_id} return gateway_json_response(resp) raise BadRequest('Invalid Upload') except Exception as e: return build_error_response(e)
class TestSchedulerService(IonIntegrationTestCase): def setUp(self): self.interval_timer_count = 0 self.interval_timer_sent_time = 0 self.interval_timer_received_time = 0 self.interval_timer_interval = 3 # Start container self._start_container() self.container.start_rel_from_url('res/deploy/r2deploy.yml') process = FakeProcess() self.ssclient = SchedulerServiceProcessClient(node=self.container.node, process=process) self.rrclient = ResourceRegistryServiceProcessClient(node=self.container.node, process=process) def tearDown(self): pass def now_utc(self): return time.time() def test_create_interval_timer(self): # create the interval timer resource # create the event listener # call scheduler to set the timer # receive a few intervals, validate that arrival time is as expected # cancel the timer # wait until after next interval to verify that timer was correctly cancelled self.interval_timer_count = 0 self.interval_timer_sent_time = 0 self.interval_timer_received_time = 0 self.interval_timer_interval = 3 event_origin = "Interval_Timer_233" sub = EventSubscriber(event_type="TimerEvent", callback=self.interval_timer_callback, origin=event_origin) sub.start() self.addCleanup(sub.stop) start_time = self.now_utc() self.interval_timer_end_time = start_time + 10 id = self.ssclient.create_interval_timer(start_time="now" , interval=self.interval_timer_interval, end_time=self.interval_timer_end_time, event_origin=event_origin, event_subtype="") self.interval_timer_sent_time = datetime.datetime.utcnow() self.assertEqual(type(id), str) # Validate the timer is stored in RR ss = self.rrclient.read(id) self.assertEqual(ss.entry.event_origin, event_origin) # Wait until two events are published gevent.sleep((self.interval_timer_interval * 2) + 1) time_diff = (datetime.datetime.utcnow() - self.interval_timer_sent_time).seconds timer_counts = math.floor(time_diff/self.interval_timer_interval) #Cancle the timer ss = self.ssclient.cancel_timer(id) # wait until after next interval to verify that timer was correctly cancelled gevent.sleep(self.interval_timer_interval) # Validate the timer correctly cancelled with self.assertRaises(BadRequest): self.ssclient.cancel_timer(id) # Validate the timer is removed from resource regsitry with self.assertRaises(NotFound): self.rrclient.read(id) # Validate the number of timer counts self.assertEqual(self.interval_timer_count, timer_counts, "Invalid number of timeouts generated. Number of timeout: %d Expected timeout: %d Timer id: %s " %(self.interval_timer_count, timer_counts, id)) def test_system_restart(self): # create the interval timer resource # create the event listener # call scheduler to set the timer # receive a few intervals, validate that arrival time is as expected # cancel the timer # wait until after next interval to verify that timer was correctly cancelled self.interval_timer_count = 0 self.interval_timer_sent_time = 0 self.interval_timer_received_time = 0 self.interval_timer_interval = 3 event_origin = "Interval_Timer_4444" sub = EventSubscriber(event_type="TimerEvent", callback=self.on_restart_callback, origin=event_origin) sub.start() self.addCleanup(sub.stop) start_time = self.now_utc() self.interval_timer_end_time = start_time + 20 id = self.ssclient.create_interval_timer(start_time="now", interval=self.interval_timer_interval, end_time=self.interval_timer_end_time, event_origin=event_origin, event_subtype="") self.interval_timer_sent_time = datetime.datetime.utcnow() self.assertEqual(type(id), str) # Validate the timer is stored in RR ss = self.rrclient.read(id) self.assertEqual(ss.entry.event_origin, event_origin) # Wait until 1 event is published gevent.sleep((self.interval_timer_interval) + 1) time_diff = (datetime.datetime.utcnow() - self.interval_timer_sent_time).seconds timer_counts = math.floor(time_diff/self.interval_timer_interval) # Validate the number of events generated self.assertEqual(self.interval_timer_count, timer_counts, "Invalid number of timeouts generated. Number of timeout: %d Expected timeout: %d Timer id: %s " %(self.interval_timer_count, timer_counts, id)) self.ssclient.on_system_restart() # after system restart, validate the timer is restored ss = self.rrclient.read(id) self.assertEqual(ss.entry.event_origin, event_origin) # Wait until another event is published start_time = datetime.datetime.utcnow() gevent.sleep((self.interval_timer_interval * 2) + 1) time_diff = (datetime.datetime.utcnow() - start_time).seconds timer_counts = math.floor(time_diff/self.interval_timer_interval) # Validate the number of events generated self.assertGreater(self.interval_timer_count, timer_counts) #Cancle the timer ss = self.ssclient.cancel_timer(id) # wait until after next interval to verify that timer was correctly cancelled gevent.sleep(self.interval_timer_interval) # Validate the timer correctly cancelled with self.assertRaises(BadRequest): self.ssclient.cancel_timer(id) # Validate the timer is removed from resource regsitry with self.assertRaises(NotFound): self.rrclient.read(id) def on_restart_callback(self, *args, **kwargs): self.interval_timer_count += 1 log.debug("test_scheduler: on_restart_callback: time: " + str(self.now_utc()) + " count: " + str(self.interval_timer_count)) def test_create_interval_timer_with_end_time(self): # create the interval timer resource # create the event listener # call scheduler to set the timer # receive a few intervals, validate that arrival time is as expected # Validate no more events are published after end_time expires # Validate the timer was canceled after the end_time expires self.interval_timer_count_2 = 0 self.interval_timer_sent_time_2 = 0 self.interval_timer_received_time_2 = 0 self.interval_timer_interval_2 = 3 event_origin = "Interval_Timer_2" sub = EventSubscriber(event_type="TimerEvent", callback=self.interval_timer_callback_with_end_time, origin=event_origin) sub.start() self.addCleanup(sub.stop) start_time = self.now_utc() self.interval_timer_end_time_2 = start_time + 7 id = self.ssclient.create_interval_timer(start_time="now" , interval=self.interval_timer_interval_2, end_time=self.interval_timer_end_time_2, event_origin=event_origin, event_subtype="") self.interval_timer_sent_time_2 = datetime.datetime.utcnow() self.assertEqual(type(id), str) # Wait until all events are published gevent.sleep((self.interval_timer_end_time_2 - start_time) + self.interval_timer_interval_2 + 1) # Validate the number of events generated self.assertEqual(self.interval_timer_count_2, 2, "Invalid number of timeouts generated. Number of event: %d Expected: 2 Timer id: %s " %(self.interval_timer_count_2, id)) # Validate the timer was canceled after the end_time is expired with self.assertRaises(BadRequest): self.ssclient.cancel_timer(id) def interval_timer_callback_with_end_time(self, *args, **kwargs): self.interval_timer_received_time_2 = datetime.datetime.utcnow() self.interval_timer_count_2 += 1 time_diff = math.fabs( ((self.interval_timer_received_time_2 - self.interval_timer_sent_time_2).total_seconds()) - (self.interval_timer_interval_2 * self.interval_timer_count_2) ) # Assert expire time is within +-10 seconds self.assertTrue(time_diff <= 10) log.debug("test_scheduler: interval_timer_callback_with_end_time: time:" + str(self.interval_timer_received_time_2) + " count:" + str(self.interval_timer_count_2)) def interval_timer_callback(self, *args, **kwargs): self.interval_timer_received_time = datetime.datetime.utcnow() self.interval_timer_count += 1 time_diff = math.fabs( ((self.interval_timer_received_time - self.interval_timer_sent_time).total_seconds()) - (self.interval_timer_interval * self.interval_timer_count) ) # Assert expire time is within +-10 seconds self.assertTrue(time_diff <= 10) log.debug("test_scheduler: interval_timer_callback: time:" + str(self.interval_timer_received_time) + " count:" + str(self.interval_timer_count)) def test_cancel_single_timer(self): # test creating a new timer that is one-time-only # create the timer resource # create the event listener # call scheduler to set the timer # create then cancel the timer, verify that event is not received # create the timer resource # create the event listener # call scheduler to set the timer # call scheduler to cancel the timer # wait until after expiry to verify that event is not sent self.single_timer_count = 0 event_origin = "Time_of_Day" sub = EventSubscriber(event_type="TimerEvent", callback=self.single_timer_callback, origin=event_origin) sub.start() self.addCleanup(sub.stop) now = datetime.datetime.utcnow() + timedelta(seconds=3) times_of_day =[{'hour': str(now.hour),'minute' : str(now.minute), 'second':str(now.second) }] id = self.ssclient.create_time_of_day_timer(times_of_day=times_of_day, expires=self.now_utc()+3, event_origin=event_origin, event_subtype="test") self.assertEqual(type(id), str) self.ssclient.cancel_timer(id) gevent.sleep(3) # Validate the event is not generated self.assertEqual(self.single_timer_count, 0, "Invalid number of timeouts generated. Number of timeout: %d Expected timeout: 0 Timer id: %s " %(self.single_timer_count, id)) def single_timer_callback (self, *args, **kwargs): self.single_timer_count =+ 1 log.debug("test_scheduler: single_timer_call_back: time:" + str(self.now_utc()) + " count:" + str(self.single_timer_count)) def test_create_forever_interval_timer(self): # Test creating interval timer that runs forever self.interval_timer_count = 0 self.interval_timer_sent_time = 0 self.interval_timer_received_time = 0 self.interval_timer_interval = 3 event_origin = "Interval Timer Forever" sub = EventSubscriber(event_type="TimerEvent", callback=self.interval_timer_callback, origin=event_origin) sub.start() self.addCleanup(sub.stop) id = self.ssclient.create_interval_timer(start_time=str(self.now_utc()), interval=self.interval_timer_interval, end_time="-1", event_origin=event_origin, event_subtype=event_origin) self.interval_timer_sent_time = datetime.datetime.utcnow() self.assertEqual(type(id), str) # Wait for 4 events to be published gevent.sleep((self.interval_timer_interval * 4) + 1) self.ssclient.cancel_timer(id) time_diff = (datetime.datetime.utcnow() - self.interval_timer_sent_time).seconds timer_counts = math.floor(time_diff/self.interval_timer_interval) # Validate the timer id is invalid once it has been canceled with self.assertRaises(BadRequest): self.ssclient.cancel_timer(id) # Validate events are not generated after canceling the timer self.assertEqual(self.interval_timer_count, timer_counts, "Invalid number of timeouts generated. Number of timeout: %d Expected timeout: %d Timer id: %s " %(self.interval_timer_count, timer_counts, id)) def test_timeoffday_timer(self): # test creating a new timer that is one-time-only # create the timer resource # get the current time, set the timer to several seconds from current time # create the event listener # call scheduler to set the timer # verify that event arrival is within one/two seconds of current time event_origin = "Time Of Day2" self.expire_sec_1 = 4 self.expire_sec_2 = 5 self.tod_count = 0 expire1 = datetime.datetime.utcnow() + timedelta(seconds=self.expire_sec_1) expire2 = datetime.datetime.utcnow() + timedelta(seconds=self.expire_sec_2) # Create two timers times_of_day =[{'hour': str(expire1.hour),'minute' : str(expire1.minute), 'second':str(expire1.second) }, {'hour': str(expire2.hour),'minute' : str(expire2.minute), 'second':str(expire2.second)}] sub = EventSubscriber(event_type="TimerEvent", callback=self.tod_callback, origin=event_origin) sub.start() self.addCleanup(sub.stop) # Expires in one days expires = calendar.timegm((datetime.datetime.utcnow() + timedelta(days=2)).timetuple()) self.tod_sent_time = datetime.datetime.utcnow() id = self.ssclient.create_time_of_day_timer(times_of_day=times_of_day, expires=expires, event_origin=event_origin, event_subtype="") self.interval_timer_sent_time = datetime.datetime.utcnow() self.assertEqual(type(id), str) # Wait until all events are generated gevent.sleep(9) time_diff = (datetime.datetime.utcnow() - self.interval_timer_sent_time).seconds timer_counts = math.floor(time_diff/self.expire_sec_1) + math.floor(time_diff/self.expire_sec_2) # After waiting, validate only 2 events are generated. self.assertEqual(self.tod_count, 2, "Invalid number of timeouts generated. Number of timeout: %d Expected timeout: %d Timer id: %s " %(self.tod_count, timer_counts, id)) # Cancel the timer self.ssclient.cancel_timer(id) def tod_callback(self, *args, **kwargs): tod_receive_time = datetime.datetime.utcnow() self.tod_count += 1 if self.tod_count == 1: time_diff = math.fabs((tod_receive_time - self.tod_sent_time).total_seconds() - self.expire_sec_1) self.assertTrue(time_diff <= 2) elif self.tod_count == 2: time_diff = math.fabs((tod_receive_time - self.tod_sent_time).total_seconds() - self.expire_sec_2) self.assertTrue(time_diff <= 2) log.debug("test_scheduler: tod_callback: time:" + str(tod_receive_time) + " count:" + str(self.tod_count)) def test_timeoffday_timer_in_past_seconds(self): # test creating a new timer that is one-time-only # create the timer resource # get the current time, set the timer to several seconds from current time # create the event listener # call scheduler to set the timer # verify that event arrival is within one/two seconds of current time event_origin = "Time_Of_Day3" expire_sec = -4 self.tod_count2 = 0 now = datetime.datetime.utcnow() expire1 = now + timedelta(seconds=expire_sec) # Create two timers times_of_day = [{'hour': str(expire1.hour), 'minute': str(expire1.minute), 'second': str(expire1.second)}] sub = EventSubscriber(event_type="TimerEvent", callback=self.tod_callback2, origin=event_origin) sub.start() self.addCleanup(sub.stop) # Expires in 3 days expires = calendar.timegm((datetime.datetime.utcnow() + timedelta(days=3)).timetuple()) self.tod_sent_time = datetime.datetime.utcnow() id = self.ssclient.create_time_of_day_timer(times_of_day=times_of_day, expires=expires, event_origin=event_origin, event_subtype="") self.interval_timer_sent_time = datetime.datetime.utcnow() self.assertEqual(type(id), str) # Wait and see if the any events are generated gevent.sleep(5) # After waiting, validate no event is generated self.assertEqual(self.tod_count2, 0, "Invalid number of timeouts generated. Number of timeout: %d Expected timeout: 0 Timer id: %s " %(self.tod_count2, id)) # Cancel the timer self.ssclient.cancel_timer(id) # This is example for the following case # Example current time is 8:00AM. User setups a timer for 6:00AM. Since it is 8am, it tries to # setup a timer for tomorrow 6am but the expire time is set at 5AM tomorrow event_origin = "Time_Of_Day4" expire_sec = -4 self.tod_count2 = 0 now = datetime.datetime.utcnow() expire1 = now + timedelta(seconds=expire_sec) times_of_day = [{'hour': str(expire1.hour), 'minute': str(expire1.minute), 'second': str(expire1.second)}] sub = EventSubscriber(event_type="TimerEvent", callback=self.tod_callback2, origin=event_origin) sub.start() self.addCleanup(sub.stop) # Expires before the first event time_delta = timedelta(days=1) + timedelta(seconds=-(abs(expire_sec*2))) # Notice the minus sign. It expires before the first event expires = calendar.timegm((now + time_delta).timetuple()) self.tod_sent_time = datetime.datetime.utcnow() with self.assertRaises(BadRequest): id = self.ssclient.create_time_of_day_timer(times_of_day=times_of_day, expires=expires, event_origin=event_origin, event_subtype="") def tod_callback2(self, *args, **kwargs): tod_receive_time = datetime.datetime.utcnow() self.tod_count2 += 1 log.debug("test_scheduler: tod_callback2: time:") @unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False), 'Skip test while in CEI LAUNCH mode') def test_quit_stops_timers(self): ar = AsyncResult() def cb(*args, **kwargs): ar.set(args) self.interval_timer_count += 1 event_origin = "test_quitter" sub = EventSubscriber(event_type="TimerEvent", callback=cb, origin=event_origin) sub.start() self.addCleanup(sub.stop) tid = self.ssclient.create_interval_timer(start_time="now", end_time="-1", interval=1, event_origin=event_origin) # wait until at least one scheduled message ar.get(timeout=5) # shut it down! p = self.container.proc_manager.procs_by_name['scheduler'] self.container.terminate_process(p.id) # assert empty self.assertEquals(p.schedule_entries, {})
def upload_data(dataproduct_id): upload_folder = FileSystem.get_url(FS.TEMP,'uploads') try: rr_client = ResourceRegistryServiceProcessClient(process=service_gateway_instance) object_store = Container.instance.object_store try: rr_client.read(str(dataproduct_id)) except BadRequest: raise BadRequest('Unknown DataProduct ID %s' % dataproduct_id) # required fields upload = request.files['file'] # <input type=file name="file"> # determine filetype filetype = _check_magic(upload) upload.seek(0) # return to beginning for save if upload and filetype is not None: # upload file - run filename through werkzeug.secure_filename filename = secure_filename(upload.filename) path = os.path.join(upload_folder, filename) upload_time = time.time() upload.save(path) # register upload file_upload_context = { # TODO add dataproduct_id 'name':'User uploaded file %s' % filename, 'filename':filename, 'filetype':filetype, 'path':path, 'upload_time':upload_time, 'status':'File uploaded to server' } fuc_id, _ = object_store.create_doc(file_upload_context) # client to process dispatch pd_client = ProcessDispatcherServiceClient() # create process definition process_definition = ProcessDefinition( name='upload_data_processor', executable={ 'module':'ion.processes.data.upload.upload_data_processing', 'class':'UploadDataProcessing' } ) process_definition_id = pd_client.create_process_definition(process_definition) # create process process_id = pd_client.create_process(process_definition_id) #schedule process config = DotDict() config.process.fuc_id = fuc_id config.process.dp_id = dataproduct_id pid = pd_client.schedule_process(process_definition_id, process_id=process_id, configuration=config) log.info('UploadDataProcessing process created %s' % pid) # response - only FileUploadContext ID and determined filetype for UX display resp = {'fuc_id': fuc_id} return gateway_json_response(resp) raise BadRequest('Invalid Upload') except Exception as e: return build_error_response(e)
class TestWorkflowManagementIntegration(VisualizationIntegrationTestHelper): def setUp(self): # Start container self._start_container() self.container.start_rel_from_url('res/deploy/r2deploy.yml') # simulate preloading preload_ion_params(self.container) #Instantiate a process to represent the test process = WorkflowServiceTestProcess() # Now create client to DataProductManagementService self.rrclient = ResourceRegistryServiceProcessClient( node=self.container.node, process=process) self.damsclient = DataAcquisitionManagementServiceProcessClient( node=self.container.node, process=process) self.pubsubclient = PubsubManagementServiceProcessClient( node=self.container.node, process=process) self.ingestclient = IngestionManagementServiceProcessClient( node=self.container.node, process=process) self.imsclient = InstrumentManagementServiceProcessClient( node=self.container.node, process=process) self.dataproductclient = DataProductManagementServiceProcessClient( node=self.container.node, process=process) self.dataprocessclient = DataProcessManagementServiceProcessClient( node=self.container.node, process=process) self.datasetclient = DatasetManagementServiceProcessClient( node=self.container.node, process=process) self.workflowclient = WorkflowManagementServiceProcessClient( node=self.container.node, process=process) self.process_dispatcher = ProcessDispatcherServiceProcessClient( node=self.container.node, process=process) self.data_retriever = DataRetrieverServiceProcessClient( node=self.container.node, process=process) self.ctd_stream_def = SBE37_CDM_stream_definition() @attr('LOCOINT') @unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False), 'Not integrated for CEI') def test_SA_transform_components(self): assertions = self.assertTrue #The list of data product streams to monitor data_product_stream_ids = list() #Create the input data product ctd_stream_id, ctd_parsed_data_product_id = self.create_ctd_input_stream_and_data_product( ) data_product_stream_ids.append(ctd_stream_id) ### ### Setup the first transformation ### # Salinity: Data Process Definition ctd_L2_salinity_dprocdef_id = self.create_salinity_data_process_definition( ) l2_salinity_all_data_process_id, ctd_l2_salinity_output_dp_id = self.create_transform_process( ctd_L2_salinity_dprocdef_id, ctd_parsed_data_product_id, 'salinity') ## get the stream id for the transform outputs stream_ids, _ = self.rrclient.find_objects( ctd_l2_salinity_output_dp_id, PRED.hasStream, None, True) assertions(len(stream_ids) > 0) sal_stream_id = stream_ids[0] data_product_stream_ids.append(sal_stream_id) ### ### Setup the second transformation ### # Salinity Doubler: Data Process Definition salinity_doubler_dprocdef_id = self.create_salinity_doubler_data_process_definition( ) salinity_double_data_process_id, salinity_doubler_output_dp_id = self.create_transform_process( salinity_doubler_dprocdef_id, ctd_l2_salinity_output_dp_id, 'salinity') stream_ids, _ = self.rrclient.find_objects( salinity_doubler_output_dp_id, PRED.hasStream, None, True) assertions(len(stream_ids) > 0) sal_dbl_stream_id = stream_ids[0] data_product_stream_ids.append(sal_dbl_stream_id) #Start the output stream listener to monitor and collect messages results = self.start_output_stream_and_listen(ctd_stream_id, data_product_stream_ids) #Stop the transform processes self.dataprocessclient.deactivate_data_process( salinity_double_data_process_id) self.dataprocessclient.deactivate_data_process( l2_salinity_all_data_process_id) #Validate the data from each of the messages along the way self.validate_messages(results) @attr('LOCOINT') @attr('SMOKE') @unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False), 'Not integrated for CEI') def test_transform_workflow(self): assertions = self.assertTrue log.debug("Building the workflow definition") workflow_def_obj = IonObject( RT.WorkflowDefinition, name='Salinity_Test_Workflow', description='tests a workflow of multiple transform data processes' ) workflow_data_product_name = 'TEST-Workflow_Output_Product' #Set a specific output product name #------------------------------------------------------------------------------------------------------------------------- log.debug("Adding a transformation process definition for salinity") #------------------------------------------------------------------------------------------------------------------------- ctd_L2_salinity_dprocdef_id = self.create_salinity_data_process_definition( ) workflow_step_obj = IonObject( 'DataProcessWorkflowStep', data_process_definition_id=ctd_L2_salinity_dprocdef_id, persist_process_output_data=False ) #Don't persist the intermediate data product workflow_def_obj.workflow_steps.append(workflow_step_obj) #------------------------------------------------------------------------------------------------------------------------- log.debug( "Adding a transformation process definition for salinity doubler") #------------------------------------------------------------------------------------------------------------------------- salinity_doubler_dprocdef_id = self.create_salinity_doubler_data_process_definition( ) workflow_step_obj = IonObject( 'DataProcessWorkflowStep', data_process_definition_id=salinity_doubler_dprocdef_id, ) workflow_def_obj.workflow_steps.append(workflow_step_obj) log.debug("Creating workflow def in the resource registry") workflow_def_id = self.workflowclient.create_workflow_definition( workflow_def_obj) aids = self.rrclient.find_associations(workflow_def_id, PRED.hasDataProcessDefinition) assertions(len(aids) == 2) #The list of data product streams to monitor data_product_stream_ids = list() log.debug("Creating the input data product") ctd_stream_id, ctd_parsed_data_product_id = self.create_ctd_input_stream_and_data_product( ) data_product_stream_ids.append(ctd_stream_id) log.debug("Creating and starting the workflow") workflow_id, workflow_product_id = self.workflowclient.create_data_process_workflow( workflow_def_id, ctd_parsed_data_product_id, persist_workflow_data_product=True, output_data_product_name=workflow_data_product_name, timeout=300) workflow_output_ids, _ = self.rrclient.find_subjects( RT.Workflow, PRED.hasOutputProduct, workflow_product_id, True) assertions(len(workflow_output_ids) == 1) log.debug("persisting the output product") #self.dataproductclient.activate_data_product_persistence(workflow_product_id) dataset_ids, _ = self.rrclient.find_objects(workflow_product_id, PRED.hasDataset, RT.Dataset, True) assertions(len(dataset_ids) == 1) dataset_id = dataset_ids[0] log.debug( "Verifying the output data product name matches what was specified in the workflow definition" ) workflow_product = self.rrclient.read(workflow_product_id) assertions( workflow_product.name.startswith(workflow_data_product_name), 'Nope: %s != %s' % (workflow_product.name, workflow_data_product_name)) log.debug( "Walking the associations to find the appropriate output data streams to validate the messages" ) workflow_dp_ids, _ = self.rrclient.find_objects( workflow_id, PRED.hasDataProduct, RT.DataProduct, True) assertions(len(workflow_dp_ids) == 2) for dp_id in workflow_dp_ids: stream_ids, _ = self.rrclient.find_objects(dp_id, PRED.hasStream, None, True) assertions(len(stream_ids) == 1) data_product_stream_ids.append(stream_ids[0]) log.debug("data_product_stream_ids: %s" % data_product_stream_ids) log.debug( "Starting the output stream listener to monitor to collect messages" ) results = self.start_output_stream_and_listen(ctd_stream_id, data_product_stream_ids) log.debug("results::: %s" % results) log.debug("Stopping the workflow processes") self.workflowclient.terminate_data_process_workflow( workflow_id, False, timeout=250) # Should test true at some point log.debug("Making sure the Workflow object was removed") objs, _ = self.rrclient.find_resources(restype=RT.Workflow) assertions(len(objs) == 0) log.debug( "Validating the data from each of the messages along the way") self.validate_messages(results) log.debug( "Checking to see if dataset id = %s, was persisted, and that it can be retrieved...." % dataset_id) self.validate_data_ingest_retrieve(dataset_id) log.debug("Cleaning up to make sure delete is correct.") self.workflowclient.delete_workflow_definition(workflow_def_id) """ workflow_def_ids,_ = self.rrclient.find_resources(restype=RT.WorkflowDefinition) assertions(len(workflow_def_ids) == 0 ) """ @attr('LOCOINT') @unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False), 'Not integrated for CEI') def test_highcharts_transform_workflow(self): assertions = self.assertTrue # Build the workflow definition workflow_def_obj = IonObject( RT.WorkflowDefinition, name='HighCharts_Test_Workflow', description= 'Tests the workflow of converting stream data to HighCharts') #Add a transformation process definition highcharts_procdef_id = self.create_highcharts_data_process_definition( ) workflow_step_obj = IonObject( 'DataProcessWorkflowStep', data_process_definition_id=highcharts_procdef_id, persist_process_output_data=False) workflow_def_obj.workflow_steps.append(workflow_step_obj) #Create it in the resource registry workflow_def_id = self.workflowclient.create_workflow_definition( workflow_def_obj) #The list of data product streams to monitor data_product_stream_ids = list() #Create the input data product ctd_stream_id, ctd_parsed_data_product_id = self.create_ctd_input_stream_and_data_product( ) data_product_stream_ids.append(ctd_stream_id) #Create and start the workflow workflow_id, workflow_product_id = self.workflowclient.create_data_process_workflow( workflow_def_id, ctd_parsed_data_product_id, timeout=60) workflow_output_ids, _ = self.rrclient.find_subjects( RT.Workflow, PRED.hasOutputProduct, workflow_product_id, True) assertions(len(workflow_output_ids) == 1) #Walk the associations to find the appropriate output data streams to validate the messages workflow_dp_ids, _ = self.rrclient.find_objects( workflow_id, PRED.hasDataProduct, RT.DataProduct, True) assertions(len(workflow_dp_ids) == 1) for dp_id in workflow_dp_ids: stream_ids, _ = self.rrclient.find_objects(dp_id, PRED.hasStream, None, True) assertions(len(stream_ids) == 1) data_product_stream_ids.append(stream_ids[0]) #Start the output stream listener to monitor and collect messages results = self.start_output_stream_and_listen(ctd_stream_id, data_product_stream_ids) #Stop the workflow processes self.workflowclient.terminate_data_process_workflow( workflow_id=workflow_id, delete_data_products=False, timeout=60) # Should test true at some point #Validate the data from each of the messages along the way self.validate_highcharts_transform_results(results) #Cleanup to make sure delete is correct. self.workflowclient.delete_workflow_definition(workflow_def_id) """ workflow_def_ids,_ = self.rrclient.find_resources(restype=RT.WorkflowDefinition) assertions(len(workflow_def_ids) == 0 ) """ @attr('LOCOINT') @unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False), 'Not integrated for CEI') def test_mpl_graphs_transform_workflow(self): assertions = self.assertTrue # Build the workflow definition workflow_def_obj = IonObject( RT.WorkflowDefinition, name='Mpl_Graphs_Test_Workflow', description= 'Tests the workflow of converting stream data to Matplotlib graphs' ) #Add a transformation process definition mpl_graphs_procdef_id = self.create_mpl_graphs_data_process_definition( ) workflow_step_obj = IonObject( 'DataProcessWorkflowStep', data_process_definition_id=mpl_graphs_procdef_id, persist_process_output_data=False) workflow_def_obj.workflow_steps.append(workflow_step_obj) #Create it in the resource registry workflow_def_id = self.workflowclient.create_workflow_definition( workflow_def_obj) #The list of data product streams to monitor data_product_stream_ids = list() #Create the input data product ctd_stream_id, ctd_parsed_data_product_id = self.create_ctd_input_stream_and_data_product( ) data_product_stream_ids.append(ctd_stream_id) #Create and start the workflow workflow_id, workflow_product_id = self.workflowclient.create_data_process_workflow( workflow_def_id, ctd_parsed_data_product_id, persist_workflow_data_product=True, timeout=60) workflow_output_ids, _ = self.rrclient.find_subjects( RT.Workflow, PRED.hasOutputProduct, workflow_product_id, True) assertions(len(workflow_output_ids) == 1) #Walk the associations to find the appropriate output data streams to validate the messages workflow_dp_ids, _ = self.rrclient.find_objects( workflow_id, PRED.hasDataProduct, RT.DataProduct, True) assertions(len(workflow_dp_ids) == 1) for dp_id in workflow_dp_ids: stream_ids, _ = self.rrclient.find_objects(dp_id, PRED.hasStream, None, True) assertions(len(stream_ids) == 1) data_product_stream_ids.append(stream_ids[0]) #Start the output stream listener to monitor and collect messages results = self.start_output_stream_and_listen(ctd_stream_id, data_product_stream_ids) #Stop the workflow processes self.workflowclient.terminate_data_process_workflow( workflow_id=workflow_id, delete_data_products=False, timeout=60) # Should test true at some point #Validate the data from each of the messages along the way self.validate_mpl_graphs_transform_results(results) # Check to see if ingestion worked. Extract the granules from data_retrieval. # First find the dataset associated with the output dp product ds_ids, _ = self.rrclient.find_objects( workflow_dp_ids[len(workflow_dp_ids) - 1], PRED.hasDataset, RT.Dataset, True) retrieved_granule = self.data_retriever.retrieve_last_data_points( ds_ids[0], 10) #Validate the data from each of the messages along the way self.validate_mpl_graphs_transform_results(retrieved_granule) #Cleanup to make sure delete is correct. self.workflowclient.delete_workflow_definition(workflow_def_id) """ workflow_def_ids,_ = self.rrclient.find_resources(restype=RT.WorkflowDefinition) assertions(len(workflow_def_ids) == 0 ) """ @attr('LOCOINT') @unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False), 'Not integrated for CEI') def test_multiple_workflow_instances(self): assertions = self.assertTrue # Build the workflow definition workflow_def_obj = IonObject( RT.WorkflowDefinition, name='Multiple_Test_Workflow', description='Tests the workflow of converting stream data') #Add a transformation process definition highcharts_procdef_id = self.create_highcharts_data_process_definition( ) workflow_step_obj = IonObject( 'DataProcessWorkflowStep', data_process_definition_id=highcharts_procdef_id, persist_process_output_data=False) workflow_def_obj.workflow_steps.append(workflow_step_obj) #Create it in the resource registry workflow_def_id = self.workflowclient.create_workflow_definition( workflow_def_obj) #The list of data product streams to monitor data_product_stream_ids = list() #Create the first input data product ctd_stream_id1, ctd_parsed_data_product_id1 = self.create_ctd_input_stream_and_data_product( 'ctd_parsed1') data_product_stream_ids.append(ctd_stream_id1) #Create and start the first workflow workflow_id1, workflow_product_id1 = self.workflowclient.create_data_process_workflow( workflow_def_id, ctd_parsed_data_product_id1, timeout=60) #Create the second input data product ctd_stream_id2, ctd_parsed_data_product_id2 = self.create_ctd_input_stream_and_data_product( 'ctd_parsed2') data_product_stream_ids.append(ctd_stream_id2) #Create and start the second workflow workflow_id2, workflow_product_id2 = self.workflowclient.create_data_process_workflow( workflow_def_id, ctd_parsed_data_product_id2, timeout=60) #Walk the associations to find the appropriate output data streams to validate the messages workflow_ids, _ = self.rrclient.find_resources(restype=RT.Workflow) assertions(len(workflow_ids) == 2) #Start the first input stream process ctd_sim_pid1 = self.start_sinusoidal_input_stream_process( ctd_stream_id1) #Start the second input stream process ctd_sim_pid2 = self.start_simple_input_stream_process(ctd_stream_id2) #Start the output stream listener to monitor a set number of messages being sent through the workflows results = self.start_output_stream_and_listen( None, data_product_stream_ids, message_count_per_stream=5) # stop the flow of messages... self.process_dispatcher.cancel_process( ctd_sim_pid1 ) # kill the ctd simulator process - that is enough data self.process_dispatcher.cancel_process(ctd_sim_pid2) #Stop the first workflow processes self.workflowclient.terminate_data_process_workflow( workflow_id=workflow_id1, delete_data_products=False, timeout=60) # Should test true at some point #Stop the second workflow processes self.workflowclient.terminate_data_process_workflow( workflow_id=workflow_id2, delete_data_products=False, timeout=60) # Should test true at some point workflow_ids, _ = self.rrclient.find_resources(restype=RT.Workflow) assertions(len(workflow_ids) == 0) #Cleanup to make sure delete is correct. self.workflowclient.delete_workflow_definition(workflow_def_id) """