def on_start(self): super(VizTransformProcForMatplotlibGraphs, self).on_start() #assert len(self.streams)==1 self.initDataFlag = True self.graph_data = { } # Stores a dictionary of variables : [List of values] # Need some clients self.rr_cli = ResourceRegistryServiceProcessClient( process=self, node=self.container.node) self.pubsub_cli = PubsubManagementServiceClient( node=self.container.node) # extract the various parameters passed to the transform process self.out_stream_id = self.CFG.get('process').get( 'publish_streams').get('visualization_service_submit_stream_id') # Create a publisher on the output stream #stream_route = self.pubsub_cli.register_producer(stream_id=self.out_stream_id) out_stream_pub_registrar = StreamPublisherRegistrar( process=self.container, node=self.container.node) self.out_stream_pub = out_stream_pub_registrar.create_publisher( stream_id=self.out_stream_id) self.data_product_id = self.CFG.get('data_product_id') self.stream_def_id = self.CFG.get("stream_def_id") self.stream_def = self.rr_cli.read(self.stream_def_id) # Start the thread responsible for keeping track of time and generating graphs # Mutex for ensuring proper concurrent communications between threads self.lock = RLock() self.rendering_proc = Greenlet(self.rendering_thread) self.rendering_proc.start()
def on_start(self): super(VizTransformProcForMatplotlibGraphs,self).on_start() #assert len(self.streams)==1 self.initDataFlag = True self.graph_data = {} # Stores a dictionary of variables : [List of values] # Need some clients self.rr_cli = ResourceRegistryServiceProcessClient(process = self, node = self.container.node) self.pubsub_cli = PubsubManagementServiceClient(node=self.container.node) # extract the various parameters passed to the transform process self.out_stream_id = self.CFG.get('process').get('publish_streams').get('visualization_service_submit_stream_id') # Create a publisher on the output stream #stream_route = self.pubsub_cli.register_producer(stream_id=self.out_stream_id) out_stream_pub_registrar = StreamPublisherRegistrar(process=self.container, node=self.container.node) self.out_stream_pub = out_stream_pub_registrar.create_publisher(stream_id=self.out_stream_id) self.data_product_id = self.CFG.get('data_product_id') self.stream_def_id = self.CFG.get("stream_def_id") self.stream_def = self.rr_cli.read(self.stream_def_id) # Start the thread responsible for keeping track of time and generating graphs # Mutex for ensuring proper concurrent communications between threads self.lock = RLock() self.rendering_proc = Greenlet(self.rendering_thread) self.rendering_proc.start()
def on_start(self): super(VizTransformProcForGoogleDT,self).on_start() self.initDataTableFlag = True # need some clients self.rr_cli = ResourceRegistryServiceProcessClient(process = self, node = self.container.node) self.pubsub_cli = PubsubManagementServiceClient(node=self.container.node) # extract the various parameters passed self.out_stream_id = self.CFG.get('process').get('publish_streams').get('visualization_service_submit_stream_id') # Create a publisher on the output stream out_stream_pub_registrar = StreamPublisherRegistrar(process=self.container, node=self.container.node) self.out_stream_pub = out_stream_pub_registrar.create_publisher(stream_id=self.out_stream_id) self.data_product_id = self.CFG.get('data_product_id') self.stream_def_id = self.CFG.get("stream_def_id") stream_def_resource = self.rr_cli.read(self.stream_def_id) self.stream_def = stream_def_resource.container self.realtime_flag = False if self.CFG.get("realtime_flag") == "True": self.realtime_flag = True else: self.data_product_id_token = self.CFG.get('data_product_id_token') # extract the stream_id associated with the DP. Needed later stream_ids,_ = self.rr_cli.find_objects(self.data_product_id, PRED.hasStream, None, True) self.stream_id = stream_ids[0] self.dataDescription = [] self.dataTableContent = [] self.varTuple = [] self.total_num_of_records_recvd = 0
def setUp(self): self._start_container() self.cc = ContainerAgentClient(node=self.container.node,name=self.container.name) self.cc.start_rel_from_url('res/deploy/r2dm.yml') self.pubsub_cli = PubsubManagementServiceClient(node=self.cc.node) self.ctd_stream1_id = self.pubsub_cli.create_stream(name="SampleStream1", description="Sample Stream 1 Description") self.ctd_stream2_id = self.pubsub_cli.create_stream(name="SampleStream2", description="Sample Stream 2 Description") # Make a subscription to two input streams exchange_name = "a_queue" query = StreamQuery([self.ctd_stream1_id, self.ctd_stream2_id]) self.ctd_subscription_id = self.pubsub_cli.create_subscription(query, exchange_name, "SampleSubscription", "Sample Subscription Description") # Make a subscription to all streams on an exchange point exchange_name = "another_queue" query = ExchangeQuery() self.exchange_subscription_id = self.pubsub_cli.create_subscription(query, exchange_name, "SampleExchangeSubscription", "Sample Exchange Subscription Description") pid = self.container.spawn_process(name='dummy_process_for_test', module='pyon.ion.process', cls='SimpleProcess', config={}) dummy_process = self.container.proc_manager.procs[pid] # Normally the user does not see or create the publisher, this is part of the containers business. # For the test we need to set it up explicitly publisher_registrar = StreamPublisherRegistrar(process=dummy_process, node=self.cc.node) self.ctd_stream1_publisher = publisher_registrar.create_publisher(stream_id=self.ctd_stream1_id) self.ctd_stream2_publisher = publisher_registrar.create_publisher(stream_id=self.ctd_stream2_id) # Cheat and use the cc as the process - I don't think it is used for anything... self.stream_subscriber = StreamSubscriberRegistrar(process=dummy_process, node=self.cc.node)
def setUp(self): self._start_container() self.container.start_rel_from_url('res/deploy/r2dm.yml') self.pubsub_cli = PubsubManagementServiceClient(node=self.container.node) self.ctd_stream1_id = self.pubsub_cli.create_stream(name="SampleStream1", description="Sample Stream 1 Description") self.ctd_stream2_id = self.pubsub_cli.create_stream(name="SampleStream2", description="Sample Stream 2 Description") # Make a subscription to two input streams exchange_name = "a_queue" query = StreamQuery([self.ctd_stream1_id, self.ctd_stream2_id]) self.ctd_subscription_id = self.pubsub_cli.create_subscription(query, exchange_name, "SampleSubscription", "Sample Subscription Description") # Make a subscription to all streams on an exchange point exchange_name = "another_queue" query = ExchangeQuery() self.exchange_subscription_id = self.pubsub_cli.create_subscription(query, exchange_name, "SampleExchangeSubscription", "Sample Exchange Subscription Description") pid = self.container.spawn_process(name='dummy_process_for_test', module='pyon.ion.process', cls='SimpleProcess', config={}) dummy_process = self.container.proc_manager.procs[pid] # Normally the user does not see or create the publisher, this is part of the containers business. # For the test we need to set it up explicitly publisher_registrar = StreamPublisherRegistrar(process=dummy_process, node=self.container.node) self.ctd_stream1_publisher = publisher_registrar.create_publisher(stream_id=self.ctd_stream1_id) self.ctd_stream2_publisher = publisher_registrar.create_publisher(stream_id=self.ctd_stream2_id) # Cheat and use the cc as the process - I don't think it is used for anything... self.stream_subscriber = StreamSubscriberRegistrar(process=dummy_process, node=self.container.node)
def on_start(self): super(VizTransformProcForGoogleDT, self).on_start() self.initDataTableFlag = True # need some clients self.rr_cli = ResourceRegistryServiceProcessClient( process=self, node=self.container.node) self.pubsub_cli = PubsubManagementServiceClient( node=self.container.node) # extract the various parameters passed self.out_stream_id = self.CFG.get('process').get( 'publish_streams').get('visualization_service_submit_stream_id') # Create a publisher on the output stream out_stream_pub_registrar = StreamPublisherRegistrar( process=self.container, node=self.container.node) self.out_stream_pub = out_stream_pub_registrar.create_publisher( stream_id=self.out_stream_id) self.data_product_id = self.CFG.get('data_product_id') self.stream_def_id = self.CFG.get("stream_def_id") stream_def_resource = self.rr_cli.read(self.stream_def_id) self.stream_def = stream_def_resource.container self.realtime_flag = False if self.CFG.get("realtime_flag") == "True": self.realtime_flag = True else: self.data_product_id_token = self.CFG.get('data_product_id_token') # extract the stream_id associated with the DP. Needed later stream_ids, _ = self.rr_cli.find_objects(self.data_product_id, PRED.hasStream, None, True) self.stream_id = stream_ids[0] self.dataDescription = [] self.dataTableContent = [] self.varTuple = [] self.total_num_of_records_recvd = 0
def test_createDataProcess(self): #------------------------------- # Data Process Definition #------------------------------- log.debug("TestIntDataProcessManagementService: create data process definition") dpd_obj = IonObject(RT.DataProcessDefinition, name='data_process_definition', description='some new dpd', module='ion.processes.data.transforms.transform_example', class_name='TransformExample', process_source='some_source_reference') try: dprocdef_id = self.Processclient.create_data_process_definition(dpd_obj) except BadRequest as ex: self.fail("failed to create new data process definition: %s" %ex) # test Data Process Definition creation in rr dprocdef_obj = self.Processclient.read_data_process_definition(dprocdef_id) self.assertEquals(dprocdef_obj.name,'data_process_definition') # Create an input instrument instrument_obj = IonObject(RT.InstrumentDevice, name='Inst1',description='an instrument that is creating the data product') instrument_id, rev = self.RRclient.create(instrument_obj) # Register the instrument so that the data producer and stream object are created data_producer_id = self.DAMSclient.register_instrument(instrument_id) log.debug("TestIntDataProcessManagementService data_producer_id %s" % data_producer_id) #------------------------------- # Input Data Product #------------------------------- log.debug("TestIntDataProcessManagementService: create input data product") input_dp_obj = IonObject(RT.DataProduct, name='InputDataProduct', description='some new dp') try: input_dp_id = self.DPMSclient.create_data_product(input_dp_obj, instrument_id) except BadRequest as ex: self.fail("failed to create new input data product: %s" %ex) # Retrieve the stream via the DataProduct->Stream associations stream_ids, _ = self.RRclient.find_objects(input_dp_id, PRED.hasStream, None, True) log.debug("TestIntDataProcessManagementService: in stream_ids " + str(stream_ids)) self.in_stream_id = stream_ids[0] log.debug("TestIntDataProcessManagementService: Input Stream: " + str( self.in_stream_id)) #------------------------------- # Output Data Product #------------------------------- log.debug("TestIntDataProcessManagementService: create output data product") output_dp_obj = IonObject(RT.DataProduct, name='OutDataProduct',description='transform output') output_dp_id = self.DPMSclient.create_data_product(output_dp_obj) # this will NOT create a stream for the product becuase the data process (source) resource has not been created yet. #------------------------------- # Create the data process #------------------------------- log.debug("TestIntDataProcessManagementService: create_data_process start") try: dproc_id = self.Processclient.create_data_process(dprocdef_id, input_dp_id, output_dp_id) except BadRequest as ex: self.fail("failed to create new data process: %s" %ex) self.DAMSclient.assign_data_product(dproc_id, output_dp_id, False) log.debug("TestIntDataProcessManagementService: create_data_process return") #------------------------------- # Producer (Sample Input) #------------------------------- # Create a producing example process # cheat to make a publisher object to send messages in the test. # it is really hokey to pass process=self.cc but it works #stream_route = self.PubSubClient.register_producer(exchange_name='producer_doesnt_have_a_name1', stream_id=self.in_stream_id) #self.ctd_stream1_publisher = StreamPublisher(node=self.container.node, name=('science_data',stream_route.routing_key), process=self.container) pid = self.container.spawn_process(name='dummy_process_for_test', module='pyon.ion.process', cls='SimpleProcess', config={}) dummy_process = self.container.proc_manager.procs[pid] publisher_registrar = StreamPublisherRegistrar(process=dummy_process, node=self.container.node) self.ctd_stream1_publisher = publisher_registrar.create_publisher(stream_id=self.in_stream_id) msg = {'num':'3'} self.ctd_stream1_publisher.publish(msg) time.sleep(1) msg = {'num':'5'} self.ctd_stream1_publisher.publish(msg) time.sleep(1) msg = {'num':'9'} self.ctd_stream1_publisher.publish(msg) # See /tmp/transform_output for results..... # clean up the data process try: self.Processclient.delete_data_process(dproc_id) except BadRequest as ex: self.fail("failed to create new data process definition: %s" %ex) with self.assertRaises(NotFound) as e: self.Processclient.read_data_process(dproc_id) try: self.Processclient.delete_data_process_definition(dprocdef_id) except BadRequest as ex: self.fail("failed to create new data process definition: %s" %ex) with self.assertRaises(NotFound) as e: self.Processclient.read_data_process_definition(dprocdef_id)
def test_integrated_transform(self): ''' This example script runs a chained three way transform: B A < C Where A is the even_odd transform (generates a stream of even and odd numbers from input) and B and C are the basic transforms that receive even and odd input ''' cc = self.container assertions = self.assertTrue pubsub_cli = PubsubManagementServiceClient(node=cc.node) rr_cli = ResourceRegistryServiceClient(node=cc.node) tms_cli = TransformManagementServiceClient(node=cc.node) #------------------------------- # Process Definition #------------------------------- # Create the process definition for the basic transform process_definition = IonObject(RT.ProcessDefinition, name='basic_transform_definition') process_definition.executable = { 'module': 'ion.processes.data.transforms.transform_example', 'class':'TransformExample' } basic_transform_definition_id, _ = rr_cli.create(process_definition) # Create The process definition for the TransformEvenOdd process_definition = IonObject(RT.ProcessDefinition, name='evenodd_transform_definition') process_definition.executable = { 'module': 'ion.processes.data.transforms.transform_example', 'class':'TransformEvenOdd' } evenodd_transform_definition_id, _ = rr_cli.create(process_definition) #------------------------------- # Streams #------------------------------- streams = [pubsub_cli.create_stream() for i in xrange(5)] #------------------------------- # Subscriptions #------------------------------- query = StreamQuery(stream_ids=[streams[0]]) input_subscription_id = pubsub_cli.create_subscription(query=query, exchange_name='input_queue') query = StreamQuery(stream_ids = [streams[1]]) # even output even_subscription_id = pubsub_cli.create_subscription(query=query, exchange_name='even_queue') query = StreamQuery(stream_ids = [streams[2]]) # odd output odd_subscription_id = pubsub_cli.create_subscription(query=query, exchange_name='odd_queue') #------------------------------- # Launch the EvenOdd Transform #------------------------------- evenodd_id = tms_cli.create_transform(name='even_odd', in_subscription_id=input_subscription_id, out_streams={'even':streams[1], 'odd':streams[2]}, process_definition_id=evenodd_transform_definition_id, configuration={}) tms_cli.activate_transform(evenodd_id) #------------------------------- # Launch the Even Processing Transform #------------------------------- even_transform_id = tms_cli.create_transform(name='even_transform', in_subscription_id = even_subscription_id, out_streams={'even_plus1':streams[3]}, process_definition_id=basic_transform_definition_id, configuration={}) tms_cli.activate_transform(even_transform_id) #------------------------------- # Launch the Odd Processing Transform #------------------------------- odd_transform_id = tms_cli.create_transform(name='odd_transform', in_subscription_id = odd_subscription_id, out_streams={'odd_plus1':streams[4]}, process_definition_id=basic_transform_definition_id, configuration={}) tms_cli.activate_transform(odd_transform_id) #------------------------------- # Set up final subscribers #------------------------------- evenplus1_subscription_id = pubsub_cli.create_subscription( query=StreamQuery([streams[3]]), exchange_name='evenplus1_queue', name='EvenPlus1Subscription', description='EvenPlus1 SubscriptionDescription' ) oddplus1_subscription_id = pubsub_cli.create_subscription( query=StreamQuery([streams[4]]), exchange_name='oddplus1_queue', name='OddPlus1Subscription', description='OddPlus1 SubscriptionDescription' ) total_msg_count = 2 msgs = gevent.queue.Queue() def even1_message_received(message, headers): input = int(message.get('num')) assertions( (input % 2) ) # Assert it is odd (transform adds 1) msgs.put(True) def odd1_message_received(message, headers): input = int(message.get('num')) assertions(not (input % 2)) # Assert it is even msgs.put(True) subscriber_registrar = StreamSubscriberRegistrar(process=cc, container=cc) even_subscriber = subscriber_registrar.create_subscriber(exchange_name='evenplus1_queue', callback=even1_message_received) odd_subscriber = subscriber_registrar.create_subscriber(exchange_name='oddplus1_queue', callback=odd1_message_received) # Start subscribers even_subscriber.start() odd_subscriber.start() # Activate subscriptions pubsub_cli.activate_subscription(evenplus1_subscription_id) pubsub_cli.activate_subscription(oddplus1_subscription_id) #------------------------------- # Set up fake stream producer #------------------------------- pid = cc.spawn_process(name='dummy_process_for_test', module='pyon.ion.process', cls='SimpleProcess', config={}) dummy_process = cc.proc_manager.procs[pid] # Normally the user does not see or create the publisher, this is part of the containers business. # For the test we need to set it up explicitly publisher_registrar = StreamPublisherRegistrar(process=dummy_process, container=cc) stream_publisher = publisher_registrar.create_publisher(stream_id=streams[0]) #------------------------------- # Start test #------------------------------- # Publish a stream for i in xrange(total_msg_count): stream_publisher.publish({'num':str(i)}) time.sleep(0.5) for i in xrange(total_msg_count * 2): try: msgs.get() except Empty: assertions(False, "Failed to process all messages correctly.")
def test_integrated_transform(self): ''' This example script runs a chained three way transform: B A < C Where A is the even_odd transform (generates a stream of even and odd numbers from input) and B and C are the basic transforms that receive even and odd input ''' cc = self.container assertions = self.assertTrue pubsub_cli = PubsubManagementServiceClient(node=cc.node) rr_cli = ResourceRegistryServiceClient(node=cc.node) tms_cli = TransformManagementServiceClient(node=cc.node) #------------------------------- # Process Definition #------------------------------- # Create the process definition for the basic transform process_definition = IonObject(RT.ProcessDefinition, name='basic_transform_definition') process_definition.executable = { 'module': 'ion.processes.data.transforms.transform_example', 'class':'TransformExample' } basic_transform_definition_id, _ = rr_cli.create(process_definition) # Create The process definition for the TransformEvenOdd process_definition = IonObject(RT.ProcessDefinition, name='evenodd_transform_definition') process_definition.executable = { 'module': 'ion.processes.data.transforms.transform_example', 'class':'TransformEvenOdd' } evenodd_transform_definition_id, _ = rr_cli.create(process_definition) #------------------------------- # Streams #------------------------------- streams = [pubsub_cli.create_stream() for i in xrange(5)] #------------------------------- # Subscriptions #------------------------------- query = StreamQuery(stream_ids=[streams[0]]) input_subscription_id = pubsub_cli.create_subscription(query=query, exchange_name='input_queue') query = StreamQuery(stream_ids = [streams[1]]) # even output even_subscription_id = pubsub_cli.create_subscription(query=query, exchange_name='even_queue') query = StreamQuery(stream_ids = [streams[2]]) # odd output odd_subscription_id = pubsub_cli.create_subscription(query=query, exchange_name='odd_queue') #------------------------------- # Launch the EvenOdd Transform #------------------------------- evenodd_id = tms_cli.create_transform(name='even_odd', in_subscription_id=input_subscription_id, out_streams={'even':streams[1], 'odd':streams[2]}, process_definition_id=evenodd_transform_definition_id, configuration={}) tms_cli.activate_transform(evenodd_id) #------------------------------- # Launch the Even Processing Transform #------------------------------- even_transform_id = tms_cli.create_transform(name='even_transform', in_subscription_id = even_subscription_id, out_streams={'even_plus1':streams[3]}, process_definition_id=basic_transform_definition_id, configuration={}) tms_cli.activate_transform(even_transform_id) #------------------------------- # Launch the Odd Processing Transform #------------------------------- odd_transform_id = tms_cli.create_transform(name='odd_transform', in_subscription_id = odd_subscription_id, out_streams={'odd_plus1':streams[4]}, process_definition_id=basic_transform_definition_id, configuration={}) tms_cli.activate_transform(odd_transform_id) #------------------------------- # Set up final subscribers #------------------------------- evenplus1_subscription_id = pubsub_cli.create_subscription( query=StreamQuery([streams[3]]), exchange_name='evenplus1_queue', name='EvenPlus1Subscription', description='EvenPlus1 SubscriptionDescription' ) oddplus1_subscription_id = pubsub_cli.create_subscription( query=StreamQuery([streams[4]]), exchange_name='oddplus1_queue', name='OddPlus1Subscription', description='OddPlus1 SubscriptionDescription' ) total_msg_count = 2 msgs = gevent.queue.Queue() def even1_message_received(message, headers): input = int(message.get('num')) assertions( (input % 2) ) # Assert it is odd (transform adds 1) msgs.put(True) def odd1_message_received(message, headers): input = int(message.get('num')) assertions(not (input % 2)) # Assert it is even msgs.put(True) subscriber_registrar = StreamSubscriberRegistrar(process=cc, node=cc.node) even_subscriber = subscriber_registrar.create_subscriber(exchange_name='evenplus1_queue', callback=even1_message_received) odd_subscriber = subscriber_registrar.create_subscriber(exchange_name='oddplus1_queue', callback=odd1_message_received) # Start subscribers even_subscriber.start() odd_subscriber.start() # Activate subscriptions pubsub_cli.activate_subscription(evenplus1_subscription_id) pubsub_cli.activate_subscription(oddplus1_subscription_id) #------------------------------- # Set up fake stream producer #------------------------------- pid = cc.spawn_process(name='dummy_process_for_test', module='pyon.ion.process', cls='SimpleProcess', config={}) dummy_process = cc.proc_manager.procs[pid] # Normally the user does not see or create the publisher, this is part of the containers business. # For the test we need to set it up explicitly publisher_registrar = StreamPublisherRegistrar(process=dummy_process, node=cc.node) stream_publisher = publisher_registrar.create_publisher(stream_id=streams[0]) #------------------------------- # Start test #------------------------------- # Publish a stream for i in xrange(total_msg_count): stream_publisher.publish({'num':str(i)}) time.sleep(0.5) for i in xrange(total_msg_count * 2): try: msgs.get() except Empty: assertions(False, "Failed to process all messages correctly.")
def test_replay_integration(self): ''' Test full DM Services Integration ''' cc = self.container ### Every thing below here can be run as a script: pubsub_management_service = PubsubManagementServiceClient(node=cc.node) ingestion_management_service = IngestionManagementServiceClient(node=cc.node) dataset_management_service = DatasetManagementServiceClient(node=cc.node) data_retriever_service = DataRetrieverServiceClient(node=cc.node) resource_registry_service = ResourceRegistryServiceClient(node=cc.node) #------------------------------------------------------------------------------------------------------ # Datastore name #------------------------------------------------------------------------------------------------------ datastore_name = 'test_replay_integration' #------------------------------------------------------------------------------------------------------ # Spawn process #------------------------------------------------------------------------------------------------------ pid = cc.spawn_process(name='dummy_process_for_test', module='pyon.ion.process', cls='SimpleProcess', config={}) dummy_process = cc.proc_manager.procs[pid] #------------------------------------------------------------------------------------------------------ # Set up subscriber #------------------------------------------------------------------------------------------------------ # Normally the user does not see or create the publisher, this is part of the containers business. # For the test we need to set it up explicitly publisher_registrar = StreamPublisherRegistrar(process=dummy_process, node=cc.node) subscriber_registrar = StreamSubscriberRegistrar(process=cc, node=cc.node) #------------------------------------------------------------------------------------------------------ # Set up ingestion #------------------------------------------------------------------------------------------------------ # Configure ingestion using eight workers, ingesting to test_dm_integration datastore with the SCIDATA profile ingestion_configuration_id = ingestion_management_service.create_ingestion_configuration( exchange_point_id='science_data', couch_storage=CouchStorage(datastore_name=datastore_name, datastore_profile='SCIDATA'), hdf_storage=HdfStorage(), number_of_workers=1, ) ingestion_management_service.activate_ingestion_configuration( ingestion_configuration_id=ingestion_configuration_id) #------------------------------------------------------------------------------------------------------ # Grab the transforms acting as ingestion workers #------------------------------------------------------------------------------------------------------ transforms = [resource_registry_service.read(assoc.o) for assoc in resource_registry_service.find_associations(ingestion_configuration_id, PRED.hasTransform)] proc_1 = cc.proc_manager.procs[transforms[0].process_id] log.info("PROCESS 1: %s" % str(proc_1)) #------------------------------------------------------------------------------------------------------ # Set up the test hooks for the gevent event AsyncResult object #------------------------------------------------------------------------------------------------------ def ingestion_worker_received(message, headers): ar.set(message) proc_1.ingest_process_test_hook = ingestion_worker_received #------------------------------------------------------------------------------------------------------ # Set up the producers (CTD Simulators) #------------------------------------------------------------------------------------------------------ ctd_stream_def = ctd_stream_definition() stream_def_id = pubsub_management_service.create_stream_definition(container=ctd_stream_def, name='Junk definition') stream_id = pubsub_management_service.create_stream(stream_definition_id=stream_def_id) #------------------------------------------------------------------------------------------------------ # Set up the dataset config #------------------------------------------------------------------------------------------------------ dataset_id = dataset_management_service.create_dataset( stream_id=stream_id, datastore_name=datastore_name, view_name='datasets/stream_join_granule' ) dataset_config_id = ingestion_management_service.create_dataset_configuration( dataset_id = dataset_id, archive_data = True, archive_metadata = True, ingestion_configuration_id = ingestion_configuration_id ) #------------------------------------------------------------------------------------------------------ # Launch a ctd_publisher #------------------------------------------------------------------------------------------------------ publisher = publisher_registrar.create_publisher(stream_id=stream_id) #------------------------------------------------------------------------ # Create a packet and publish it #------------------------------------------------------------------------ ctd_packet = _create_packet(stream_id) published_hdfstring = ctd_packet.identifiables['ctd_data'].values publisher.publish(ctd_packet) #------------------------------------------------------------------------------------------------------ # Catch what the ingestion worker gets! Assert it is the same packet that was published! #------------------------------------------------------------------------------------------------------ packet = ar.get(timeout=2) #------------------------------------------------------------------------------------------------------ # Create subscriber to listen to the replays #------------------------------------------------------------------------------------------------------ replay_id, replay_stream_id = data_retriever_service.define_replay(dataset_id) query = StreamQuery(stream_ids=[replay_stream_id]) subscription_id = pubsub_management_service.create_subscription(query = query, exchange_name='replay_capture_point' ,name = 'replay_capture_point') # It is not required or even generally a good idea to use the subscription resource name as the queue name, but it makes things simple here # Normally the container creates and starts subscribers for you when a transform process is spawned subscriber = subscriber_registrar.create_subscriber(exchange_name='replay_capture_point', callback=_subscriber_call_back) subscriber.start() pubsub_management_service.activate_subscription(subscription_id) #------------------------------------------------------------------------------------------------------ # Start the replay #------------------------------------------------------------------------------------------------------ data_retriever_service.start_replay(replay_id) #------------------------------------------------------------------------------------------------------ # Get the hdf string from the captured stream in the replay #------------------------------------------------------------------------------------------------------ retrieved_hdf_string = ar2.get(timeout=2) ### Non scriptable portion of the test #------------------------------------------------------------------------------------------------------ # Assert that it matches the message we sent #------------------------------------------------------------------------------------------------------ self.assertEquals(packet.identifiables['stream_encoding'].sha1, ctd_packet.identifiables['stream_encoding'].sha1) self.assertEquals(retrieved_hdf_string, published_hdfstring)