def publish_hifi(self, stream_id, offset=0): pub = SimpleStreamPublisher.new_publisher(self.container, self.exchange_point_name, stream_id) black_box = CoverageCraft() black_box.rdt["time"] = np.arange(10) + (offset * 10) black_box.rdt["temp"] = (np.arange(10) + (offset * 10)) * 2 granule = black_box.to_granule() pub.publish(granule)
def setUp(self): logging.disable(logging.ERROR) self._start_container() self.container.start_rel_from_url('res/deploy/r2dm.yml') logging.disable(logging.NOTSET) self.pubsub_cli = PubsubManagementServiceClient(node=self.container.node) self.ctd_stream1_id = self.pubsub_cli.create_stream(name="SampleStream1", description="Sample Stream 1 Description") self.ctd_stream2_id = self.pubsub_cli.create_stream(name="SampleStream2", description="Sample Stream 2 Description") # Make a subscription to two input streams self.exchange_name = "a_queue" self.exchange_point = 'an_exchange' query = StreamQuery([self.ctd_stream1_id, self.ctd_stream2_id]) self.ctd_subscription_id = self.pubsub_cli.create_subscription(query=query, exchange_name=self.exchange_name, exchange_point=self.exchange_point, name="SampleSubscription", description="Sample Subscription Description") # Make a subscription to all streams on an exchange point self.exchange2_name = "another_queue" query = ExchangeQuery() self.exchange_subscription_id = self.pubsub_cli.create_subscription(query=query, exchange_name=self.exchange2_name, exchange_point=self.exchange_point, name="SampleExchangeSubscription", description="Sample Exchange Subscription Description") # Normally the user does not see or create the publisher, this is part of the containers business. # For the test we need to set it up explicitly self.ctd_stream1_publisher = SimpleStreamPublisher.new_publisher(self.container, self.exchange_point, stream_id=self.ctd_stream1_id) self.ctd_stream2_publisher = SimpleStreamPublisher.new_publisher(self.container, self.exchange_point, stream_id=self.ctd_stream2_id) self.purge_queues()
def test_coverage_ingest(self): stream_id = self.pubsub_management.create_stream() dataset_id = self.create_dataset() # I freaking hate this bug self.get_datastore(dataset_id) ingestion_config_id = self.get_ingestion_config() self.ingestion_management.persist_data_stream( stream_id=stream_id, ingestion_configuration_id=ingestion_config_id, dataset_id=dataset_id ) black_box = CoverageCraft() black_box.rdt["time"] = np.arange(20) black_box.rdt["temp"] = np.random.random(20) * 10 black_box.sync_with_granule() granule = black_box.to_granule() publisher = SimpleStreamPublisher.new_publisher(self.container, self.exchange_point_name, stream_id) publisher.publish(granule) self.wait_until_we_have_enough_granules(dataset_id, 1) coverage = DatasetManagementService._get_coverage(dataset_id) black_box = CoverageCraft(coverage) black_box.sync_rdt_with_coverage() comp = black_box.rdt["time"] == np.arange(20) self.assertTrue(comp.all()) black_box = CoverageCraft() black_box.rdt["time"] = np.arange(20) + 20 black_box.rdt["temp"] = np.random.random(20) * 10 black_box.sync_with_granule() granule = black_box.to_granule() publisher.publish(granule) self.wait_until_we_have_enough_granules(dataset_id, 2) coverage = DatasetManagementService._get_coverage(dataset_id) black_box = CoverageCraft(coverage) black_box.sync_rdt_with_coverage() comp = black_box.rdt["time"] == np.arange(40) self.assertTrue(comp.all()) granule = self.data_retriever.retrieve(dataset_id) black_box = CoverageCraft() black_box.sync_rdt_with_granule(granule) comp = black_box.rdt["time"] == np.arange(40) self.assertTrue(comp.all())
def test_stream_pub_sub(self): exchange_name = 'queue' exchange_point = 'test_exchagne' self.event = Event() def verify(m,h): self.event.set() sub = SimpleStreamSubscriber.new_subscriber(self.container,exchange_name, verify) sub.start() xn = self.container.ex_manager.create_xn_queue(exchange_name) xp = self.container.ex_manager.create_xp(exchange_point) xn.bind('stream_id.data', xp) pub = SimpleStreamPublisher.new_publisher(self.container, exchange_point,'stream_id') pub.publish('test') self.assertTrue(self.event.wait(10)) sub.stop()
def on_start(self): self.exchange_point = self.CFG.get_safe('process.exchange_point', '') self.publisher = SimpleStreamPublisher.new_publisher(self.container, self.exchange_point,'')
def test_stream_processing(self): ''' Test that streams are processed by the transforms according to a provided algorithm ''' #todo: In this simple implementation, we are checking if the stream has the word, PUBLISH, #todo(contd) and if the word VALUE=<number> exists and that number is less than something #todo later on we are going to use complex algorithms to make this prototype powerful #------------------------------------------------------------------------------------- # Start a subscriber to listen for an alert event from the Stream Alert Transform #------------------------------------------------------------------------------------- queue = gevent.queue.Queue() def event_received(message, headers): queue.put(message) event_subscriber = EventSubscriber( origin="StreamAlertTransform", event_type="DeviceEvent", callback=event_received) event_subscriber.start() #------------------------------------------------------------------------------------- # The configuration for the Stream Alert Transform... set up the event types to listen to #------------------------------------------------------------------------------------- config = { 'process':{ 'queue_name': 'a_queue', 'value': 10 } } #------------------------------------------------------------------------------------- # Create the process #------------------------------------------------------------------------------------- pid = TransformPrototypeIntTest.create_process( name= 'transform_data_process', module='ion.processes.data.transforms.event_alert_transform', class_name='StreamAlertTransform', configuration= config) self.assertIsNotNone(pid) #------------------------------------------------------------------------------------- # Publish streams and make assertions about alerts #------------------------------------------------------------------------------------- exchange_name = 'a_queue' exchange_point = 'test_exchange' xn = self.container.ex_manager.create_xn_queue(exchange_name) xp = self.container.ex_manager.create_xp(exchange_point) xn.bind('stream_id.data', xp) pub = SimpleStreamPublisher.new_publisher(self.container, exchange_point,'stream_id') message = "A dummy example message containing the word PUBLISH, and with VALUE = 5 . This message" + \ " will trigger an alert event from the StreamAlertTransform" pub.publish(message) gevent.sleep(4) event = queue.get() self.assertEquals(event.type_, "DeviceEvent") self.assertEquals(event.origin, "StreamAlertTransform") self.purge_queues(exchange_name)