def publish_hifi(self, stream_id, offset=0):
pub = SimpleStreamPublisher.new_publisher(self.container, self.exchange_point_name, stream_id)
black_box = CoverageCraft()
black_box.rdt["time"] = np.arange(10) + (offset * 10)
black_box.rdt["temp"] = (np.arange(10) + (offset * 10)) * 2
granule = black_box.to_granule()
pub.publish(granule)
def setUp(self):
logging.disable(logging.ERROR)
self._start_container()
self.container.start_rel_from_url('res/deploy/r2dm.yml')
logging.disable(logging.NOTSET)
self.pubsub_cli = PubsubManagementServiceClient(node=self.container.node)
self.ctd_stream1_id = self.pubsub_cli.create_stream(name="SampleStream1",
description="Sample Stream 1 Description")
self.ctd_stream2_id = self.pubsub_cli.create_stream(name="SampleStream2",
description="Sample Stream 2 Description")
# Make a subscription to two input streams
self.exchange_name = "a_queue"
self.exchange_point = 'an_exchange'
query = StreamQuery([self.ctd_stream1_id, self.ctd_stream2_id])
self.ctd_subscription_id = self.pubsub_cli.create_subscription(query=query,
exchange_name=self.exchange_name,
exchange_point=self.exchange_point,
name="SampleSubscription",
description="Sample Subscription Description")
# Make a subscription to all streams on an exchange point
self.exchange2_name = "another_queue"
query = ExchangeQuery()
self.exchange_subscription_id = self.pubsub_cli.create_subscription(query=query,
exchange_name=self.exchange2_name,
exchange_point=self.exchange_point,
name="SampleExchangeSubscription",
description="Sample Exchange Subscription Description")
# Normally the user does not see or create the publisher, this is part of the containers business.
# For the test we need to set it up explicitly
self.ctd_stream1_publisher = SimpleStreamPublisher.new_publisher(self.container, self.exchange_point, stream_id=self.ctd_stream1_id)
self.ctd_stream2_publisher = SimpleStreamPublisher.new_publisher(self.container, self.exchange_point, stream_id=self.ctd_stream2_id)
self.purge_queues()
def test_coverage_ingest(self):
stream_id = self.pubsub_management.create_stream()
dataset_id = self.create_dataset()
# I freaking hate this bug
self.get_datastore(dataset_id)
ingestion_config_id = self.get_ingestion_config()
self.ingestion_management.persist_data_stream(
stream_id=stream_id, ingestion_configuration_id=ingestion_config_id, dataset_id=dataset_id
)
black_box = CoverageCraft()
black_box.rdt["time"] = np.arange(20)
black_box.rdt["temp"] = np.random.random(20) * 10
black_box.sync_with_granule()
granule = black_box.to_granule()
publisher = SimpleStreamPublisher.new_publisher(self.container, self.exchange_point_name, stream_id)
publisher.publish(granule)
self.wait_until_we_have_enough_granules(dataset_id, 1)
coverage = DatasetManagementService._get_coverage(dataset_id)
black_box = CoverageCraft(coverage)
black_box.sync_rdt_with_coverage()
comp = black_box.rdt["time"] == np.arange(20)
self.assertTrue(comp.all())
black_box = CoverageCraft()
black_box.rdt["time"] = np.arange(20) + 20
black_box.rdt["temp"] = np.random.random(20) * 10
black_box.sync_with_granule()
granule = black_box.to_granule()
publisher.publish(granule)
self.wait_until_we_have_enough_granules(dataset_id, 2)
coverage = DatasetManagementService._get_coverage(dataset_id)
black_box = CoverageCraft(coverage)
black_box.sync_rdt_with_coverage()
comp = black_box.rdt["time"] == np.arange(40)
self.assertTrue(comp.all())
granule = self.data_retriever.retrieve(dataset_id)
black_box = CoverageCraft()
black_box.sync_rdt_with_granule(granule)
comp = black_box.rdt["time"] == np.arange(40)
self.assertTrue(comp.all())
def test_stream_pub_sub(self):
exchange_name = 'queue'
exchange_point = 'test_exchagne'
self.event = Event()
def verify(m,h):
self.event.set()
sub = SimpleStreamSubscriber.new_subscriber(self.container,exchange_name, verify)
sub.start()
xn = self.container.ex_manager.create_xn_queue(exchange_name)
xp = self.container.ex_manager.create_xp(exchange_point)
xn.bind('stream_id.data', xp)
pub = SimpleStreamPublisher.new_publisher(self.container, exchange_point,'stream_id')
pub.publish('test')
self.assertTrue(self.event.wait(10))
sub.stop()
def on_start(self):
self.exchange_point = self.CFG.get_safe('process.exchange_point', '')
self.publisher = SimpleStreamPublisher.new_publisher(self.container, self.exchange_point,'')
def test_stream_processing(self):
'''
Test that streams are processed by the transforms according to a provided algorithm
'''
#todo: In this simple implementation, we are checking if the stream has the word, PUBLISH,
#todo(contd) and if the word VALUE=<number> exists and that number is less than something
#todo later on we are going to use complex algorithms to make this prototype powerful
#-------------------------------------------------------------------------------------
# Start a subscriber to listen for an alert event from the Stream Alert Transform
#-------------------------------------------------------------------------------------
queue = gevent.queue.Queue()
def event_received(message, headers):
queue.put(message)
event_subscriber = EventSubscriber( origin="StreamAlertTransform",
event_type="DeviceEvent",
callback=event_received)
event_subscriber.start()
#-------------------------------------------------------------------------------------
# The configuration for the Stream Alert Transform... set up the event types to listen to
#-------------------------------------------------------------------------------------
config = {
'process':{
'queue_name': 'a_queue',
'value': 10
}
}
#-------------------------------------------------------------------------------------
# Create the process
#-------------------------------------------------------------------------------------
pid = TransformPrototypeIntTest.create_process( name= 'transform_data_process',
module='ion.processes.data.transforms.event_alert_transform',
class_name='StreamAlertTransform',
configuration= config)
self.assertIsNotNone(pid)
#-------------------------------------------------------------------------------------
# Publish streams and make assertions about alerts
#-------------------------------------------------------------------------------------
exchange_name = 'a_queue'
exchange_point = 'test_exchange'
xn = self.container.ex_manager.create_xn_queue(exchange_name)
xp = self.container.ex_manager.create_xp(exchange_point)
xn.bind('stream_id.data', xp)
pub = SimpleStreamPublisher.new_publisher(self.container, exchange_point,'stream_id')
message = "A dummy example message containing the word PUBLISH, and with VALUE = 5 . This message" + \
" will trigger an alert event from the StreamAlertTransform"
pub.publish(message)
gevent.sleep(4)
event = queue.get()
self.assertEquals(event.type_, "DeviceEvent")
self.assertEquals(event.origin, "StreamAlertTransform")
self.purge_queues(exchange_name)
