def test_unbind_exchange_subscription(self):
event = gevent.event.Event()
def message_received(message, headers):
self.assertIn('test_unbind_exchange_subscription',message)
self.assertFalse(event.is_set())
event.set()
subscriber = SimpleStreamSubscriber.new_subscriber(self.container,'another_queue',callback=message_received)
subscriber.start()
self.pubsub_cli.activate_subscription(self.exchange_subscription_id)
self.ctd_stream1_publisher.publish('test_unbind_exchange_subscription')
self.assertTrue(event.wait(10))
event.clear()
self.pubsub_cli.deactivate_subscription(self.exchange_subscription_id)
self.ctd_stream2_publisher.publish('test_unbind_exchange_subscription')
self.assertFalse(event.wait(2))
subscriber.stop()
def on_start(self):
self.queue_name = self.CFG.get_safe("process.queue_name", self.id)
self.value = self.CFG.get_safe("process.value", 0)
self.subscriber = SimpleStreamSubscriber.new_subscriber(self.container, self.queue_name, self.recv_packet)
self.subscriber.start()
# Create the publisher that will publish the Alert message
self.event_publisher = EventPublisher()
def on_start(self): #pragma no cover
self.queue_name = self.CFG.get_safe('process.queue_name','ingestion_queue')
self.datastore_name = self.CFG.get_safe('process.datastore_name', 'datasets')
self.subscriber = SimpleStreamSubscriber.new_subscriber(self.container,self.queue_name,self.consume)
self.db = self.container.datastore_manager.get_datastore(self.datastore_name, DataStore.DS_PROFILE.SCIDATA)
log.debug('Created datastore %s', self.datastore_name)
self.subscriber.start()
def test_stream_pub_sub(self):
exchange_name = 'queue'
exchange_point = 'test_exchagne'
self.event = Event()
def verify(m,h):
self.event.set()
sub = SimpleStreamSubscriber.new_subscriber(self.container,exchange_name, verify)
sub.start()
xn = self.container.ex_manager.create_xn_queue(exchange_name)
xp = self.container.ex_manager.create_xp(exchange_point)
xn.bind('stream_id.data', xp)
pub = SimpleStreamPublisher.new_publisher(self.container, exchange_point,'stream_id')
pub.publish('test')
self.assertTrue(self.event.wait(10))
sub.stop()
def on_start(self):
self.queue_name = self.CFG.get_safe('process.queue_name',self.id)
self.subscriber = SimpleStreamSubscriber.new_subscriber(self.container, self.queue_name, self.recv_packet)
self.subscriber.start()
def test_update_stream_subscription(self):
event = gevent.event.Event()
def message_received(message, headers):
self.assertIn('test_update_stream_subscription',message)
self.assertFalse(event.is_set())
event.set()
subscriber = SimpleStreamSubscriber.new_subscriber(self.container,self.exchange_name,callback=message_received)
subscriber.start()
self.pubsub_cli.activate_subscription(self.ctd_subscription_id)
self.ctd_stream1_publisher.publish('test_update_stream_subscription')
self.assertTrue(event.wait(2))
event.clear()
self.ctd_stream2_publisher.publish('test_update_stream_subscription')
self.assertTrue(event.wait(2))
event.clear()
# Update the subscription by removing a stream...
subscription = self.pubsub_cli.read_subscription(self.ctd_subscription_id)
stream_ids = list(subscription.query.stream_ids)
stream_ids.remove(self.ctd_stream2_id)
self.pubsub_cli.update_subscription(
subscription_id=subscription._id,
query=StreamQuery(stream_ids=stream_ids)
)
# Stream 2 is no longer received
self.ctd_stream2_publisher.publish('test_update_stream_subscription')
self.assertFalse(event.wait(0.5))
# Stream 1 is as before
self.ctd_stream1_publisher.publish('test_update_stream_subscription')
self.assertTrue(event.wait(2))
event.clear()
# Now swith the active streams...
# Update the subscription by removing a stream...
self.pubsub_cli.update_subscription(
subscription_id=self.ctd_subscription_id,
query=StreamQuery([self.ctd_stream2_id])
)
# Stream 1 is no longer received
self.ctd_stream1_publisher.publish('test_update_stream_subscription')
self.assertFalse(event.wait(1))
# Stream 2 is received
self.ctd_stream2_publisher.publish('test_update_stream_subscription')
self.assertTrue(event.wait(2))
event.clear()
subscriber.stop()
def test_dm_end_2_end(self):
# --------------------------------------------------------------------------------
# Set up a stream and have a mock instrument (producer) send data
# --------------------------------------------------------------------------------
stream_id = self.pubsub_management.create_stream()
self.launch_producer(stream_id)
# --------------------------------------------------------------------------------
# Start persisting the data on the stream
# - Get the ingestion configuration from the resource registry
# - Create the dataset
# - call persist_data_stream to setup the subscription for the ingestion workers
# on the stream that you specify which causes the data to be persisted
# --------------------------------------------------------------------------------
ingest_config_id = self.get_ingestion_config()
dataset_id = self.create_dataset()
self.ingestion_management.persist_data_stream(
stream_id=stream_id, ingestion_configuration_id=ingest_config_id, dataset_id=dataset_id
)
# --------------------------------------------------------------------------------
# Now the granules are ingesting and persisted
# --------------------------------------------------------------------------------
self.wait_until_we_have_enough_granules(dataset_id, 4)
# --------------------------------------------------------------------------------
# Now get the data in one chunk using an RPC Call to start_retreive
# --------------------------------------------------------------------------------
replay_data = self.data_retriever.retrieve(dataset_id)
self.assertIsInstance(replay_data, Granule)
# --------------------------------------------------------------------------------
# Now to try the streamed approach
# --------------------------------------------------------------------------------
replay_id, stream_id = self.data_retriever.define_replay(dataset_id)
# --------------------------------------------------------------------------------
# Create the listening endpoint for the the retriever to talk to
# --------------------------------------------------------------------------------
xp = self.container.ex_manager.create_xp(self.exchange_point_name)
xn = self.container.ex_manager.create_xn_queue(self.exchange_space_name)
xn.bind("%s.data" % stream_id, xp)
subscriber = SimpleStreamSubscriber.new_subscriber(
self.container, self.exchange_space_name, self.validate_granule_subscription
)
subscriber.start()
self.data_retriever.start_replay(replay_id)
fail = False
try:
self.event.wait(10)
except gevent.Timeout:
fail = True
subscriber.stop()
self.assertTrue(not fail, "Failed to validate the data.")
