Ejemplo n.º 1
0
    def test_unbind_exchange_subscription(self):

        event = gevent.event.Event()

        def message_received(message, headers):
            self.assertIn('test_unbind_exchange_subscription',message)
            self.assertFalse(event.is_set())
            event.set()


        subscriber = SimpleStreamSubscriber.new_subscriber(self.container,'another_queue',callback=message_received)
        subscriber.start()

        self.pubsub_cli.activate_subscription(self.exchange_subscription_id)

        self.ctd_stream1_publisher.publish('test_unbind_exchange_subscription')
        self.assertTrue(event.wait(10))
        event.clear()


        self.pubsub_cli.deactivate_subscription(self.exchange_subscription_id)


        self.ctd_stream2_publisher.publish('test_unbind_exchange_subscription')
        self.assertFalse(event.wait(2))

        subscriber.stop()
    def on_start(self):
        self.queue_name = self.CFG.get_safe("process.queue_name", self.id)
        self.value = self.CFG.get_safe("process.value", 0)

        self.subscriber = SimpleStreamSubscriber.new_subscriber(self.container, self.queue_name, self.recv_packet)
        self.subscriber.start()

        # Create the publisher that will publish the Alert message
        self.event_publisher = EventPublisher()
    def on_start(self): #pragma no cover
        self.queue_name = self.CFG.get_safe('process.queue_name','ingestion_queue')
        self.datastore_name = self.CFG.get_safe('process.datastore_name', 'datasets')


        self.subscriber = SimpleStreamSubscriber.new_subscriber(self.container,self.queue_name,self.consume)
        self.db = self.container.datastore_manager.get_datastore(self.datastore_name, DataStore.DS_PROFILE.SCIDATA)
        log.debug('Created datastore %s', self.datastore_name)
        self.subscriber.start()
Ejemplo n.º 4
0
    def test_stream_pub_sub(self):
        exchange_name = 'queue'
        exchange_point = 'test_exchagne'
        self.event = Event()
        def verify(m,h):
            self.event.set()

        sub = SimpleStreamSubscriber.new_subscriber(self.container,exchange_name, verify)
        sub.start()

        xn = self.container.ex_manager.create_xn_queue(exchange_name)
        xp = self.container.ex_manager.create_xp(exchange_point)
        xn.bind('stream_id.data', xp)

        pub = SimpleStreamPublisher.new_publisher(self.container, exchange_point,'stream_id')

        pub.publish('test')
        self.assertTrue(self.event.wait(10))
        sub.stop()
Ejemplo n.º 5
0
    def on_start(self):
        self.queue_name = self.CFG.get_safe('process.queue_name',self.id)

        self.subscriber = SimpleStreamSubscriber.new_subscriber(self.container, self.queue_name, self.recv_packet)
        self.subscriber.start()
Ejemplo n.º 6
0
    def test_update_stream_subscription(self):

        event = gevent.event.Event()

        def message_received(message, headers):
            self.assertIn('test_update_stream_subscription',message)
            self.assertFalse(event.is_set())
            event.set()

        subscriber = SimpleStreamSubscriber.new_subscriber(self.container,self.exchange_name,callback=message_received)
        subscriber.start()

        self.pubsub_cli.activate_subscription(self.ctd_subscription_id)


        self.ctd_stream1_publisher.publish('test_update_stream_subscription')
        self.assertTrue(event.wait(2))
        event.clear()

        self.ctd_stream2_publisher.publish('test_update_stream_subscription')
        self.assertTrue(event.wait(2))
        event.clear()



        # Update the subscription by removing a stream...
        subscription = self.pubsub_cli.read_subscription(self.ctd_subscription_id)
        stream_ids = list(subscription.query.stream_ids)
        stream_ids.remove(self.ctd_stream2_id)
        self.pubsub_cli.update_subscription(
            subscription_id=subscription._id,
            query=StreamQuery(stream_ids=stream_ids)
        )


        # Stream 2 is no longer received
        self.ctd_stream2_publisher.publish('test_update_stream_subscription')

        self.assertFalse(event.wait(0.5))


        # Stream 1 is as before
        self.ctd_stream1_publisher.publish('test_update_stream_subscription')
        self.assertTrue(event.wait(2))
        event.clear()


        # Now swith the active streams...

        # Update the subscription by removing a stream...
        self.pubsub_cli.update_subscription(
            subscription_id=self.ctd_subscription_id,
            query=StreamQuery([self.ctd_stream2_id])
        )


        # Stream 1 is no longer received
        self.ctd_stream1_publisher.publish('test_update_stream_subscription')
        self.assertFalse(event.wait(1))


        # Stream 2 is received
        self.ctd_stream2_publisher.publish('test_update_stream_subscription')
        self.assertTrue(event.wait(2))
        event.clear()




        subscriber.stop()
Ejemplo n.º 7
0
    def test_dm_end_2_end(self):
        # --------------------------------------------------------------------------------
        # Set up a stream and have a mock instrument (producer) send data
        # --------------------------------------------------------------------------------

        stream_id = self.pubsub_management.create_stream()

        self.launch_producer(stream_id)

        # --------------------------------------------------------------------------------
        # Start persisting the data on the stream
        # - Get the ingestion configuration from the resource registry
        # - Create the dataset
        # - call persist_data_stream to setup the subscription for the ingestion workers
        #   on the stream that you specify which causes the data to be persisted
        # --------------------------------------------------------------------------------

        ingest_config_id = self.get_ingestion_config()
        dataset_id = self.create_dataset()
        self.ingestion_management.persist_data_stream(
            stream_id=stream_id, ingestion_configuration_id=ingest_config_id, dataset_id=dataset_id
        )

        # --------------------------------------------------------------------------------
        # Now the granules are ingesting and persisted
        # --------------------------------------------------------------------------------

        self.wait_until_we_have_enough_granules(dataset_id, 4)

        # --------------------------------------------------------------------------------
        # Now get the data in one chunk using an RPC Call to start_retreive
        # --------------------------------------------------------------------------------

        replay_data = self.data_retriever.retrieve(dataset_id)
        self.assertIsInstance(replay_data, Granule)

        # --------------------------------------------------------------------------------
        # Now to try the streamed approach
        # --------------------------------------------------------------------------------

        replay_id, stream_id = self.data_retriever.define_replay(dataset_id)

        # --------------------------------------------------------------------------------
        # Create the listening endpoint for the the retriever to talk to
        # --------------------------------------------------------------------------------
        xp = self.container.ex_manager.create_xp(self.exchange_point_name)
        xn = self.container.ex_manager.create_xn_queue(self.exchange_space_name)
        xn.bind("%s.data" % stream_id, xp)
        subscriber = SimpleStreamSubscriber.new_subscriber(
            self.container, self.exchange_space_name, self.validate_granule_subscription
        )
        subscriber.start()

        self.data_retriever.start_replay(replay_id)

        fail = False
        try:
            self.event.wait(10)
        except gevent.Timeout:
            fail = True

        subscriber.stop()

        self.assertTrue(not fail, "Failed to validate the data.")