Example #1
0
    def setUp(self):

        mock_clients = self._create_service_mock('ingestion_management')

        self.ingestion_management = IngestionManagementService()

        self.ingestion_management.clients = mock_clients

        self.rr_create = mock_clients.resource_registry.create
        self.rr_read = mock_clients.resource_registry.read
        self.rr_update = mock_clients.resource_registry.update
        self.rr_delete = mock_clients.resource_registry.delete
        self.rr_find_objs = mock_clients.resource_registry.find_objects
        self.rr_find_assocs = mock_clients.resource_registry.find_associations
        self.rr_find_res = mock_clients.resource_registry.find_resources
        self.rr_create_assoc = mock_clients.resource_registry.create_association
        self.rr_find_subjects = mock_clients.resource_registry.find_subjects
        self.rr_del_assoc = mock_clients.resource_registry.delete_association
        self.pubsub_create_sub = mock_clients.pubsub_management.create_subscription
        self.pubsub_read = mock_clients.pubsub_management.read_stream
        self.pubsub_is_persisted = mock_clients.pubsub_management.is_persisted
        self.pubsub_del_sub = mock_clients.pubsub_management.delete_subscription
        self.pubsub_act_sub = mock_clients.pubsub_management.activate_subscription
        self.pubsub_deact_sub = mock_clients.pubsub_management.deactivate_subscription
        self.dataset_create = mock_clients.dataset_management.create_dataset
    def setUp(self):
        
        mock_clients = self._create_service_mock('ingestion_management')

        self.ingestion_management = IngestionManagementService()

        self.ingestion_management.clients = mock_clients

        self.rr_create = mock_clients.resource_registry.create
        self.rr_read = mock_clients.resource_registry.read
        self.rr_update = mock_clients.resource_registry.update
        self.rr_delete = mock_clients.resource_registry.delete
        self.rr_find_objs = mock_clients.resource_registry.find_objects
        self.rr_find_assocs = mock_clients.resource_registry.find_associations
        self.rr_find_res = mock_clients.resource_registry.find_resources
        self.rr_create_assoc = mock_clients.resource_registry.create_association
        self.rr_find_subjects = mock_clients.resource_registry.find_subjects
        self.rr_del_assoc =  mock_clients.resource_registry.delete_association
        self.pubsub_create_sub = mock_clients.pubsub_management.create_subscription
        self.pubsub_read = mock_clients.pubsub_management.read_stream 
        self.pubsub_is_persisted = mock_clients.pubsub_management.is_persisted
        self.pubsub_del_sub = mock_clients.pubsub_management.delete_subscription
        self.pubsub_act_sub = mock_clients.pubsub_management.activate_subscription
        self.pubsub_deact_sub = mock_clients.pubsub_management.deactivate_subscription
        self.dataset_create = mock_clients.dataset_management.create_dataset
Example #3
0
class IngestionManagementUnitTest(PyonTestCase):
    def setUp(self):

        mock_clients = self._create_service_mock('ingestion_management')

        self.ingestion_management = IngestionManagementService()

        self.ingestion_management.clients = mock_clients

        self.rr_create = mock_clients.resource_registry.create
        self.rr_read = mock_clients.resource_registry.read
        self.rr_update = mock_clients.resource_registry.update
        self.rr_delete = mock_clients.resource_registry.delete
        self.rr_find_objs = mock_clients.resource_registry.find_objects
        self.rr_find_assocs = mock_clients.resource_registry.find_associations
        self.rr_find_res = mock_clients.resource_registry.find_resources
        self.rr_create_assoc = mock_clients.resource_registry.create_association
        self.rr_find_subjects = mock_clients.resource_registry.find_subjects
        self.rr_del_assoc = mock_clients.resource_registry.delete_association
        self.pubsub_create_sub = mock_clients.pubsub_management.create_subscription
        self.pubsub_read = mock_clients.pubsub_management.read_stream
        self.pubsub_is_persisted = mock_clients.pubsub_management.is_persisted
        self.pubsub_del_sub = mock_clients.pubsub_management.delete_subscription
        self.pubsub_act_sub = mock_clients.pubsub_management.activate_subscription
        self.pubsub_deact_sub = mock_clients.pubsub_management.deactivate_subscription
        self.dataset_create = mock_clients.dataset_management.create_dataset

    def test_read_ingestion(self):
        testval = 'test'
        self.rr_read.return_value = testval
        retval = self.ingestion_management.read_ingestion_configuration(
            'testconfig')
        self.assertTrue(retval == testval)

    def test_update_ingestion(self):
        testval = {0: 0}
        self.ingestion_management.update_ingestion_configuration(testval)
        self.rr_update.assert_called_once_with(testval)

    def test_create_ingestion(self):
        testval = 'config_id'
        self.rr_create.return_value = (testval, '0')

        queue = IngestionQueue()
        retval = self.ingestion_management.create_ingestion_configuration(
            name='blah', exchange_point_id='test', queues=[queue])

        self.assertTrue(retval == testval)
        self.assertTrue(self.rr_create.call_count == 1)

    def test_delete_ingestion(self):
        testval = DotDict()
        testval.o = 'sub_id'
        self.rr_find_assocs.return_value = [testval]

        self.ingestion_management.delete_ingestion_configuration('test')

        self.rr_del_assoc.assert_called_once_with(testval)
        self.pubsub_del_sub.assert_called_once_with(testval.o)

        self.rr_delete.assert_called_once_with('test')

    def test_persist_data_stream(self):
        config = IngestionConfiguration()

        self.ingestion_management.read_ingestion_configuration = Mock()
        self.ingestion_management.read_ingestion_configuration.return_value = config

        self.ingestion_management.is_persisted = Mock()
        self.ingestion_management.is_persisted.return_value = False

        self.ingestion_management.setup_queues = Mock()
        self.ingestion_management.setup_queues.return_value = True

        self.pubsub_read.return_value = DotDict(persisted=False)

        retval = self.ingestion_management.persist_data_stream(
            'stream_id', 'config_id', 'dataset_id')

        self.assertEquals(retval, 'dataset_id')

    @unittest.skip('Need to refactor')
    def test_setup_queues(self):
        ingestion_config = IngestionConfiguration()
        ingestion_config.queues = [IngestionQueue()]
        setattr(ingestion_config, '_id', 'config_id')

        self.pubsub_create_sub.return_value = 'subscription_id'

        self.ingestion_management._existing_dataset = Mock()

        retval = self.ingestion_management.setup_queues(
            ingestion_config, 'stream_id', 'dataset_id')

        self.assertTrue(retval)

        self.ingestion_management._existing_dataset.assert_called_once_with(
            'stream_id', 'dataset_id')

    def test_unpersist_data_stream(self):

        self.rr_find_objs.return_value = [('sub'), ('assoc')]
        self.rr_find_assocs.return_value = ['assoc']
        self.rr_find_subjects.return_value = ([], [])

        self.ingestion_management.unpersist_data_stream(
            'stream_id', 'ingestion_id')

        self.assertTrue(self.pubsub_deact_sub.call_count)
        self.assertTrue(self.rr_del_assoc.call_count)
        self.assertTrue(self.pubsub_del_sub.call_count)

    def test_determine_queue(self):
        pass  #unimplemented

    def test_list_ingestion(self):
        testval = (['resource'], ['other'])
        self.rr_find_res.return_value = testval
        retval = self.ingestion_management.list_ingestion_configurations(
            id_only=True)
        self.assertTrue(retval == testval[0])
        self.assertTrue(self.rr_find_res.call_count)

    def test_is_persisted(self):
        self.pubsub_is_persisted.return_value = True

        retval = self.ingestion_management.is_persisted('stream_id')

        self.assertEquals(retval, True)
class IngestionManagementUnitTest(PyonTestCase):
    def setUp(self):

        mock_clients = self._create_service_mock("ingestion_management")

        self.ingestion_management = IngestionManagementService()

        self.ingestion_management.clients = mock_clients

        self.rr_create = mock_clients.resource_registry.create
        self.rr_read = mock_clients.resource_registry.read
        self.rr_update = mock_clients.resource_registry.update
        self.rr_delete = mock_clients.resource_registry.delete
        self.rr_find_objs = mock_clients.resource_registry.find_objects
        self.rr_find_assocs = mock_clients.resource_registry.find_associations
        self.rr_find_res = mock_clients.resource_registry.find_resources
        self.rr_create_assoc = mock_clients.resource_registry.create_association
        self.rr_find_subjects = mock_clients.resource_registry.find_subjects
        self.rr_del_assoc = mock_clients.resource_registry.delete_association
        self.pubsub_create_sub = mock_clients.pubsub_management.create_subscription
        self.pubsub_read = mock_clients.pubsub_management.read_stream
        self.pubsub_del_sub = mock_clients.pubsub_management.delete_subscription
        self.pubsub_act_sub = mock_clients.pubsub_management.activate_subscription
        self.pubsub_deact_sub = mock_clients.pubsub_management.deactivate_subscription
        self.dataset_create = mock_clients.dataset_management.create_dataset

    def test_read_ingestion(self):
        testval = "test"
        self.rr_read.return_value = testval
        retval = self.ingestion_management.read_ingestion_configuration("testconfig")
        self.assertTrue(retval == testval)

    def test_update_ingestion(self):
        testval = {0: 0}
        self.ingestion_management.update_ingestion_configuration(testval)
        self.rr_update.assert_called_once_with(testval)

    def test_create_ingestion(self):
        testval = "config_id"
        self.rr_create.return_value = (testval, "0")

        queue = IngestionQueue()
        retval = self.ingestion_management.create_ingestion_configuration(
            name="blah", exchange_point_id="test", queues=[queue]
        )

        self.assertTrue(retval == testval)
        self.assertTrue(self.rr_create.call_count == 1)

    def test_delete_ingestion(self):
        testval = DotDict()
        testval.o = "sub_id"
        self.rr_find_assocs.return_value = [testval]

        self.ingestion_management.delete_ingestion_configuration("test")

        self.rr_del_assoc.assert_called_once_with(testval)
        self.pubsub_del_sub.assert_called_once_with(testval.o)

        self.rr_delete.assert_called_once_with("test")

    def test_persist_data_stream(self):
        ingestval = DotDict()
        ingestval.queues = None
        ingestval.exchange_point = "test"

        queueval = DotDict()
        queueval.name = "test"
        queueval.datastore_name = "test"

        testval = "dataset_id"

        self.ingestion_management.read_ingestion_configuration = Mock()
        self.ingestion_management.read_ingestion_configuration.return_value = ingestval
        self.ingestion_management._determine_queue = Mock()
        self.ingestion_management._determine_queue.return_value = queueval
        self.pubsub_read.return_value = DotDict({"persisted": False})

        self.ingestion_management._existing_dataset = Mock()

        retval = self.ingestion_management.persist_data_stream("stream_id", "config_id", "dataset_id")

        self.assertTrue(self.pubsub_act_sub.call_count)
        self.assertTrue(self.pubsub_create_sub.call_count)
        self.assertTrue(self.rr_create_assoc.call_count)

        self.assertTrue(retval == testval)

    def test_unpersist_data_stream(self):

        self.rr_find_objs.return_value = [("sub"), ("assoc")]
        self.rr_find_assocs.return_value = ["assoc"]
        self.rr_find_subjects.return_value = ([], [])

        self.ingestion_management.unpersist_data_stream("stream_id", "ingestion_id")

        self.assertTrue(self.pubsub_deact_sub.call_count)
        self.assertTrue(self.rr_del_assoc.call_count)
        self.assertTrue(self.pubsub_del_sub.call_count)

    def test_determine_queue(self):
        pass  # unimplemented

    def test_list_ingestion(self):
        testval = (["resource"], ["other"])
        self.rr_find_res.return_value = testval
        retval = self.ingestion_management.list_ingestion_configurations(id_only=True)
        self.assertTrue(retval == testval[0])
        self.assertTrue(self.rr_find_res.call_count)

    def test_is_persisted(self):
        stream = DotDict()
        stream.persisted = True
        self.pubsub_read.return_value = stream

        retval = self.ingestion_management.is_persisted("stream_id")

        self.assertEquals(retval, True)
class IngestionManagementUnitTest(PyonTestCase):
    def setUp(self):
        
        mock_clients = self._create_service_mock('ingestion_management')

        self.ingestion_management = IngestionManagementService()

        self.ingestion_management.clients = mock_clients

        self.rr_create = mock_clients.resource_registry.create
        self.rr_read = mock_clients.resource_registry.read
        self.rr_update = mock_clients.resource_registry.update
        self.rr_delete = mock_clients.resource_registry.delete
        self.rr_find_objs = mock_clients.resource_registry.find_objects
        self.rr_find_assocs = mock_clients.resource_registry.find_associations
        self.rr_find_res = mock_clients.resource_registry.find_resources
        self.rr_create_assoc = mock_clients.resource_registry.create_association
        self.rr_find_subjects = mock_clients.resource_registry.find_subjects
        self.rr_del_assoc =  mock_clients.resource_registry.delete_association
        self.pubsub_create_sub = mock_clients.pubsub_management.create_subscription
        self.pubsub_read = mock_clients.pubsub_management.read_stream 
        self.pubsub_is_persisted = mock_clients.pubsub_management.is_persisted
        self.pubsub_del_sub = mock_clients.pubsub_management.delete_subscription
        self.pubsub_act_sub = mock_clients.pubsub_management.activate_subscription
        self.pubsub_deact_sub = mock_clients.pubsub_management.deactivate_subscription
        self.dataset_create = mock_clients.dataset_management.create_dataset

    def test_read_ingestion(self):
        testval = 'test'
        self.rr_read.return_value = testval
        retval = self.ingestion_management.read_ingestion_configuration('testconfig')
        self.assertTrue(retval == testval)
        
    def test_update_ingestion(self):
        testval = {0:0}
        self.ingestion_management.update_ingestion_configuration(testval)
        self.rr_update.assert_called_once_with(testval)

    def test_create_ingestion(self):
        testval = 'config_id'
        self.rr_create.return_value = (testval, '0')

        queue = IngestionQueue()
        retval = self.ingestion_management.create_ingestion_configuration(name='blah', exchange_point_id='test', queues=[queue])

        self.assertTrue(retval == testval)
        self.assertTrue(self.rr_create.call_count == 1)

    def test_delete_ingestion(self):
        testval = DotDict()
        testval.o = 'sub_id'
        self.rr_find_assocs.return_value = [testval]

        self.ingestion_management.delete_ingestion_configuration('test')

        self.rr_del_assoc.assert_called_once_with(testval)
        self.pubsub_del_sub.assert_called_once_with(testval.o)

        self.rr_delete.assert_called_once_with('test')

    def test_persist_data_stream(self):
        config = IngestionConfiguration()

        self.ingestion_management.read_ingestion_configuration = Mock()
        self.ingestion_management.read_ingestion_configuration.return_value = config

        self.ingestion_management.is_persisted = Mock()
        self.ingestion_management.is_persisted.return_value = False

        self.ingestion_management.setup_queues = Mock()
        self.ingestion_management.setup_queues.return_value = True

        self.pubsub_read.return_value = DotDict(persisted=False)

        retval = self.ingestion_management.persist_data_stream('stream_id', 'config_id', 'dataset_id')

        self.assertEquals(retval,'dataset_id')

    @unittest.skip('Need to refactor')
    def test_setup_queues(self):
        ingestion_config = IngestionConfiguration()
        ingestion_config.queues = [IngestionQueue()]
        setattr(ingestion_config,'_id','config_id')

        self.pubsub_create_sub.return_value = 'subscription_id'

        self.ingestion_management._existing_dataset = Mock()

        retval = self.ingestion_management.setup_queues(ingestion_config, 'stream_id', 'dataset_id')

        self.assertTrue(retval)

        self.ingestion_management._existing_dataset.assert_called_once_with('stream_id','dataset_id')

    def test_unpersist_data_stream(self):

        self.rr_find_objs.return_value = [('sub'),('assoc')]
        self.rr_find_assocs.return_value = ['assoc']
        self.rr_find_subjects.return_value = ([],[])

        self.ingestion_management.unpersist_data_stream('stream_id','ingestion_id')

        self.assertTrue(self.pubsub_deact_sub.call_count)
        self.assertTrue(self.rr_del_assoc.call_count)
        self.assertTrue(self.pubsub_del_sub.call_count)


    def test_determine_queue(self):
        pass #unimplemented

    def test_list_ingestion(self):
        testval = (['resource'], ['other'])
        self.rr_find_res.return_value = testval
        retval = self.ingestion_management.list_ingestion_configurations(id_only=True)
        self.assertTrue(retval == testval[0])
        self.assertTrue(self.rr_find_res.call_count)

    def test_is_persisted(self):
        self.pubsub_is_persisted.return_value = True

        retval = self.ingestion_management.is_persisted('stream_id')

        self.assertEquals(retval,True)