def setUp(self):
mock_clients = self._create_service_mock('ingestion_management')
self.ingestion_management = IngestionManagementService()
self.ingestion_management.clients = mock_clients
self.rr_create = mock_clients.resource_registry.create
self.rr_read = mock_clients.resource_registry.read
self.rr_update = mock_clients.resource_registry.update
self.rr_delete = mock_clients.resource_registry.delete
self.rr_find_objs = mock_clients.resource_registry.find_objects
self.rr_find_assocs = mock_clients.resource_registry.find_associations
self.rr_find_res = mock_clients.resource_registry.find_resources
self.rr_create_assoc = mock_clients.resource_registry.create_association
self.rr_find_subjects = mock_clients.resource_registry.find_subjects
self.rr_del_assoc = mock_clients.resource_registry.delete_association
self.pubsub_create_sub = mock_clients.pubsub_management.create_subscription
self.pubsub_read = mock_clients.pubsub_management.read_stream
self.pubsub_is_persisted = mock_clients.pubsub_management.is_persisted
self.pubsub_del_sub = mock_clients.pubsub_management.delete_subscription
self.pubsub_act_sub = mock_clients.pubsub_management.activate_subscription
self.pubsub_deact_sub = mock_clients.pubsub_management.deactivate_subscription
self.dataset_create = mock_clients.dataset_management.create_dataset
def setUp(self):
mock_clients = self._create_service_mock('ingestion_management')
self.ingestion_management = IngestionManagementService()
self.ingestion_management.clients = mock_clients
self.rr_create = mock_clients.resource_registry.create
self.rr_read = mock_clients.resource_registry.read
self.rr_update = mock_clients.resource_registry.update
self.rr_delete = mock_clients.resource_registry.delete
self.rr_find_objs = mock_clients.resource_registry.find_objects
self.rr_find_assocs = mock_clients.resource_registry.find_associations
self.rr_find_res = mock_clients.resource_registry.find_resources
self.rr_create_assoc = mock_clients.resource_registry.create_association
self.rr_find_subjects = mock_clients.resource_registry.find_subjects
self.rr_del_assoc = mock_clients.resource_registry.delete_association
self.pubsub_create_sub = mock_clients.pubsub_management.create_subscription
self.pubsub_read = mock_clients.pubsub_management.read_stream
self.pubsub_is_persisted = mock_clients.pubsub_management.is_persisted
self.pubsub_del_sub = mock_clients.pubsub_management.delete_subscription
self.pubsub_act_sub = mock_clients.pubsub_management.activate_subscription
self.pubsub_deact_sub = mock_clients.pubsub_management.deactivate_subscription
self.dataset_create = mock_clients.dataset_management.create_dataset
class IngestionManagementUnitTest(PyonTestCase):
def setUp(self):
mock_clients = self._create_service_mock('ingestion_management')
self.ingestion_management = IngestionManagementService()
self.ingestion_management.clients = mock_clients
self.rr_create = mock_clients.resource_registry.create
self.rr_read = mock_clients.resource_registry.read
self.rr_update = mock_clients.resource_registry.update
self.rr_delete = mock_clients.resource_registry.delete
self.rr_find_objs = mock_clients.resource_registry.find_objects
self.rr_find_assocs = mock_clients.resource_registry.find_associations
self.rr_find_res = mock_clients.resource_registry.find_resources
self.rr_create_assoc = mock_clients.resource_registry.create_association
self.rr_find_subjects = mock_clients.resource_registry.find_subjects
self.rr_del_assoc = mock_clients.resource_registry.delete_association
self.pubsub_create_sub = mock_clients.pubsub_management.create_subscription
self.pubsub_read = mock_clients.pubsub_management.read_stream
self.pubsub_is_persisted = mock_clients.pubsub_management.is_persisted
self.pubsub_del_sub = mock_clients.pubsub_management.delete_subscription
self.pubsub_act_sub = mock_clients.pubsub_management.activate_subscription
self.pubsub_deact_sub = mock_clients.pubsub_management.deactivate_subscription
self.dataset_create = mock_clients.dataset_management.create_dataset
def test_read_ingestion(self):
testval = 'test'
self.rr_read.return_value = testval
retval = self.ingestion_management.read_ingestion_configuration(
'testconfig')
self.assertTrue(retval == testval)
def test_update_ingestion(self):
testval = {0: 0}
self.ingestion_management.update_ingestion_configuration(testval)
self.rr_update.assert_called_once_with(testval)
def test_create_ingestion(self):
testval = 'config_id'
self.rr_create.return_value = (testval, '0')
queue = IngestionQueue()
retval = self.ingestion_management.create_ingestion_configuration(
name='blah', exchange_point_id='test', queues=[queue])
self.assertTrue(retval == testval)
self.assertTrue(self.rr_create.call_count == 1)
def test_delete_ingestion(self):
testval = DotDict()
testval.o = 'sub_id'
self.rr_find_assocs.return_value = [testval]
self.ingestion_management.delete_ingestion_configuration('test')
self.rr_del_assoc.assert_called_once_with(testval)
self.pubsub_del_sub.assert_called_once_with(testval.o)
self.rr_delete.assert_called_once_with('test')
def test_persist_data_stream(self):
config = IngestionConfiguration()
self.ingestion_management.read_ingestion_configuration = Mock()
self.ingestion_management.read_ingestion_configuration.return_value = config
self.ingestion_management.is_persisted = Mock()
self.ingestion_management.is_persisted.return_value = False
self.ingestion_management.setup_queues = Mock()
self.ingestion_management.setup_queues.return_value = True
self.pubsub_read.return_value = DotDict(persisted=False)
retval = self.ingestion_management.persist_data_stream(
'stream_id', 'config_id', 'dataset_id')
self.assertEquals(retval, 'dataset_id')
@unittest.skip('Need to refactor')
def test_setup_queues(self):
ingestion_config = IngestionConfiguration()
ingestion_config.queues = [IngestionQueue()]
setattr(ingestion_config, '_id', 'config_id')
self.pubsub_create_sub.return_value = 'subscription_id'
self.ingestion_management._existing_dataset = Mock()
retval = self.ingestion_management.setup_queues(
ingestion_config, 'stream_id', 'dataset_id')
self.assertTrue(retval)
self.ingestion_management._existing_dataset.assert_called_once_with(
'stream_id', 'dataset_id')
def test_unpersist_data_stream(self):
self.rr_find_objs.return_value = [('sub'), ('assoc')]
self.rr_find_assocs.return_value = ['assoc']
self.rr_find_subjects.return_value = ([], [])
self.ingestion_management.unpersist_data_stream(
'stream_id', 'ingestion_id')
self.assertTrue(self.pubsub_deact_sub.call_count)
self.assertTrue(self.rr_del_assoc.call_count)
self.assertTrue(self.pubsub_del_sub.call_count)
def test_determine_queue(self):
pass #unimplemented
def test_list_ingestion(self):
testval = (['resource'], ['other'])
self.rr_find_res.return_value = testval
retval = self.ingestion_management.list_ingestion_configurations(
id_only=True)
self.assertTrue(retval == testval[0])
self.assertTrue(self.rr_find_res.call_count)
def test_is_persisted(self):
self.pubsub_is_persisted.return_value = True
retval = self.ingestion_management.is_persisted('stream_id')
self.assertEquals(retval, True)
class IngestionManagementUnitTest(PyonTestCase):
def setUp(self):
mock_clients = self._create_service_mock("ingestion_management")
self.ingestion_management = IngestionManagementService()
self.ingestion_management.clients = mock_clients
self.rr_create = mock_clients.resource_registry.create
self.rr_read = mock_clients.resource_registry.read
self.rr_update = mock_clients.resource_registry.update
self.rr_delete = mock_clients.resource_registry.delete
self.rr_find_objs = mock_clients.resource_registry.find_objects
self.rr_find_assocs = mock_clients.resource_registry.find_associations
self.rr_find_res = mock_clients.resource_registry.find_resources
self.rr_create_assoc = mock_clients.resource_registry.create_association
self.rr_find_subjects = mock_clients.resource_registry.find_subjects
self.rr_del_assoc = mock_clients.resource_registry.delete_association
self.pubsub_create_sub = mock_clients.pubsub_management.create_subscription
self.pubsub_read = mock_clients.pubsub_management.read_stream
self.pubsub_del_sub = mock_clients.pubsub_management.delete_subscription
self.pubsub_act_sub = mock_clients.pubsub_management.activate_subscription
self.pubsub_deact_sub = mock_clients.pubsub_management.deactivate_subscription
self.dataset_create = mock_clients.dataset_management.create_dataset
def test_read_ingestion(self):
testval = "test"
self.rr_read.return_value = testval
retval = self.ingestion_management.read_ingestion_configuration("testconfig")
self.assertTrue(retval == testval)
def test_update_ingestion(self):
testval = {0: 0}
self.ingestion_management.update_ingestion_configuration(testval)
self.rr_update.assert_called_once_with(testval)
def test_create_ingestion(self):
testval = "config_id"
self.rr_create.return_value = (testval, "0")
queue = IngestionQueue()
retval = self.ingestion_management.create_ingestion_configuration(
name="blah", exchange_point_id="test", queues=[queue]
)
self.assertTrue(retval == testval)
self.assertTrue(self.rr_create.call_count == 1)
def test_delete_ingestion(self):
testval = DotDict()
testval.o = "sub_id"
self.rr_find_assocs.return_value = [testval]
self.ingestion_management.delete_ingestion_configuration("test")
self.rr_del_assoc.assert_called_once_with(testval)
self.pubsub_del_sub.assert_called_once_with(testval.o)
self.rr_delete.assert_called_once_with("test")
def test_persist_data_stream(self):
ingestval = DotDict()
ingestval.queues = None
ingestval.exchange_point = "test"
queueval = DotDict()
queueval.name = "test"
queueval.datastore_name = "test"
testval = "dataset_id"
self.ingestion_management.read_ingestion_configuration = Mock()
self.ingestion_management.read_ingestion_configuration.return_value = ingestval
self.ingestion_management._determine_queue = Mock()
self.ingestion_management._determine_queue.return_value = queueval
self.pubsub_read.return_value = DotDict({"persisted": False})
self.ingestion_management._existing_dataset = Mock()
retval = self.ingestion_management.persist_data_stream("stream_id", "config_id", "dataset_id")
self.assertTrue(self.pubsub_act_sub.call_count)
self.assertTrue(self.pubsub_create_sub.call_count)
self.assertTrue(self.rr_create_assoc.call_count)
self.assertTrue(retval == testval)
def test_unpersist_data_stream(self):
self.rr_find_objs.return_value = [("sub"), ("assoc")]
self.rr_find_assocs.return_value = ["assoc"]
self.rr_find_subjects.return_value = ([], [])
self.ingestion_management.unpersist_data_stream("stream_id", "ingestion_id")
self.assertTrue(self.pubsub_deact_sub.call_count)
self.assertTrue(self.rr_del_assoc.call_count)
self.assertTrue(self.pubsub_del_sub.call_count)
def test_determine_queue(self):
pass # unimplemented
def test_list_ingestion(self):
testval = (["resource"], ["other"])
self.rr_find_res.return_value = testval
retval = self.ingestion_management.list_ingestion_configurations(id_only=True)
self.assertTrue(retval == testval[0])
self.assertTrue(self.rr_find_res.call_count)
def test_is_persisted(self):
stream = DotDict()
stream.persisted = True
self.pubsub_read.return_value = stream
retval = self.ingestion_management.is_persisted("stream_id")
self.assertEquals(retval, True)
class IngestionManagementUnitTest(PyonTestCase):
def setUp(self):
mock_clients = self._create_service_mock('ingestion_management')
self.ingestion_management = IngestionManagementService()
self.ingestion_management.clients = mock_clients
self.rr_create = mock_clients.resource_registry.create
self.rr_read = mock_clients.resource_registry.read
self.rr_update = mock_clients.resource_registry.update
self.rr_delete = mock_clients.resource_registry.delete
self.rr_find_objs = mock_clients.resource_registry.find_objects
self.rr_find_assocs = mock_clients.resource_registry.find_associations
self.rr_find_res = mock_clients.resource_registry.find_resources
self.rr_create_assoc = mock_clients.resource_registry.create_association
self.rr_find_subjects = mock_clients.resource_registry.find_subjects
self.rr_del_assoc = mock_clients.resource_registry.delete_association
self.pubsub_create_sub = mock_clients.pubsub_management.create_subscription
self.pubsub_read = mock_clients.pubsub_management.read_stream
self.pubsub_is_persisted = mock_clients.pubsub_management.is_persisted
self.pubsub_del_sub = mock_clients.pubsub_management.delete_subscription
self.pubsub_act_sub = mock_clients.pubsub_management.activate_subscription
self.pubsub_deact_sub = mock_clients.pubsub_management.deactivate_subscription
self.dataset_create = mock_clients.dataset_management.create_dataset
def test_read_ingestion(self):
testval = 'test'
self.rr_read.return_value = testval
retval = self.ingestion_management.read_ingestion_configuration('testconfig')
self.assertTrue(retval == testval)
def test_update_ingestion(self):
testval = {0:0}
self.ingestion_management.update_ingestion_configuration(testval)
self.rr_update.assert_called_once_with(testval)
def test_create_ingestion(self):
testval = 'config_id'
self.rr_create.return_value = (testval, '0')
queue = IngestionQueue()
retval = self.ingestion_management.create_ingestion_configuration(name='blah', exchange_point_id='test', queues=[queue])
self.assertTrue(retval == testval)
self.assertTrue(self.rr_create.call_count == 1)
def test_delete_ingestion(self):
testval = DotDict()
testval.o = 'sub_id'
self.rr_find_assocs.return_value = [testval]
self.ingestion_management.delete_ingestion_configuration('test')
self.rr_del_assoc.assert_called_once_with(testval)
self.pubsub_del_sub.assert_called_once_with(testval.o)
self.rr_delete.assert_called_once_with('test')
def test_persist_data_stream(self):
config = IngestionConfiguration()
self.ingestion_management.read_ingestion_configuration = Mock()
self.ingestion_management.read_ingestion_configuration.return_value = config
self.ingestion_management.is_persisted = Mock()
self.ingestion_management.is_persisted.return_value = False
self.ingestion_management.setup_queues = Mock()
self.ingestion_management.setup_queues.return_value = True
self.pubsub_read.return_value = DotDict(persisted=False)
retval = self.ingestion_management.persist_data_stream('stream_id', 'config_id', 'dataset_id')
self.assertEquals(retval,'dataset_id')
@unittest.skip('Need to refactor')
def test_setup_queues(self):
ingestion_config = IngestionConfiguration()
ingestion_config.queues = [IngestionQueue()]
setattr(ingestion_config,'_id','config_id')
self.pubsub_create_sub.return_value = 'subscription_id'
self.ingestion_management._existing_dataset = Mock()
retval = self.ingestion_management.setup_queues(ingestion_config, 'stream_id', 'dataset_id')
self.assertTrue(retval)
self.ingestion_management._existing_dataset.assert_called_once_with('stream_id','dataset_id')
def test_unpersist_data_stream(self):
self.rr_find_objs.return_value = [('sub'),('assoc')]
self.rr_find_assocs.return_value = ['assoc']
self.rr_find_subjects.return_value = ([],[])
self.ingestion_management.unpersist_data_stream('stream_id','ingestion_id')
self.assertTrue(self.pubsub_deact_sub.call_count)
self.assertTrue(self.rr_del_assoc.call_count)
self.assertTrue(self.pubsub_del_sub.call_count)
def test_determine_queue(self):
pass #unimplemented
def test_list_ingestion(self):
testval = (['resource'], ['other'])
self.rr_find_res.return_value = testval
retval = self.ingestion_management.list_ingestion_configurations(id_only=True)
self.assertTrue(retval == testval[0])
self.assertTrue(self.rr_find_res.call_count)
def test_is_persisted(self):
self.pubsub_is_persisted.return_value = True
retval = self.ingestion_management.is_persisted('stream_id')
self.assertEquals(retval,True)