class NotifiersList(object):
"""
This class is only used to support a list of notifications modules.
"""
def __init__(self, options, metadata):
self.options = options
self.module_loader = ModuleLoader()
self.notifiers = self._populate_notifiers(metadata)
def _populate_notifiers(self, metadata):
notifiers_list = []
for notifier in self.options:
notifier_object = self.module_loader.load_notifier(
notifier, metadata)
notifiers_list.append(notifier_object)
return notifiers_list
def notify_start_dump(self, receivers=None):
if receivers is None:
receivers = []
for notifier in self.notifiers:
notifier.notify_start_dump(receivers)
def notify_complete_dump(self, receivers=None):
if receivers is None:
receivers = []
for notifier in self.notifiers:
notifier.notify_complete_dump(receivers)
def notify_failed_job(self, msg, stack_strace, receivers=None):
if receivers is None:
receivers = []
for notifier in self.notifiers:
notifier.notify_failed_job(msg, stack_strace, receivers)
class NotifiersList(object):
"""
This class is only used to support a list of notifications modules.
"""
def __init__(self, options, metadata):
self.options = options
self.module_loader = ModuleLoader()
self.notifiers = self._populate_notifiers(metadata)
def _populate_notifiers(self, metadata):
notifiers_list = []
for notifier in self.options:
notifier_object = self.module_loader.load_notifier(notifier, metadata)
notifiers_list.append(notifier_object)
return notifiers_list
def notify_start_dump(self, receivers=None):
if receivers is None:
receivers = []
for notifier in self.notifiers:
notifier.notify_start_dump(receivers)
def notify_complete_dump(self, receivers=None):
if receivers is None:
receivers = []
for notifier in self.notifiers:
notifier.notify_complete_dump(receivers)
def notify_failed_job(self, msg, stack_strace, receivers=None):
if receivers is None:
receivers = []
for notifier in self.notifiers:
notifier.notify_failed_job(msg, stack_strace, receivers)
class ModuleLoaderTest(unittest.TestCase):
def setUp(self):
self.module_loader = ModuleLoader()
def test_reader_valid_class(self):
options = valid_config_with_updates({
'exporter_options': {
'LOG_LEVEL': 'DEBUG',
'LOGGER_NAME': 'export-pipeline'
},
'reader': {
'name': 'exporters.transform.no_transform.NoTransform',
'options': {}
}
})
with self.assertRaises(TypeError):
o = ExporterConfig(options)
self.module_loader.load_reader(o.reader_options)
def test_writer_valid_class(self):
options = valid_config_with_updates({
'exporter_options': {
'LOG_LEVEL': 'DEBUG',
'LOGGER_NAME': 'export-pipeline'
},
'writer': {
'name': 'exporters.readers.random_reader.RandomReader',
'options': {
'number_of_items': 1000,
'batch_size': 100
}
},
})
with self.assertRaises(TypeError):
o = ExporterConfig(options)
self.module_loader.load_writer(o.writer_options)
def test_persistence_valid_class(self):
options = valid_config_with_updates({
'exporter_options': {
'LOG_LEVEL': 'DEBUG',
'LOGGER_NAME': 'export-pipeline'
},
'persistence': {
'name': 'exporters.transform.no_transform.NoTransform',
'options': {}
}
})
o = ExporterConfig(options)
with self.assertRaises(TypeError):
self.module_loader.load_persistence(o.persistence_options)
def test_formatter_valid_class(self):
options = valid_config_with_updates({
'exporter_options': {
'LOG_LEVEL': 'DEBUG',
'LOGGER_NAME': 'export-pipeline',
"EXPORTER": 'exporters.writers.console_writer.ConsoleWriter',
},
'formatter': {
'name': 'exporters.transform.no_transform.NoTransform',
'options': {}
},
})
with self.assertRaises(TypeError):
o = ExporterConfig(options)
self.module_loader.load_formatter(o.reader_options)
def test_notifier_valid_class(self):
options = valid_config_with_updates({
'exporter_options': {
'LOG_LEVEL': 'DEBUG',
'LOGGER_NAME': 'export-pipeline'
},
'notifier': {
'name': 'exporters.transform.no_transform.NoTransform',
'options': {}
}
})
with self.assertRaises(TypeError):
o = ExporterConfig(options)
self.module_loader.load_notifier(o.notifiers)
def test_grouper_valid_class(self):
options = valid_config_with_updates({
'exporter_options': {
'LOG_LEVEL': 'DEBUG',
'LOGGER_NAME': 'export-pipeline'
},
'grouper': {
'name': 'exporters.transform.no_transform.NoTransform',
'options': {}
}
})
with self.assertRaises(TypeError):
o = ExporterConfig(options)
self.module_loader.load_grouper(o.grouper_options)
def test_filter_valid_class(self):
options = valid_config_with_updates({
'exporter_options': {
'LOG_LEVEL': 'DEBUG',
'LOGGER_NAME': 'export-pipeline'
},
'filter': {
'name': 'exporters.transform.no_transform.NoTransform',
'options': {}
},
})
with self.assertRaises(TypeError):
o = ExporterConfig(options)
self.module_loader.load_filter(o.filter_before_options)
def test_transform_valid_class(self):
options = valid_config_with_updates({
'exporter_options': {
'LOG_LEVEL': 'DEBUG',
'LOGGER_NAME': 'export-pipeline'
},
'transform': {
'name': 'exporters.filters.no_filter.NoFilter',
'options': {}
}
})
with self.assertRaises(TypeError):
o = ExporterConfig(options)
self.module_loader.load_transform(o.transform_options)
def test_load_grouper(self):
grouper = {
'name': 'exporters.groupers.file_key_grouper.FileKeyGrouper',
'options': {
'keys': ['country_code', 'state', 'city']
}
}
self.assertIsInstance(self.module_loader.load_grouper(grouper, None),
BaseGrouper)
class ModuleLoaderTest(unittest.TestCase):
def setUp(self):
self.module_loader = ModuleLoader()
def test_reader_valid_class(self):
options = valid_config_with_updates({
'exporter_options': {
'LOG_LEVEL': 'DEBUG',
'LOGGER_NAME': 'export-pipeline'
},
'reader': {
'name': 'exporters.transform.no_transform.NoTransform',
'options': {
}
}
})
with self.assertRaises(TypeError):
o = ExporterConfig(options)
self.module_loader.load_reader(o.reader_options)
def test_writer_valid_class(self):
options = valid_config_with_updates({
'exporter_options': {
'LOG_LEVEL': 'DEBUG',
'LOGGER_NAME': 'export-pipeline'
},
'writer': {
'name': 'exporters.readers.random_reader.RandomReader',
'options': {
'number_of_items': 1000,
'batch_size': 100
}
},
})
with self.assertRaises(TypeError):
o = ExporterConfig(options)
self.module_loader.load_writer(o.writer_options)
def test_persistence_valid_class(self):
options = valid_config_with_updates({
'exporter_options': {
'LOG_LEVEL': 'DEBUG',
'LOGGER_NAME': 'export-pipeline'
},
'persistence': {
'name': 'exporters.transform.no_transform.NoTransform',
'options': {
}
}
})
o = ExporterConfig(options)
with self.assertRaises(TypeError):
self.module_loader.load_persistence(o.persistence_options)
def test_formatter_valid_class(self):
options = valid_config_with_updates({
'exporter_options': {
'LOG_LEVEL': 'DEBUG',
'LOGGER_NAME': 'export-pipeline',
"EXPORTER": 'exporters.writers.console_writer.ConsoleWriter',
},
'formatter': {
'name': 'exporters.transform.no_transform.NoTransform',
'options': {
}
},
})
with self.assertRaises(TypeError):
o = ExporterConfig(options)
self.module_loader.load_formatter(o.reader_options)
def test_notifier_valid_class(self):
options = valid_config_with_updates({
'exporter_options': {
'LOG_LEVEL': 'DEBUG',
'LOGGER_NAME': 'export-pipeline'
},
'notifier': {
'name': 'exporters.transform.no_transform.NoTransform',
'options': {
}
}
})
with self.assertRaises(TypeError):
o = ExporterConfig(options)
self.module_loader.load_notifier(o.notifiers)
def test_grouper_valid_class(self):
options = valid_config_with_updates({
'exporter_options': {
'LOG_LEVEL': 'DEBUG',
'LOGGER_NAME': 'export-pipeline'
},
'grouper': {
'name': 'exporters.transform.no_transform.NoTransform',
'options': {
}
}
})
with self.assertRaises(TypeError):
o = ExporterConfig(options)
self.module_loader.load_grouper(o.grouper_options)
def test_filter_valid_class(self):
options = valid_config_with_updates({
'exporter_options': {
'LOG_LEVEL': 'DEBUG',
'LOGGER_NAME': 'export-pipeline'
},
'filter': {
'name': 'exporters.transform.no_transform.NoTransform',
'options': {
}
},
})
with self.assertRaises(TypeError):
o = ExporterConfig(options)
self.module_loader.load_filter(o.filter_before_options)
def test_transform_valid_class(self):
options = valid_config_with_updates({
'exporter_options': {
'LOG_LEVEL': 'DEBUG',
'LOGGER_NAME': 'export-pipeline'
},
'transform': {
'name': 'exporters.filters.no_filter.NoFilter',
'options': {
}
}
})
with self.assertRaises(TypeError):
o = ExporterConfig(options)
self.module_loader.load_transform(o.transform_options)
def test_load_grouper(self):
grouper = {
'name': 'exporters.groupers.file_key_grouper.FileKeyGrouper',
'options': {
'keys': ['country_code', 'state', 'city']
}
}
self.assertIsInstance(self.module_loader.load_grouper(grouper, None),
BaseGrouper)
