class App(object):
def __init__(self, cfg_file_path):
"""Start the various components. The order of startup must be:
- Writers
- Transformers
- Parsers
- Readers
"""
self.configurator = Cfg()
cfg = self.configurator.get_cfg(cfg_file_path)
# ## Logging configuration and setup
try:
logging.config.fileConfig(cfg.get("logging", "cfg_file").strip())
except Exception:
raise Exception(
"Error loading logging configuration. Please confirm that \
logging configuration is correct")
self.logger = logging.getLogger(__name__ + "." + self.__class__.__name__)
# ## Get message broker producers
self.brokers = get_message_broker_producers(cfg, self.configurator)
# ## Writer apps
writers = get_writer_cfg(cfg)
writer_apps = []
for consumer_cfg, writer_class in writers:
writer_apps.append(AppCreator(cfg, self.brokers, consumer_cfg, writer_class))
# ## Transformer apps
transformers = get_transformer_cfg(cfg)
transformer_apps = []
for consumer_cfg, transformer_class in transformers:
transformer_apps.append(AppCreator(cfg, self.brokers, consumer_cfg, transformer_class))
# ## Parser apps
parser_consumer_cfg, parser_class = get_parser_cfg(cfg)
parser_app = AppCreator(cfg, self.brokers.get("parser"), parser_consumer_cfg, parser_class)
# ## Reader apps
reader_consumer_cfg, reader_classes = get_reader_cfg(cfg)
reader_apps = []
for reader_class in reader_classes:
reader_apps.append(AppCreator(cfg, self.brokers, reader_consumer_cfg, reader_class))
# ## Start all the apps
for writer_app in writer_apps:
writer_app.start()
for transformer_app in transformer_apps:
transformer_app.start()
parser_app.start()
for reader_app in reader_apps:
reader_app.start()
#!/usr/bin/env python
"""App: This is the assembly point for the ETL pipelines
@author: okello
@created: 12-aug-2015
"""
import sys
import os
sys.path.append(os.path.abspath("lib"))
from refunite_etl.cfg import Cfg
from refunite_etl.transformers.db_transformer import DBTransformer
if __name__ == "__main__":
app_args = sys.argv
if len(app_args) < 2:
raise Exception("Required arguments not provided. Include location of config file.")
configurator = Cfg()
DBTransformer(configurator.get_cfg(os.path.abspath(app_args[1]))).run()
