class AppCreator(threading.Thread):
def __init__(self, cfg, broker_producer, consumer_cfg, class_cfg):
self.logger = logging.getLogger(__name__ + "." + self.__class__.__name__)
threading.Thread.__init__(self)
self.logger.debug("Creating app; Consumer Config: %s, Class: %s..." % (consumer_cfg, class_cfg))
# ## Load configurations
self.configurator = Cfg()
self.cfg = cfg
self.broker_producer = broker_producer
self.consumer_cfg = consumer_cfg
self.class_cfg = class_cfg
def run(self, ):
# ## Configure Message Consumer service
self.logger.info("Getting Message Consumer Service...")
msg_consumer_class = self.configurator.get_class(
self.consumer_cfg["class"])
msg_consumer_service = msg_consumer_class(
self.consumer_cfg["url"],
self.consumer_cfg["topic"])
# ## Configure data parsers
self.logger.info("Configuring data parsers...")
csv_class = self.configurator.get_class(
self.class_cfg)
csv_class(
self.cfg, self.broker_producer,
msg_consumer_service).run()
class App(object):
def __init__(self, cfg_file_path):
"""Start the various components. The order of startup must be:
- Writers
- Transformers
- Parsers
- Readers
"""
self.configurator = Cfg()
cfg = self.configurator.get_cfg(cfg_file_path)
# ## Logging configuration and setup
try:
logging.config.fileConfig(cfg.get("logging", "cfg_file").strip())
except Exception:
raise Exception(
"Error loading logging configuration. Please confirm that \
logging configuration is correct")
self.logger = logging.getLogger(__name__ + "." + self.__class__.__name__)
# ## Get message broker producers
self.brokers = get_message_broker_producers(cfg, self.configurator)
# ## Writer apps
writers = get_writer_cfg(cfg)
writer_apps = []
for consumer_cfg, writer_class in writers:
writer_apps.append(AppCreator(cfg, self.brokers, consumer_cfg, writer_class))
# ## Transformer apps
transformers = get_transformer_cfg(cfg)
transformer_apps = []
for consumer_cfg, transformer_class in transformers:
transformer_apps.append(AppCreator(cfg, self.brokers, consumer_cfg, transformer_class))
# ## Parser apps
parser_consumer_cfg, parser_class = get_parser_cfg(cfg)
parser_app = AppCreator(cfg, self.brokers.get("parser"), parser_consumer_cfg, parser_class)
# ## Reader apps
reader_consumer_cfg, reader_classes = get_reader_cfg(cfg)
reader_apps = []
for reader_class in reader_classes:
reader_apps.append(AppCreator(cfg, self.brokers, reader_consumer_cfg, reader_class))
# ## Start all the apps
for writer_app in writer_apps:
writer_app.start()
for transformer_app in transformer_apps:
transformer_app.start()
parser_app.start()
for reader_app in reader_apps:
reader_app.start()
def __init__(self, cfg, broker_producer, consumer_cfg, class_cfg):
self.logger = logging.getLogger(__name__ + "." + self.__class__.__name__)
threading.Thread.__init__(self)
self.logger.debug("Creating app; Consumer Config: %s, Class: %s..." % (consumer_cfg, class_cfg))
# ## Load configurations
self.configurator = Cfg()
self.cfg = cfg
self.broker_producer = broker_producer
self.consumer_cfg = consumer_cfg
self.class_cfg = class_cfg
#!/usr/bin/env python
"""App: This is the assembly point for the ETL pipelines
@author: okello
@created: 12-aug-2015
"""
import sys
import os
sys.path.append(os.path.abspath("lib"))
from refunite_etl.cfg import Cfg
from refunite_etl.transformers.db_transformer import DBTransformer
if __name__ == "__main__":
app_args = sys.argv
if len(app_args) < 2:
raise Exception("Required arguments not provided. Include location of config file.")
configurator = Cfg()
DBTransformer(configurator.get_cfg(os.path.abspath(app_args[1]))).run()
