def insert(self, metadata, data):
try:
dao = self.metadata.get('sink')
db_table = '%s.%s' % (self.metadata.get("database"), self.metadata.get('table'))
dao().insert(db_table=db_table, dataframe=data)
except Exception as error:
logger.error(error)
def createFact(self):
try:
datasource = DAO.from_csv(self.metadata.get('datasource'))
table_name = self.metadata.get('fact_table')
Fact.build(datasource, self.metadata.get(table_name)).save()
logger.warn("Fact builder ran successfully...")
except Exception as error:
logger.error(error)
def configuration(abs_path=None):
root = os.path.dirname(os.path.abspath("__main__"))
path = abs_path if abs_path is not None else os.path.join(root, DEFAULT_CONFIGURATION_FILE)
try:
with open(path) as cfg:
return yaml.load(cfg, Loader=yaml.FullLoader)
except Exception as error:
logger.error('Fail to read configuration file: %s' % error)
def createMiddleEntities(self):
try:
datasource = DAO.from_csv(self.metadata.get('datasource'))
dimension_tables = self.metadata.get('middle_entity_tables')
tuple(
map(
lambda table: MiddleEntity.build(
datasource, self.metadata.get(table)).save(),
dimension_tables))
logger.warn("Middle Entities builder ran successfully...")
except Exception as error:
logger.error(error)
def createDimensions(self):
""" Build all dimensional tables given pipeline information in configuration file """
try:
datasource = DAO.from_csv(self.metadata.get('datasource'))
dimension_tables = self.metadata.get('dimension_tables')
tuple(
map(
lambda table: Dimension.build(
datasource, self.metadata.get(table)).save(),
dimension_tables))
logger.warn("Dimension's builder ran successfully...")
except Exception as error:
logger.error(error)
def select(self, db_table):
"""Query engine to this DAO implementation"""
try:
return spark.getOrCreate().read\
.format('jdbc')\
.option('url', "jdbc:postgresql://%s:%s/%s" % (self.host, self.port, self.database))\
.option('dbtable', db_table)\
.option('user', self.user)\
.option('password', self.password)\
.option('driver', 'org.postgresql.Driver')\
.load()
except Exception as error:
logger.error(error)
def insert(self, db_table, dataframe):
"""Data ingestion engine to this DAO implementation"""
try:
dataframe.write\
.format('jdbc')\
.option('url', "jdbc:postgresql://%s:%s/%s" % (self.host, self.port, self.database))\
.option('dbtable', db_table)\
.option('user', self.user)\
.option('password', self.password)\
.option('driver', 'org.postgresql.Driver')\
.mode('append')\
.save()
except Exception as error:
logger.error(error)