class timestamp_dim_loader:
def __init__(self):
# Reading configuration file ( YAML file )
self.logger = EtlLogger.get_logger(
self.__class__.__name__) # use class name as the log name
self.lock = JobLock(
self.__class__.__name__) # use class name as the lock name
self.env = yaml.load(open(config['ENV']))
self.db = OnlineDB(self.env['DSN'], logger=self.logger)
set_schema_sql = self.env['SET_SCHEMA_SQL']
self.db.executeSQL(set_schema_sql)
self.config = config
def loadTimestamp(self, date_sid):
row_sql = config['CHECK_COUNT'] % (date_sid)
existing_rows = self.db.retrieveSQL(row_sql)
if not existing_rows:
date_part = date_sid[0:4] + "-" + date_sid[4:6] + "-" + date_sid[
6:8]
self.logger.info("Starting to load hourly timestamp")
sql = config['INSERT_NEW_RECORD'] % (date_sid, date_part,
date_part)
self.db.executeSQL(sql, False)
self.logger.info("inserted records for %s" % (date_part))
def updateLoadState(self):
# update load_state and commit
self.db.updateLoadStateWithCurrentTime(self.config['LOAD_STATE_VAR'])
self.db.commit()
self.logger.info("Updated load_state variable %s and committed txn" %
(self.config['LOAD_STATE_VAR']))
class timestamp_dim_loader:
def __init__( self ):
# Reading configuration file ( YAML file )
self.logger=EtlLogger.get_logger(self.__class__.__name__) # use class name as the log name
self.lock = JobLock(self.__class__.__name__) # use class name as the lock name
self.env= yaml.load( open( config['ENV'] ) )
self.db = OnlineDB( self.env['DSN'], logger=self.logger )
set_schema_sql = self.env['SET_SCHEMA_SQL']
self.db.executeSQL( set_schema_sql )
self.config = config
def loadTimestamp( self,date_sid):
row_sql=config['CHECK_COUNT'] %(date_sid)
existing_rows = self.db.retrieveSQL(row_sql)
if not existing_rows:
date_part=date_sid[0:4]+"-"+date_sid[4:6]+"-"+date_sid[6:8]
self.logger.info("Starting to load hourly timestamp")
sql=config['INSERT_NEW_RECORD'] %(date_sid,date_part,date_part)
self.db.executeSQL(sql, False)
self.logger.info("inserted records for %s" %(date_part))
def updateLoadState(self):
# update load_state and commit
self.db.updateLoadStateWithCurrentTime(self.config['LOAD_STATE_VAR'])
self.db.commit()
self.logger.info("Updated load_state variable %s and committed txn" % ( self.config['LOAD_STATE_VAR']))
