def LoadSourceToStaging(self, filePath, sinkDbConnectionString,
sourceTableNames, extension):
try:
objList = []
dbFiles = self.getSortesListofFiles(filePath)
sourceTableNameList = sourceTableNames.split(",")
# logging.info('sourceTableNameList:'+ str(sourceTableNameList))
# validate data fetched from configuration file
if (not filePath):
raise ValueError('Source File path is empty!')
elif (not sinkDbConnectionString):
raise ValueError('Sink DB connection string is empty!')
elif (not sourceTableNames):
raise ValueError('Source Table Names is empty!')
elif (not dbFiles or len(dbFiles) == 0):
raise ValueError('No Db files present!')
# perform transfer of data from source db to sink db
for sourceTableName in sourceTableNameList:
obj = Fetcher()
isCleaned = 0
for dbfile in dbFiles:
if str(dbfile) in skippedDBList:
continue
dbfile = filePath + dbfile
sourcedbconnectionstring = 'sqlite:///' + dbfile
if (isCleaned == 0):
obj.cleansink(sourcedbconnectionstring,
sourceTableName, sinkDbConnectionString)
isCleaned = 1
# obj.transferSourceToSink(sourcedbconnectionstring, sourceTableName, sinkDbConnectionString, dbfile)
objArgs = {
'sourceDbString': sourcedbconnectionstring,
'tableName': sourceTableName,
'sinkDbSring': sinkDbConnectionString,
'dbName': dbfile
}
objList.append(objArgs)
obj = Fetcher()
p = Pool(5)
p.map(obj.transferSourceToSink, objList)
p.close()
p.join()
except Exception as e:
logging.basicConfig(filename='Fetcher.log', level=logging.ERROR)
logging.error('There is an exception in the code FetcherPlumber!')
logging.error(e)
logging.error(traceback.format_exc())
raise
def test_cleansink(self):
try:
sourcedbconnectionstring = 'sqlite:///../../ContentData/eb99f209f9c34ba192f6e695aeb37e4f.sqlite3'
sourceTableName = 'Content_contentnode'
fetchObj = Fetcher()
fetchObj.cleansink(sourcedbconnectionstring, sourceTableName,
self.sinkDbConnectionString)
self.assertTrue(True)
except Exception as e:
print(e)
self.assertFalse(True,
'There was an exception while cleaning the sink')
def LoadSourceToStaging(self, filePath,sinkDbConnectionString,sourceTableNames,extension):
try:
dbFiles = self.fetchFilesFromDirectory(filePath, extension)
sourceTableNameList = sourceTableNames.split(",")
# validate data fetched from configuration file
if (not filePath):
raise ValueError('Source File path is empty!')
elif (not sinkDbConnectionString):
raise ValueError('Sink DB connection string is empty!')
elif (not sourceTableNames):
raise ValueError('Source Table Names is empty!')
elif (not dbFiles or len(dbFiles) == 0):
raise ValueError('No Db files present!')
# perform transfer of data from source db to sink db
for sourceTableName in sourceTableNameList:
obj = Fetcher()
isCleaned = 0
for dbfile in dbFiles:
dbfile=filePath+dbfile
sourcedbconnectionstring = 'sqlite:///' + dbfile
if (isCleaned == 0):
obj.cleansink(sourcedbconnectionstring, sourceTableName, sinkDbConnectionString)
isCleaned = 1
obj.transferSourceToSink(sourcedbconnectionstring, sourceTableName, sinkDbConnectionString)
except Exception as e:
logging.basicConfig(filename='Fetcher.log', level=logging.ERROR)
logging.error('There is an exception in the code FetcherPlumber!')
logging.error(e)
logging.error(traceback.format_exc())
raise
