def getConfigFromDb(self):
''' Get detail of all the active feeds from Source and Feed config databases
Raises exception if no matching entry in database
:param:
:returns list resultSet: A list of rows from source_config and feed_config
:raises DataloadApplicationException: Critical exception in database connectivity
'''
self.LOGGER.debug("getConfigFromDb")
query = cfg.query.getQueryFeedDetail().format(self.zone, self.country,
self.sourceNm)
self.LOGGER.debug("Select query is: {}".format(query))
try:
resultSet = util.getResults(query)
except ex.DataloadApplicationException as dae:
erorrCd, errorMsg = cfg.err.getErrorCd('GETFIL_GET_CONFIG_ERR')
errorMsg = errorMsg.format(dae.message)
raise ex.DataloadApplicationException(errorMsg,
code=erorrCd,
exeId=self.executionId)
if not resultSet:
erorrCd, errorMsg = cfg.err.getErrorCd('GETFIL_NO_FEED_FOUND')
errorMsg = errorMsg.format(self.sourceNm)
raise ex.DataloadApplicationException(errorMsg,
code=erorrCd,
exeId=self.executionId)
else:
self.LOGGER.info("found {} active feed details for {} ".format(
len(resultSet), self.sourceNm))
return resultSet
def getProcessMapId(self):
''' Get process map id from database
:param:
:returns:
:raises:
'''
self.LOGGER.debug("getProcessMapId")
global processMapId
try:
self.processMapId = util.getMappingIdByPorgramANDProcessName(
self.programName, self.processName)
except ex.DataloadApplicationException as dae:
erorrCd, errorMsg = cfg.err.getErrorCd('GETFIL_SET_PRCSID_ERR')
errorMsg = errorMsg.format(self.programName, self.processName, dae)
raise ex.DataloadApplicationException(errorMsg,
code=erorrCd,
exeId=self.executionId)
if not self.processMapId:
erorrCd, errorMsg = cfg.err.getErrorCd('GETFIL_PRCSID_NOT_FND')
errorMsg = errorMsg.format(self.programName, self.processName)
raise ex.DataloadApplicationException(errorMsg,
code=erorrCd,
exeId=self.executionId)
self.LOGGER.info("Process Map Id is {}".format(self.processMapId))
processMapId = self.processMapId
def transferFromVMToAzure(self, storageAccountNm, storageAccountKey,
loadDataCntnr, sourceDataDir, destDataDir):
''' Copy files from local to Azure Blob storage
:param str storageAccountNm: Name of storage account
:parm str storageAccountKey: Storage account key
:parm str loadDataCntnr: Load Data Container
:parm str sourceDataDir: Directory location of source data
:parm str destDataDir: Directory of target data
:returns:
:raises Exception DataloadApplicationException: Error while connecting to blob storage
'''
''' List all the files in the input directory on VM '''
listAllFile = listdir(sourceDataDir)
fileCounter = 0
try:
''' connect to the Azure Blob storage account'''
blobConnect = BlockBlobService(account_name=storageAccountNm,
account_key=storageAccountKey)
except Exception as e:
errorMsg = cfg.err.getErrorMsg('AZR_CONNECTION_ERR')
errorMsg = errorMsg.format(type(e), str(e))
raise ex.DataloadApplicationException(errorMsg)
else:
self.LOGGER.debug(
"connected to Blob Storage - {} container - {}".format(
storageAccountNm, loadDataCntnr))
try:
''' move each file sequentially from VM to Azure Blob storage '''
for fileNm in listAllFile:
sourceFile = sourceDataDir + fileNm
destBlob = destDataDir + fileNm
self.LOGGER.debug(
"Uploading File= {} from source= {} as blob= {}".format(
fileNm, sourceDataDir, destBlob))
blobConnect.create_blob_from_path(loadDataCntnr,
destBlob,
sourceFile,
validate_content=True)
fileCounter += 1
except Exception as e:
errorMsg = cfg.err.getErrorMsg('AZR_FILE_LOAD_ERR')
errorMsg = errorMsg.format(sourceFile, type(e), str(e))
raise ex.DataloadApplicationException(errorMsg)
return fileCounter
def targetDataCopy(self, destFeed):
''' invokes method to copy data from VM to Azure
:param str destFeed: Destination module instance object
:returns:
:raises DataloadApplicationException: Application Exception
'''
self.LOGGER.debug("targetDataCopy")
copyStartTime = datetime.datetime.utcnow()
try:
fileCount = destFeed.transferFromVMToAzure(self.destCredUsrNm,
self.destCredPwd,
self.destCntnr,
self.absTempDataDir,
self.absTargetDataDir)
except (ex.DataloadApplicationException,
ex.DataloadSystemException), dae:
erorrCd, errorMsg = cfg.err.getErrorCd('GETFIL_TARGET_COPY_ERR')
errorMsg = errorMsg.format(dae.message)
executionId = self.setExecutionId(STATUS_FAILED)
raise ex.DataloadApplicationException(errorMsg,
code=erorrCd,
exeId=executionId)
def sourceConnection(self, sourceFeed):
''' Invokes method to establish connection with the source system
:param obj sourceFeed: Source module instance object
:returns obj sourceConnectObj: Source connection object
:raises DataloadApplicationException: Exception in getting execution date from database
'''
self.LOGGER.debug("sourceConnection")
try:
sourceConnectObj = sourceFeed.connectToSource(
self.sourceHostNm, self.sourceCredUsrnm, self.sourceCredPwd)
except (ex.DataloadApplicationException,
ex.DataloadSystemException) as dae:
erorrCd, errorMsg = cfg.err.getErrorCd('GETFIL_SOURCE_CONNECT_ERR')
errorMsg = errorMsg.format(dae.message)
executionId = self.setExecutionId(STATUS_FAILED)
raise ex.DataloadApplicationException(errorMsg,
code=erorrCd,
exeId=executionId)
self.LOGGER.debug("Connection established with {}".format(
self.sourceEnvNm))
return sourceConnectObj
def setUpdateInitialExecId(self, statusCode):
''' makes an entry in the APP_RUN_STAT to mark beginning of getFile process
:param str statusCode: Status code for making default entry
:returns :
:raises DataloadApplicationException: Exceptions while making entry in the table
'''
self.LOGGER.debug("setUpdateInitialExecId")
self.LOGGER.debug(
"Process execution id before invoking util is - {}".format(
self.executionId))
self.statusCd = statusCode
self.sourceId = DB_NULL
self.feedId = DB_NULL
startDate = DB_NULL
postRunCt = DB_NULL
try:
self.executionId = util.logRunStat(self.sourceId, self.feedId,
self.processMapId,
self.workflowId, startDate,
self.statusCd, postRunCt,
self.eligForNextRun,
self.executionId)
except ex.DataloadApplicationException as dae:
erorrCd, errorMsg = cfg.err.getErrorCd('GETFIL_SET_EXECID_ERR')
errorMsg = errorMsg.format(dae.message)
raise ex.DataloadApplicationException(errorMsg,
code=erorrCd,
exeId=self.executionId)
self.LOGGER.info(
"Process execution id after invoking util is {}".format(
self.executionId))
def setExecutionId(self, statusCd):
''' makes entry in the APP_RUN_STAT and returns the execution id
:param str statusCd: status code
:returns :
:raises DataloadSystemException: Exceptions while deleting data from directory
'''
self.LOGGER.debug("setExecutionId")
''' converts the date to YYYY-MM-DD string format '''
execDate = self.executionDate.strftime('%Y-%m-%d')
nullExecId = DB_NULL
try:
executionId = util.logRunStat(self.sourceId, self.feedId,
self.processMapId, self.workflowId,
execDate, statusCd, self.postRunCt,
self.eligForNextRun, nullExecId)
except ex.DataloadApplicationException as dae:
erorrCd, errorMsg = cfg.err.getErrorCd('GETFIL_SET_EXECID_ERR')
errorMsg = errorMsg.format(dae.message)
self.LOGGER.error('{} {}'.format(errorMsg, str(dae)))
raise ex.DataloadApplicationException(errorMsg,
code=erorrCd,
exeId=0)
self.LOGGER.info(
"Status-{} for Execution id-{}, Feed Id-{}, Execution Date-{}".
format(statusCd, executionId, self.feedId, execDate))
return executionId
def setInitialExecId(self, processID, statusCd, execution_id):
''' makes entry in the APP_RUN_STAT to mark beginning of load data process
:param:
:returns : executionid
:raises DataloadSystemException: Exceptions while making entry in the table
'''
self.feedId = DB_NULL
self.statusCd = statusCd
self.sourceId = DB_NULL
self.processId = processID
startDate = dt.date.today()
startDate = startDate.strftime('%Y-%m-%d')
initialDate = DB_NULL
postRunCt = 0
eligible_next_run = DB_NULL
workflowid = self.workflowId
self.LOGGER.debug("Set Initial execution id")
try:
executionId = util.logRunStat(self.feedId, self.sourceId,
self.processId, workflowid,
initialDate, self.statusCd,
postRunCt, eligible_next_run,
execution_id)
#return executionId
except ex.DataloadApplicationException as dae:
erorrCd, errorMsg = cfg.err.getErrorCd('DATALOAD_SET_EXECID_ERR')
errorMsg = errorMsg.format(dae.message)
raise ex.DataloadApplicationException(errorMsg,
code=erorrCd,
exeId=executionId)
self.LOGGER.debug("Initial execution id {}".format(executionId))
return executionId
def setExecutionId(self, execDt, processID, statusCd, result, post_run_ct,
eligible_next_run, execution_id):
''' makes entry in the app Run stat and returns the execution id
:param int sourceId: source id
:param int feedId: feed id
:param int processID: process id
:param date execStartDt: execution start date
:param str statusCd: status code
:returns :
:raises DataloadSystemException: Exceptions while deleting data from directory
'''
self.LOGGER.debug("Set execution id")
try:
executionId = util.logRunStat(result.source_id, result.feed_id,
processID, self.workflowId, execDt,
statusCd, post_run_ct,
eligible_next_run, execution_id)
except ex.DataloadApplicationException as dae:
erorrCd, errorMsg = cfg.err.getErrorCd('DATALOAD_SET_EXECID_ERR')
errorMsg = errorMsg.format(dae.message)
raise ex.DataloadApplicationException(errorMsg,
code=erorrCd,
exeId=executionId)
self.LOGGER.debug("Execution id {}".format(executionId))
return executionId
def cleanupFiles(self):
''' Delete entire directory tree, if it exists
:param:
:returns:
:raises DataloadSystemException: Exceptions while deleting data from directory
'''
self.LOGGER.debug("cleanupFiles")
if os.path.exists(self.absTempDataDir):
try:
shutil.rmtree(self.absTempDataDir)
except Exception as e:
erorrCd, errorMsg = cfg.err.getErrorCd(
'GETFIL_VM_DIR_SPACE_ERR')
errorMsg = errorMsg.format(str(e))
executionId = self.setExecutionId(STATUS_FAILED)
raise ex.DataloadApplicationException(errorMsg,
code=erorrCd,
exeId=executionId)
self.LOGGER.info("Deleted directory {}".format(
self.absTempDataDir))
else:
self.LOGGER.warning("Cannot find directory {}".format(
self.absTempDataDir))
def getConfigFromDb(self, executionId):
''' get feed details from database
:param str zone: the zone for which data is to be copied
:parm str country: the country for which data is to be copied
:parm str sourceEnv: the source environment for which data is to be copied
:returns list resultSet: All the active rows from source_config and feed_config
:raises Exception e: Critical exception in database connectivity
'''
self.LOGGER.info("get feed details from database")
self.LOGGER.debug(
"Supplied arguments are zone={}, country={}, platform={}".format(
self.zone, self.country, self.sourceEnv))
query = cfg.query.getQueryFeedDetail().format(self.zone, self.country,
self.sourceEnv)
self.LOGGER.debug("select query is: {} ".format(query))
try:
resultSet = util.getResults(query)
except ex.DataloadApplicationException as dae:
erorrCd, errorMsg = cfg.err.getErrorCd('DATALOAD_GET_CONFIG_ERR')
errorMsg = errorMsg.format(dae.message)
raise ex.DataloadApplicationException(errorMsg,
code=erorrCd,
exeId=executionId)
else:
self.LOGGER.debug("found {} feed files for - {} ".format(
len(resultSet), self.sourceEnv))
return resultSet
def clearFileFromAzure(self, storageAccountNm, storageAccountKey,
loadDataCntnr, destDataDir):
''' Delete all files in given Azure directory
:param str storageAccountNm: Name of storage account
:parm str storageAccountKey: Storage account key
:parm str loadDataCntnr: Load Data Container
:parm str destDataDir: Target directory
:returns:
:raises Exception DataloadApplicationException: Error while connecting to blob storage
'''
try:
''' connect to the Azure Blob storage account'''
blobConnect = BlockBlobService(account_name=storageAccountNm,
account_key=storageAccountKey)
except Exception as e:
errorMsg = cfg.err.getErrorMsg('AZR_CONNECTION_ERR')
errorMsg = errorMsg.format(type(e), str(e))
raise ex.DataloadApplicationException(errorMsg)
try:
''' Get list of all file at a give path, in Azure Blob '''
listAllFile = blobConnect.list_blobs(container_name=loadDataCntnr,
prefix=destDataDir)
except Exception as e:
errorMsg = cfg.err.getErrorMsg('AZR_FILE_LIST_ERR')
errorMsg = errorMsg.format(destDataDir, type(e), str(e))
raise ex.DataloadApplicationException(errorMsg)
try:
''' Delete file from Azure Blob Storage '''
for val in listAllFile:
blobConnect.delete_blob(loadDataCntnr, val.name)
self.LOGGER.debug("Deleted blob {}".format(val.name))
except Exception as e:
errorMsg = cfg.err.getErrorMsg('AZR_FILE_DELETE_ERR')
errorMsg = errorMsg.format(val, type(e), str(e))
raise ex.DataloadApplicationException(errorMsg)
def sourceDataCopy(self, sourceFeed, sourceConnectObj):
''' Invokes method to copy file from source system to the VM
Calculates the time taken to copy files from source to VM
Raises Business exception if no matching file is found
:param str sourceFeed: Source module instance object
:param str sourceConnectObj: Source connection object
:returns:
:raises DataloadApplicationException: Application exception
:raises DataloadBusinessException: Business exception
'''
self.LOGGER.debug("sourceDataCopy")
''' If it doesn't exist already, create the directory on VM '''
self._createDirectory(self.absTempDataDir)
copyStartTime = datetime.datetime.utcnow()
skipListing = False
matchFileName = cfg.gfConf.skipFileListing()
for values in matchFileName:
if values == self.sourceNm:
skipListing = True
try:
fileCount = sourceFeed.transferFeedFromSourceToVM(
sourceConnectObj, self.sourceCntnr, self.absFileNm,
self.absSourceFeedDir, self.absTempDataDir, skipListing)
except (ex.DataloadApplicationException,
ex.DataloadSystemException) as dae:
erorrCd, errorMsg = cfg.err.getErrorCd('GETFIL_SOURCE_COPY_ERR')
errorMsg = errorMsg.format(dae.message)
executionId = self.setExecutionId(STATUS_FAILED)
raise ex.DataloadApplicationException(errorMsg,
code=erorrCd,
exeId=executionId)
''' if no matching feed is available, raise exception '''
if not fileCount:
erorrCd, errorMsg = cfg.err.getErrorCd('GETFIL_FILE_NT_RCVD')
errorMsg = errorMsg.format(self.absFileNm, self.executionDate)
executionId = self.setExecutionId(STATUS_FAILED)
raise ex.DataloadBusinessException(errorMsg,
code=erorrCd,
exeId=executionId)
''' Publish time taken to pull files '''
copyEndTime = datetime.datetime.utcnow()
self.postRunCt = fileCount
timeTaken = copyEndTime - copyStartTime
self.LOGGER.info(
"{} time taken to copy {} files from Source to VM for Feed Id-{} Date-{}"
.format(str(timeTaken), fileCount, self.feedId,
self.executionDate))
def _createDirectory(self, dataDir):
''' Create a directory, if it doesn't exists
:param str tempDataDir: directory path
:returns:
:raises DataloadApplicationException: Exceptions while creating directory
'''
self.LOGGER.debug("_createDirectory")
if not os.path.exists(dataDir):
try:
os.makedirs(dataDir)
except os.errno.EACCES as e:
erorrCd, errorMsg = cfg.err.getErrorCd(
'GETFIL_VM_DIR_PRMSN_ERR')
errorMsg = errorMsg.format(dataDir, str(e))
executionId = self.setExecutionId(STATUS_FAILED)
raise ex.DataloadApplicationException(errorMsg,
code=erorrCd,
exeId=executionId)
except os.errno.ENOSPC as e:
erorrCd, errorMsg = cfg.err.getErrorCd(
'GETFIL_VM_DIR_SPACE_ERR')
errorMsg = errorMsg.format(dataDir, str(e))
executionId = self.setExecutionId(STATUS_FAILED)
raise ex.DataloadApplicationException(errorMsg,
code=erorrCd,
exeId=executionId)
except Exception as e:
erorrCd, errorMsg = cfg.err.getErrorCd(
'GETFIL_VM_DIR_UNKWN_ERR')
errorMsg = errorMsg.format(dataDir, str(e))
executionId = self.setExecutionId(STATUS_FAILED)
raise ex.DataloadApplicationException(errorMsg,
code=erorrCd,
exeId=executionId)
else:
self.LOGGER.info("created {} directory".format(dataDir))
else:
self.LOGGER.debug("{} directory already exists".format(dataDir))
def targetDataDelete(self, destFeed):
''' invokes method to delete data from Azure Blob
:param str destFeed: Destination module instance object
:returns:
:raises DataloadApplicationException: Application Exception
'''
self.LOGGER.debug("targetDataDelete")
try:
destFeed.clearFileFromAzure(self.destCredUsrNm, self.destCredPwd,
self.destCntnr, self.absTargetDataDir)
except (ex.DataloadApplicationException,
ex.DataloadSystemException), dae:
erorrCd, errorMsg = cfg.err.getErrorCd('GETFIL_TARGET_DELETE_ERR')
errorMsg = errorMsg.format(dae.message)
executionId = self.setExecutionId(STATUS_FAILED)
raise ex.DataloadApplicationException(errorMsg,
code=erorrCd,
exeId=executionId)
def getMappingIdByPorgramANDProcess(self):
''' get Process id
:param str processName: process Name
:returns str processId: process Id
:raises DataloadSystemException: Exceptions while deleting data from directory
'''
self.LOGGER.debug("Get Process id")
try:
processMapId = util.getMappingIdByPorgramANDProcessName(
self.programName, self.processName)
except ex.DataloadApplicationException as dae:
erorrCd, errorMsg = cfg.err.getErrorCd('DATALOAD_SET_PRCSID_ERR')
errorMsg = errorMsg.format(dae.message)
raise ex.DataloadApplicationException(errorMsg,
code=erorrCd,
exeId=self.executionId)
self.LOGGER.debug("Process Map Id is {}".format(processMapId))
return processMapId
def getStartDate(self):
''' Get latest execution date from database
Set the execution day to the day after last execution
Raise business exception, if no historical data is found
:param:
:returns date execStartDt: Start date for execution
:raises DataloadApplicationException: Exception in getting execution date from database
'''
self.LOGGER.debug("getStartDate")
try:
execStartDt = util.getLastExeDateForGetFilesByFeedId(
self.feedId, processMapId)
except ex.DataloadApplicationException as dae:
erorrCd, errorMsg = cfg.err.getErrorCd('GETFIL_EXEC_DATE_ERR')
errorMsg = errorMsg.format(dae.message)
executionId = self.setExecutionId('FAILED')
raise ex.DataloadApplicationException(errorMsg,
code=erorrCd,
exeId=executionId)
else:
if execStartDt:
execStartDt = execStartDt + datetime.timedelta(days=1)
else:
erorrCd, errorMsg = cfg.err.getErrorCd(
'GETFIL_FIRST_RUN_DATE_ERR')
errorMsg = errorMsg.format(self.feedId)
executionId = self.setExecutionId(STATUS_FAILED)
raise ex.DataloadBusinessException(errorMsg,
code=erorrCd,
exeId=executionId)
self.LOGGER.info(
"Execution start date updated to-{} for feed Id-{}".format(
execStartDt, self.feedId))
return execStartDt
def setDates(self):
''' Validates execution start and end date
Converts the input date to datetime format
If not provided, End date will be set to the day before current system date
:param:
:returns tuple execStartDt, execEndDt: Execution start and end date
:raises DataloadApplicationException: Exception for invalid date format
'''
self.LOGGER.debug("setDates")
try:
if self.startDate != 'None':
execStartDt = datetime.datetime.strptime(
self.startDate, '%m/%d/%Y').date()
else:
execStartDt = self.startDate
if self.endDate != 'None':
execEndDt = datetime.datetime.strptime(self.endDate,
'%m/%d/%Y').date()
else:
execEndDt = datetime.date.today() - datetime.timedelta(days=1)
except ValueError:
erorrCd, errorMsg = cfg.err.getErrorCd('GETFIL_DATE_FRMT_ERR')
raise ex.DataloadApplicationException(errorMsg,
code=erorrCd,
exeId=self.executionId)
else:
self.LOGGER.debug(
"Execution start date is-{} and end date-{}".format(
execStartDt, execEndDt))
return execStartDt, execEndDt
def checkFrequency(self, executionDate):
''' Checks the frequency and day of execution for the file
Validates if the file is due for given execution date
No check is required, if frequency of execution is set to 'daily'
:param str executionDate: file date
:returns boolean processFeed: Indicates whether a feed file is due or not
:raises DataloadApplicationException: Exception querying database
'''
self.LOGGER.debug("checkFrequency")
self.LOGGER.debug(
"Validate if file is due for execution: sourceId-{}, feedId-{}, frequency-{}, date-()"
.format(self.sourceId, self.feedId, self.sourceFrq, executionDate))
self.executionDate = executionDate
invalidFrequency = False
processFeed = False
sourceFrqlLw = self.sourceFrq.lower()
if sourceFrqlLw == FREQUENCY_DAILY.lower():
processFeed = True
else:
try:
lastExecDt = util.getLastExeDateForGetFilesByFeedId(
self.feedId, processMapId)
except ex.DataloadApplicationException as dae:
erorrCd, errorMsg = cfg.err.getErrorCd('GETFIL_EXEC_DATE_ERR')
errorMsg = errorMsg.format(dae.message)
executionId = self.setExecutionId(STATUS_FAILED)
raise ex.DataloadApplicationException(errorMsg,
code=erorrCd,
exeId=executionId)
else:
try:
if not lastExecDt:
lastExecDt = DEFAULT_DATE
lastExecDt = datetime.datetime.strptime(
lastExecDt, '%m/%d/%Y').date()
if sourceFrqlLw == FREQUENCY_WEEKLY.lower():
if (lastExecDt.isocalendar()[0] <
executionDate.isocalendar()[0]
and self.sourcedayOfRun <=
executionDate.isoweekday()):
processFeed = True
elif (lastExecDt.isocalendar()[0]
== executionDate.isocalendar()[0]
and lastExecDt.isocalendar()[1] <
executionDate.isocalendar()[1]
and self.sourcedayOfRun <=
executionDate.isoweekday()):
processFeed = True
elif sourceFrqlLw == FREQUENCY_MONTHLY.lower():
if (lastExecDt.year < executionDate.year
and self.sourcedayOfRun <= executionDate.day):
processFeed = True
elif (lastExecDt.year == executionDate.year
and lastExecDt.month < executionDate.month
and self.sourcedayOfRun <= executionDate.day):
processFeed = True
elif sourceFrqlLw == FREQUENCY_YEARLY.lower():
if (lastExecDt.year < executionDate.year
and self.sourcedayOfRun <=
executionDate.timetuple().tm_yday):
processFeed = True
else:
invalidFrequency = True
except Exception as e:
erorrCd, errorMsg = cfg.err.getErrorCd(
'GETFIL_VALID_FREQ_ERR')
errorMsg = errorMsg.format(str(e))
executionId = self.setExecutionId(STATUS_FAILED)
raise ex.DataloadApplicationException(errorMsg,
code=erorrCd,
exeId=executionId)
if invalidFrequency:
erorrCd, errorMsg = cfg.err.getErrorCd(
'GETFIL_INVALID_FEED_FRQ')
errorMsg = errorMsg.format(self.feedId)
executionId = self.setExecutionId(STATUS_FAILED)
raise ex.DataloadBusinessException(errorMsg,
code=erorrCd,
exeId=executionId)
return processFeed
def getTempHiveTableDistinctDate(self, result, executionId):
''' function perform following actions..
- execute hdfs command to list all the files location for current data load process.
- split the output by new line and store in a list
- run for loop for each items in the list
- extract date - split by '/' and take last item
- execute hive table add partition command to assign partition for a date to the external table.
- return all folders name in a list to main function.
'''
self.LOGGER.info(
"Get list of folders for which data exists in the Temp table : {}".
format(result.hive_temp_tbl_nm))
try:
hdfsCmd = "hdfs dfs -ls '" + result.raw_feed_data_url + "'"
output = commands.getstatusoutput(hdfsCmd)
status = output[0]
listOutPut = list(output)[1].split('\n', 1000)
listOutPut = listOutPut[1:]
except ex.DataloadApplicationException as dae:
errorCd, errorMsg = cfg.err.getErrorCd('DATALOAD_BLOB_PATH_ERR')
errorMsg = errorMsg.format(dae.message)
raise ex.DataloadSystemException(errorMsg,
code=errorCd,
exeId=executionId)
else:
if status > 0:
errorCd, errorMsg = cfg.err.getErrorCd(
'DATALOAD_BLOB_PATH_ERR')
message = output[1]
errorMsg = errorMsg.format(message)
errorMsg = string.replace(errorMsg, "'", "")
raise ex.DataloadSystemException(errorMsg,
code=errorCd,
exeId=executionId)
else:
''' add parttion to temp hive table '''
listDateOutPut = []
tempTableName = result.hive_temp_tbl_nm
for val in listOutPut:
try:
partDate = int(val.split('/')[-1])
listDateOutPut.append((val.split('/')[-1]).split('\n'))
except ex.DataloadApplicationException as dae:
errorCd, errorMsg = cfg.err.getErrorCd(
'DATALOAD_FLDR_FRMT')
errorMsg = errorMsg.format(dae.message)
raise ex.DataloadApplicationException(
errorMsg, code=errorCd, exeId=executionId)
else:
try:
partDate = val.split('/')[-1]
partLoc = result.raw_feed_data_url + "/" + partDate
alterTableQuery = (
"ALTER TABLE {} add IF NOT EXISTS partition(filedate={}) LOCATION '{}';"
.format(tempTableName, partDate, partLoc))
quotedAlterTableQuery = '"' + alterTableQuery + '"'
cmd = "hive -S -e " + quotedAlterTableQuery
status, output = commands.getstatusoutput(cmd)
except ex.DataloadApplicationException as dae:
errorCd, errorMsg = cfg.err.getErrorCd(
'DATALOAD_PART_ADD_ERR')
errorMsg = errorMsg.format(dae.message)
raise ex.DataloadApplicationException(
errorMsg, code=errorCd, exeId=executionId)
else:
if status > 0:
errorCd, errorMsg = cfg.err.getErrorCd(
'DATALOAD_PART_ADD_ERR')
fm = output.index("FAILED")
message = output[fm:]
message = string.replace(message, "'", "")
errorMsg = errorMsg.format(message)
raise ex.DataloadSystemException(
errorMsg, code=errorCd, exeId=executionId)
else:
self.LOGGER.debug(
"Partition Created for table : {} for filedate : {}"
.format(tempTableName, partDate))
return listDateOutPut
