def UploadScriptsToDesignatedS3Location(localScriptsFilepath,
tableSettings):
'''
Upload the script files, typically table creation and upload, to the designated S3 location
'''
s3FolderLocation = AthenaUtilities.ComposeAthenaS3ScriptKey(
tableSettings["schemaName"], tableSettings["table"])
S3Utilities.DeleteFileFromS3TempUsingAWSCLi(s3FolderLocation,
"--recursive")
# Upload only scripts that we plan to keep for later reuse
scriptToCreateRedshift = FileUtilities.ComposeCreateTableSqlFilename(
tableSettings, localScriptsFilepath)
scriptToInsertIntoRedshift = AthenaUtilities.ComposeInsertIntoSqlFilename(
tableSettings, localScriptsFilepath)
S3Utilities.S3Copy(scriptToCreateRedshift, s3FolderLocation)
S3Utilities.S3Copy(scriptToInsertIntoRedshift, s3FolderLocation)
return s3FolderLocation
def LoadDataFromAthenaIntoRedShiftLocalScripts(self,
table,
customWhereCondition=None):
'''
If at a later time we decide to drop the Redshift table and re-load the data from Athena, we need a utility to do that
'''
# Under the hood the table will be recreated if the new flag is on or if the table does not exist
# Load the data from Athena into RedShift after that. The load query only loads what needed from Athena
scriptToCreateRedshiftTable = FileUtilities.ComposeCreateTableSqlFilename(
table, self.fileUtilities.sqlFolder)
RedshiftUtilities.PSqlExecute(scriptToCreateRedshiftTable, self.logger)
scriptToLoadDataFromAthena = AthenaUtilities.ComposeInsertIntoSqlFilename(
table, self.fileUtilities.sqlFolder)
if customWhereCondition:
# Replace the existing where clause with the custom clause
customWhereCondition = " AND " + customWhereCondition + ";"
replacements = {';': customWhereCondition}
scriptToLoadDataFromAthenaCustom = scriptToLoadDataFromAthena + "_custom.sql"
self.fileUtilities.ReplaceStringInFile(
scriptToLoadDataFromAthena, scriptToLoadDataFromAthenaCustom,
replacements)
scriptToLoadDataFromAthena = scriptToLoadDataFromAthenaCustom
RedshiftUtilities.PSqlExecute(scriptToLoadDataFromAthena, self.logger)