def __init__(self):
'''
Constructor for this class
'''
self.location = FileUtilities.PathToForwardSlash(
os.path.dirname(os.path.abspath(__file__)))
self.awsParams = ""
def __init__(self):
'''
Initial settings
'''
super(AutoLightVehiclesAthena, self).__init__()
self.location = FileUtilities.PathToForwardSlash(
os.path.dirname(os.path.abspath(__file__)))
def __init__(self, logger):
'''
Constructor
'''
self.logger = logger
self.location = FileUtilities.PathToForwardSlash(
os.path.dirname(os.path.abspath(__file__)))
def __init__(self):
'''
Constructor
'''
super(MagellanODATAAthenaSpark, self).__init__()
self.location = FileUtilities.PathToForwardSlash(
os.path.dirname(os.path.abspath(__file__)))
def __init__(self):
'''
Initial settings
'''
super(HistoricalBrent, self).__init__()
self.location = FileUtilities.PathToForwardSlash(
os.path.dirname(os.path.abspath(__file__)))
def __init__(self):
'''
Constructor
'''
super(Counter, self).__init__()
self.location = FileUtilities.PathToForwardSlash(
os.path.dirname(os.path.abspath(__file__)))
def __init__(self):
'''
Initial settings
'''
super(FHWAAthenaSpark, self).__init__()
self.location = FileUtilities.PathToForwardSlash(
os.path.dirname(os.path.abspath(__file__)))
def __init__(self):
'''
Constructor
@summary: Explanation of sql
The primary table is PeriodValues all the other tables are used as supporting
Product --> to get the definition of the product if needed
Location --> obtain the description of location and although I am against hardcoding and because of the layout of the data
1) LocationTypeID of 2 is regional and this data could be double dipped so we are omitting those as well as
2) Exceptions to the rule are USR and DDR that also has the chance to be double dippled
Category --> obtain the description of the category ID
1) We are only interested in specific categories and based on our Chemical expert they are :
a. 10 --> Production
b. 12 --> Total Supply
c. 15 --> Total Demand
d. 19 --> Domestic Demand
SubCategory --> although we are not using this table it is important to note it since we are only looking at the summary level
where the value = 0 rather than each broken down level from the PeriodValues table
Last comment is the we are storing the name of the database in the config file since it could change
periodically depending on what is provided
for instance currently the last good database is WASP_2015t
There are two pulls of the database each one 6 months apart. The first one will be in the form of WASP2015
and the next on six months later will have a t at the end such as WASP2015t
'''
super(ChemicalsAthenaSpark, self).__init__()
self.location = FileUtilities.PathToForwardSlash(os.path.dirname(os.path.abspath(__file__)))
def Start(self, logger, moduleName, filelocs):
'''
Start the process. Do the common operations.
'''
self = self.LoadEnvironmentVariables(logger)
# self.logger = logger
self.logger.info(moduleName + " - Getting configuration information.")
self.moduleName = moduleName
# Load the job parameters
self.fileUtilities = FileUtilities(logger)
jobConfigFile = self.location + '/' 'jobConfig.json'
self.job = self.fileUtilities.LoadJobConfiguration(jobConfigFile)
# This is where all the work files will be created
self.localTempDirectory = FileUtilities.PathToForwardSlash(
filelocs["relativeOutputfolder"] + "/" + moduleName)
FileUtilities.CreateFolder(self.localTempDirectory)
# This is where all the local data will be located
if "relativeInputfolder" in filelocs:
self.localTempDataDirectory = FileUtilities.PathToForwardSlash(
filelocs["relativeInputfolder"] + "/" + moduleName)
FileUtilities.CreateFolder(self.localTempDataDirectory)
self.bcpUtilities = BCPUtilities(logger, self.fileUtilities,
self.awsParams,
self.localTempDirectory)
# Create tables if we have a valid script
if "sqlScript" in self.job:
self.CreateTables(self.job["sqlScript"])
# Create etlprocess log table if it does not already exist
if "tblEtl" in filelocs:
self.etlUtilities = EtlLoggingUtilities(self.logger)
self.etlUtilities.awsParams = self.awsParams
self.etlUtilities.filelocs = filelocs
self.etlUtilities.moduleName = self.moduleName
self.etlUtilities.appschema = filelocs["tblEtl"]["appschema"]
self.etlUtilities.StartEtlLogging()
if "folders" in self.job:
self.fileUtilities.moduleName = self.moduleName
self.fileUtilities.localBaseDirectory = self.localTempDirectory
self.fileUtilities.CreateFolders(self.job["folders"])
def __init__(self):
'''
Initial settings
commonParams = {}
commonParams["cat"] = cat { see below }
commonParams["moduleName"] = self.moduleName
commonParams["loggerParams"] = "log"
commonParams["sqlFolder"] = self.sqlFolder
cat is a json nugget that defines the tables required
cat = {
"name": "giif",
"tables": [
{
"schemaName": "eaa_dev",
"srctable": "GEForcast_giif_attributes_working",
"table": "GEForcastHistory_giif_attributes",
"new": "Y",
"type": "attributes",
"partition": {
"over": "mnemonic, frequencychar",
"order": "publisheddate"
},
"fields": [
{ "name": "object_id", "type": "VARCHAR", "size": "30" },
{ "name": "name", "type": "VARCHAR", "size": "100" },
{ "name": "mnemonic", "type": "VARCHAR", "size": "100" },
{ "name": "frequencyChar", "type": "VARCHAR", "size": "2" },
{ "name": "geo", "type": "VARCHAR", "size": "20" },
{ "name": "startDate", "type": "Date"},
{ "name": "endDate", "type": "Date"},
{ "name": "updatedDate", "type": "Date"},
{ "name": "publishedDate", "type": "Date"},
{ "name": "longLabel", "type": "VARCHAR", "size": "2000" },
{ "name": "dataEdge", "type": "VARCHAR", "size": "100" }
],
"sortkey":"object_id"
},
{
"schemaName": "eaa_dev",
"srctable": "GEForcast_giif_data_working",
"table": "GEForcastHistory_giif_data",
"new": "Y",
"type": "series",
"fields": [
{ "name": "object_id", "type": "VARCHAR", "size": "30" },
{ "name": "date", "type": "DATE" },
{ "name": "value", "type": "FLOAT8" }
],
"sortkey":"object_id, date"
}]
}
'''
self.commonParams = {}
self.fileUtilities = None
self.logger = None
self.moduleName = None
self.location = FileUtilities.PathToForwardSlash(
os.path.dirname(os.path.abspath(__file__)))
def __init__(self):
'''
Initial settings
'''
super(EIA, self).__init__()
self.location = FileUtilities.PathToForwardSlash(
os.path.dirname(os.path.abspath(__file__)))
self.localDataFilename = None
def __init__(self):
'''
Constructor
'''
super(ENPMonthly, self).__init__()
self.location = FileUtilities.PathToForwardSlash(
os.path.dirname(os.path.abspath(__file__)))
self.s3subFolder = None
def __init__(self):
'''
Constructor
'''
super(OilMarketForecast, self).__init__()
self.fromDate = None
self.location = FileUtilities.PathToForwardSlash(
os.path.dirname(os.path.abspath(__file__)))
def __init__(self):
'''
Initial settings
'''
super(PGCRAirMarkets, self).__init__()
self.location = FileUtilities.PathToForwardSlash(
os.path.dirname(os.path.abspath(__file__)))
self.awsParams = ""
def __init__(self):
'''
Class Constructor
'''
super(PGCRFERCQuarterlyFilings, self).__init__()
self.location = FileUtilities.PathToForwardSlash(
os.path.dirname(os.path.abspath(__file__)))
self.pandasUtilities = None
self.packedFolder = None
def __init__(self):
'''
Initial settings
'''
super(Scenarios, self).__init__()
self.cBlank = ''
self.location = FileUtilities.PathToForwardSlash(
os.path.dirname(os.path.abspath(__file__)))
def __init__(self):
super(TotemAthenaSpark, self).__init__()
self.location = FileUtilities.PathToForwardSlash(
os.path.dirname(os.path.abspath(__file__)))
self.logonCreds = {}
self.baseUrl = None
self.sourceTableName = None
self.currProcId = None
self.fromDate = None
def __init__(self):
'''
Initial settings
'''
super(ConsensusAthenaSpark, self).__init__()
self.rawFolder = None
self.fileUtilities = FileUtilities(self.logger)
self.location = FileUtilities.PathToForwardSlash(
os.path.dirname(os.path.abspath(__file__)))
def __init__(self):
'''
Basic initialization of instance variable
'''
self.s3 = None
self.redshift = None
self.redshiftCredential = None
self.location = FileUtilities.PathToForwardSlash(
os.path.dirname(os.path.abspath(__file__)))
def __init__(self):
'''
Initial settings
'''
super(GEForecastAthenaSpark, self).__init__()
self.location = FileUtilities.PathToForwardSlash(
os.path.dirname(os.path.abspath(__file__)))
self.attrFields = None
self.dataFields = None
def __init__(self):
'''
Initial settings
'''
super(PGCRKML, self).__init__()
self.location = FileUtilities.PathToForwardSlash(
os.path.dirname(os.path.abspath(__file__)))
self.awsParams = ""
self.packedFolder = None
def __init__(self):
'''
Constructor for this class
'''
super(PGCRForm1FullLoad, self).__init__()
self.location = FileUtilities.PathToForwardSlash(
os.path.dirname(os.path.abspath(__file__)))
self.awsParams = ""
self.dbfUtilities = None
def __init__(self):
'''
Initial settings
'''
super(PGCRIHSMarkitData, self).__init__()
self.location = FileUtilities.PathToForwardSlash(
os.path.dirname(os.path.abspath(__file__)))
self.awsParams = ""
self.excelUtilities = None
def __init__(self):
'''
Initial settings
'''
super(AutoInsight, self).__init__()
self.awsParams = ""
self.processingFile = None
self.fileUtilities = FileUtilities(self.logger)
self.location = FileUtilities.PathToForwardSlash(os.path.dirname(os.path.abspath(__file__)))
def __init__(self):
'''
Constructor
'''
super(HindsightFederated, self).__init__()
self.awsParams = ''
self.seriesVersion = None
self.buildHistoryIterations = 0
self.location = FileUtilities.PathToForwardSlash(
os.path.dirname(os.path.abspath(__file__)))
def __init__(self):
'''
constructor
'''
super(LiquidsBalance, self).__init__()
self.location = FileUtilities.PathToForwardSlash(
os.path.dirname(os.path.abspath(__file__)))
self.awsParams = ""
self.lastFileName = None
self.lastFileLocation = None
def __init__(self):
'''
Initial settings
'''
super(GTA, self).__init__()
self.cBlank = ''
self.fromDate = None
self.fromKey = None
self.location = FileUtilities.PathToForwardSlash(
os.path.dirname(os.path.abspath(__file__)))
def __init__(self):
'''
constructor
'''
super(LiquidsBalanceAthenaSpark, self).__init__()
self.location = FileUtilities.PathToForwardSlash(
os.path.dirname(os.path.abspath(__file__)))
self.fileName = None
self.xlFileName = None
def __init__(self):
'''
Initial settings
'''
super(PGCREIAGrossNetGeneration, self).__init__()
self.packedFolder = None
self.folderList = []
self.toPackFiles = []
self.location = FileUtilities.PathToForwardSlash(
os.path.dirname(os.path.abspath(__file__)))
self.awsParams = ""
def __init__(self):
'''
Initial settings
'''
super(ECRConnectAthenaSpark, self).__init__()
self.jDataPulled = False
self.xRefPulled = False
self.fileUtilities = FileUtilities(self.logger)
self.location = FileUtilities.PathToForwardSlash(
os.path.dirname(os.path.abspath(__file__)))