def __init__(self): ''' Constructor for this class ''' self.location = FileUtilities.PathToForwardSlash( os.path.dirname(os.path.abspath(__file__))) self.awsParams = ""
def __init__(self): ''' Initial settings ''' super(AutoLightVehiclesAthena, self).__init__() self.location = FileUtilities.PathToForwardSlash( os.path.dirname(os.path.abspath(__file__)))
def __init__(self, logger): ''' Constructor ''' self.logger = logger self.location = FileUtilities.PathToForwardSlash( os.path.dirname(os.path.abspath(__file__)))
def __init__(self): ''' Constructor ''' super(MagellanODATAAthenaSpark, self).__init__() self.location = FileUtilities.PathToForwardSlash( os.path.dirname(os.path.abspath(__file__)))
def __init__(self): ''' Initial settings ''' super(HistoricalBrent, self).__init__() self.location = FileUtilities.PathToForwardSlash( os.path.dirname(os.path.abspath(__file__)))
def __init__(self): ''' Constructor ''' super(Counter, self).__init__() self.location = FileUtilities.PathToForwardSlash( os.path.dirname(os.path.abspath(__file__)))
def __init__(self): ''' Initial settings ''' super(FHWAAthenaSpark, self).__init__() self.location = FileUtilities.PathToForwardSlash( os.path.dirname(os.path.abspath(__file__)))
def __init__(self): ''' Constructor @summary: Explanation of sql The primary table is PeriodValues all the other tables are used as supporting Product --> to get the definition of the product if needed Location --> obtain the description of location and although I am against hardcoding and because of the layout of the data 1) LocationTypeID of 2 is regional and this data could be double dipped so we are omitting those as well as 2) Exceptions to the rule are USR and DDR that also has the chance to be double dippled Category --> obtain the description of the category ID 1) We are only interested in specific categories and based on our Chemical expert they are : a. 10 --> Production b. 12 --> Total Supply c. 15 --> Total Demand d. 19 --> Domestic Demand SubCategory --> although we are not using this table it is important to note it since we are only looking at the summary level where the value = 0 rather than each broken down level from the PeriodValues table Last comment is the we are storing the name of the database in the config file since it could change periodically depending on what is provided for instance currently the last good database is WASP_2015t There are two pulls of the database each one 6 months apart. The first one will be in the form of WASP2015 and the next on six months later will have a t at the end such as WASP2015t ''' super(ChemicalsAthenaSpark, self).__init__() self.location = FileUtilities.PathToForwardSlash(os.path.dirname(os.path.abspath(__file__)))
def Start(self, logger, moduleName, filelocs): ''' Start the process. Do the common operations. ''' self = self.LoadEnvironmentVariables(logger) # self.logger = logger self.logger.info(moduleName + " - Getting configuration information.") self.moduleName = moduleName # Load the job parameters self.fileUtilities = FileUtilities(logger) jobConfigFile = self.location + '/' 'jobConfig.json' self.job = self.fileUtilities.LoadJobConfiguration(jobConfigFile) # This is where all the work files will be created self.localTempDirectory = FileUtilities.PathToForwardSlash( filelocs["relativeOutputfolder"] + "/" + moduleName) FileUtilities.CreateFolder(self.localTempDirectory) # This is where all the local data will be located if "relativeInputfolder" in filelocs: self.localTempDataDirectory = FileUtilities.PathToForwardSlash( filelocs["relativeInputfolder"] + "/" + moduleName) FileUtilities.CreateFolder(self.localTempDataDirectory) self.bcpUtilities = BCPUtilities(logger, self.fileUtilities, self.awsParams, self.localTempDirectory) # Create tables if we have a valid script if "sqlScript" in self.job: self.CreateTables(self.job["sqlScript"]) # Create etlprocess log table if it does not already exist if "tblEtl" in filelocs: self.etlUtilities = EtlLoggingUtilities(self.logger) self.etlUtilities.awsParams = self.awsParams self.etlUtilities.filelocs = filelocs self.etlUtilities.moduleName = self.moduleName self.etlUtilities.appschema = filelocs["tblEtl"]["appschema"] self.etlUtilities.StartEtlLogging() if "folders" in self.job: self.fileUtilities.moduleName = self.moduleName self.fileUtilities.localBaseDirectory = self.localTempDirectory self.fileUtilities.CreateFolders(self.job["folders"])
def __init__(self): ''' Initial settings commonParams = {} commonParams["cat"] = cat { see below } commonParams["moduleName"] = self.moduleName commonParams["loggerParams"] = "log" commonParams["sqlFolder"] = self.sqlFolder cat is a json nugget that defines the tables required cat = { "name": "giif", "tables": [ { "schemaName": "eaa_dev", "srctable": "GEForcast_giif_attributes_working", "table": "GEForcastHistory_giif_attributes", "new": "Y", "type": "attributes", "partition": { "over": "mnemonic, frequencychar", "order": "publisheddate" }, "fields": [ { "name": "object_id", "type": "VARCHAR", "size": "30" }, { "name": "name", "type": "VARCHAR", "size": "100" }, { "name": "mnemonic", "type": "VARCHAR", "size": "100" }, { "name": "frequencyChar", "type": "VARCHAR", "size": "2" }, { "name": "geo", "type": "VARCHAR", "size": "20" }, { "name": "startDate", "type": "Date"}, { "name": "endDate", "type": "Date"}, { "name": "updatedDate", "type": "Date"}, { "name": "publishedDate", "type": "Date"}, { "name": "longLabel", "type": "VARCHAR", "size": "2000" }, { "name": "dataEdge", "type": "VARCHAR", "size": "100" } ], "sortkey":"object_id" }, { "schemaName": "eaa_dev", "srctable": "GEForcast_giif_data_working", "table": "GEForcastHistory_giif_data", "new": "Y", "type": "series", "fields": [ { "name": "object_id", "type": "VARCHAR", "size": "30" }, { "name": "date", "type": "DATE" }, { "name": "value", "type": "FLOAT8" } ], "sortkey":"object_id, date" }] } ''' self.commonParams = {} self.fileUtilities = None self.logger = None self.moduleName = None self.location = FileUtilities.PathToForwardSlash( os.path.dirname(os.path.abspath(__file__)))
def __init__(self): ''' Initial settings ''' super(EIA, self).__init__() self.location = FileUtilities.PathToForwardSlash( os.path.dirname(os.path.abspath(__file__))) self.localDataFilename = None
def __init__(self): ''' Constructor ''' super(ENPMonthly, self).__init__() self.location = FileUtilities.PathToForwardSlash( os.path.dirname(os.path.abspath(__file__))) self.s3subFolder = None
def __init__(self): ''' Constructor ''' super(OilMarketForecast, self).__init__() self.fromDate = None self.location = FileUtilities.PathToForwardSlash( os.path.dirname(os.path.abspath(__file__)))
def __init__(self): ''' Initial settings ''' super(PGCRAirMarkets, self).__init__() self.location = FileUtilities.PathToForwardSlash( os.path.dirname(os.path.abspath(__file__))) self.awsParams = ""
def __init__(self): ''' Class Constructor ''' super(PGCRFERCQuarterlyFilings, self).__init__() self.location = FileUtilities.PathToForwardSlash( os.path.dirname(os.path.abspath(__file__))) self.pandasUtilities = None self.packedFolder = None
def __init__(self): ''' Initial settings ''' super(Scenarios, self).__init__() self.cBlank = '' self.location = FileUtilities.PathToForwardSlash( os.path.dirname(os.path.abspath(__file__)))
def __init__(self): super(TotemAthenaSpark, self).__init__() self.location = FileUtilities.PathToForwardSlash( os.path.dirname(os.path.abspath(__file__))) self.logonCreds = {} self.baseUrl = None self.sourceTableName = None self.currProcId = None self.fromDate = None
def __init__(self): ''' Initial settings ''' super(ConsensusAthenaSpark, self).__init__() self.rawFolder = None self.fileUtilities = FileUtilities(self.logger) self.location = FileUtilities.PathToForwardSlash( os.path.dirname(os.path.abspath(__file__)))
def __init__(self): ''' Basic initialization of instance variable ''' self.s3 = None self.redshift = None self.redshiftCredential = None self.location = FileUtilities.PathToForwardSlash( os.path.dirname(os.path.abspath(__file__)))
def __init__(self): ''' Initial settings ''' super(GEForecastAthenaSpark, self).__init__() self.location = FileUtilities.PathToForwardSlash( os.path.dirname(os.path.abspath(__file__))) self.attrFields = None self.dataFields = None
def __init__(self): ''' Initial settings ''' super(PGCRKML, self).__init__() self.location = FileUtilities.PathToForwardSlash( os.path.dirname(os.path.abspath(__file__))) self.awsParams = "" self.packedFolder = None
def __init__(self): ''' Constructor for this class ''' super(PGCRForm1FullLoad, self).__init__() self.location = FileUtilities.PathToForwardSlash( os.path.dirname(os.path.abspath(__file__))) self.awsParams = "" self.dbfUtilities = None
def __init__(self): ''' Initial settings ''' super(PGCRIHSMarkitData, self).__init__() self.location = FileUtilities.PathToForwardSlash( os.path.dirname(os.path.abspath(__file__))) self.awsParams = "" self.excelUtilities = None
def __init__(self): ''' Initial settings ''' super(AutoInsight, self).__init__() self.awsParams = "" self.processingFile = None self.fileUtilities = FileUtilities(self.logger) self.location = FileUtilities.PathToForwardSlash(os.path.dirname(os.path.abspath(__file__)))
def __init__(self): ''' Constructor ''' super(HindsightFederated, self).__init__() self.awsParams = '' self.seriesVersion = None self.buildHistoryIterations = 0 self.location = FileUtilities.PathToForwardSlash( os.path.dirname(os.path.abspath(__file__)))
def __init__(self): ''' constructor ''' super(LiquidsBalance, self).__init__() self.location = FileUtilities.PathToForwardSlash( os.path.dirname(os.path.abspath(__file__))) self.awsParams = "" self.lastFileName = None self.lastFileLocation = None
def __init__(self): ''' Initial settings ''' super(GTA, self).__init__() self.cBlank = '' self.fromDate = None self.fromKey = None self.location = FileUtilities.PathToForwardSlash( os.path.dirname(os.path.abspath(__file__)))
def __init__(self): ''' constructor ''' super(LiquidsBalanceAthenaSpark, self).__init__() self.location = FileUtilities.PathToForwardSlash( os.path.dirname(os.path.abspath(__file__))) self.fileName = None self.xlFileName = None
def __init__(self): ''' Initial settings ''' super(PGCREIAGrossNetGeneration, self).__init__() self.packedFolder = None self.folderList = [] self.toPackFiles = [] self.location = FileUtilities.PathToForwardSlash( os.path.dirname(os.path.abspath(__file__))) self.awsParams = ""
def __init__(self): ''' Initial settings ''' super(ECRConnectAthenaSpark, self).__init__() self.jDataPulled = False self.xRefPulled = False self.fileUtilities = FileUtilities(self.logger) self.location = FileUtilities.PathToForwardSlash( os.path.dirname(os.path.abspath(__file__)))