def __init__(self): self.options = Options() self.config = SafeConfigParser() self.config.read(os.path.join(etl_util.get_app_root_path(), 'conf/globals.cfg')) env = Environment(self.config) self.environment = env.environment self.environmentId = env.environmentId self._persistentParams = {}
def main(): options = Options() # Read global configuration file configParser = _readApplicationConfigurationFiles() ##################### # Configure logging # ##################### # Determine ML_JOBNET_NAME jobnet_name = '' if os.environ.has_key('ML_JOBNET_NAME'): jobnet_name = os.environ['ML_JOBNET_NAME'] # Create log directory if it does not exist base_log_directory = configParser.get('directories', 'LOG_DIRECTORY') log_directory = "%s/%s" % (base_log_directory, jobnet_name) if not os.path.exists(log_directory): os.makedirs(log_directory) # Determine the log file name current_date = datetime.datetime.now().strftime('%Y%m%d%H%M%S') if os.access(log_directory, os.W_OK): logFilename = "%s/etl-%s.%s_%s.log" % (log_directory, options.get('action'), current_date, str(uuid.uuid4())) else: logFilename = "work/etl-%s.%s_%s.log" % (options.get('action'), current_date, str(uuid.uuid4())) # define a handler which writes INFO messages or higher console_log = logging.StreamHandler() console_log.setLevel(logging.INFO) # define a handler which writes DEBUG messages or higher file_log = logging.FileHandler(filename=logFilename) file_log.setLevel(logging.DEBUG) file_format = logging.Formatter('%(asctime)s [%(levelname)-7s] (%(filename)s,%(lineno)d) %(message)s') file_log.setFormatter(file_format) logging.getLogger().addHandler(console_log) logging.getLogger().addHandler(file_log) logging.getLogger().setLevel(logging.DEBUG) ############################ # Dynamically Load Modules # ############################ actionLabels = {'actions.py':'General Actions'} validActions = {} modules = {} for file in os.listdir('actions'): if (file.strip().endswith('.py') and file.strip() != '__init__.py'): # Dynamically Load the Module moduleName = file.split('.py')[0] className = "%sActions" % (_convertToCamelcase(moduleName)) modulePath = "actions.%s" % (moduleName) _temp = __import__(modulePath, globals(), locals(), ['object'], -1) # Dynamically get an instance of the given class classInstance = getattr(_temp, className)() # Find all executable actions validActions = _findCallableActions(classInstance, moduleName, validActions) # Validate if the action is valid. Print valid actions if no valid action is found if (options.get('action') not in validActions): logging.warn("%s is not a valid action. Please specify an action from the list below" % (options.get('action'))) actions = validActions.keys() # Group All Actions by their class descriptions actionsByClass = {} for action in actions: classDescription = validActions[action][1] if (actionsByClass.has_key(classDescription) == True): tmp = actionsByClass[classDescription] tmp.append(action) actionsByClass[classDescription] = tmp else: actionsByClass[classDescription] = [action] for classDesc in actionsByClass: print "\n%s" % (classDesc) print "----------------------" actions = actionsByClass[classDesc] actions.sort() for action in actions: print action sys.exit(197) # Determine the JC_JOBID Environment Value if (os.environ.has_key('JC_JOBID')): jcJobId = os.environ['JC_JOBID'] else: jcJobId = '' # Check if a lock file exists, exit if it does lockfile = Lockfile(options.get('action')) if (lockfile.exists() == True): logging.warn('Action is currently running!') sys.exit(198) # Log the start of the process msg = "Starting the '%s' action" % (options.get('action')) logging.debug(msg) # Call the requested action, if an error occurs, log the error and send an email try: func = validActions[options.get('action')][0] returnCode = func() except FileConfigMissing as fcm: logging.error(fcm) returnCode = 101 except MandatoryOptionMissing as mom: logging.error(mom) returnCode = 102 except Exception,e: logging.error(e) exc_type, exc_value, exc_traceback = sys.exc_info() tb = traceback.extract_tb(exc_traceback) formattedTb = traceback.format_list(tb) # All traceback concatenate for output tbOutputLine = '' for line in formattedTb: tbOutputLine = tbOutputLine + line logging.error(tbOutputLine) if (lockfile.exists() == True): lockfile.delete() returnCode = 199 #TODO: make this look up the specific action's section and see if a different return code is defined
class EtlBase(object): def __init__(self): self.options = Options() self.config = SafeConfigParser() self.config.read(os.path.join(etl_util.get_app_root_path(), 'conf/globals.cfg')) env = Environment(self.config) self.environment = env.environment self.environmentId = env.environmentId self._persistentParams = {} ##################### # Private Functions # ##################### def _createDateString(self, dateFormat, returnDays = None, returnMonths = None): currentTime = datetime.datetime.now() if (returnMonths != None): numWeeks = 4 * int(returnMonths) # TODO: Improve this part to ensure it always returns correctly delta = datetime.timedelta(weeks = numWeeks) requestedTime = currentTime - delta elif (returnDays != None): delta = datetime.timedelta(days = int(returnDays)) requestedTime = currentTime - delta else: requestedTime = currentTime dateString = None if (dateFormat == 'YYYYMMDDhhmiss'): dateString = "%s%s%s" % (requestedTime.year , str(requestedTime.month).rjust(2,'0') , str(requestedTime.day).rjust(2,'0')) dateString += "%s%s%s" % (str(requestedTime.hour).rjust(2,'0') , str(requestedTime.minute).rjust(2,'0') , str(requestedTime.second).rjust(2,'0')) elif (dateFormat == 'YYYYMMDD'): dateString = "%s%s%s" % (requestedTime.year , str(requestedTime.month).rjust(2,'0') , str(requestedTime.day).rjust(2,'0')) elif (dateFormat == 'YYYY/MM/DD_hh:mi:ss'): dateString = "%s/%s/%s_" % (requestedTime.year , str(requestedTime.month).rjust(2,'0') , str(requestedTime.day).rjust(2,'0')) dateString = "%s:%s:%s" % (requestedTime.year , str(requestedTime.month).rjust(2,'0') , str(requestedTime.day).rjust(2,'0')) elif (dateFormat == 'YYYY-MM-DD'): dateString = "%s-%s-%s" % (requestedTime.year , str(requestedTime.month).rjust(2,'0') , str(requestedTime.day).rjust(2,'0')) elif (dateFormat == 'YYYYMM'): dateString = "%s%s" % (requestedTime.year , str(requestedTime.month).rjust(2,'0')) elif (dateFormat == 'YYYY/MM'): dateString = "%s/%s" % (requestedTime.year , str(requestedTime.month).rjust(2,'0')) else: msg = "The Date Format \"%s\" has not been implemented. Aborting Process" % (dateFormat) logging.error(msg) sys.exit(RC_ERROR) return dateString def _createLockFile(self, lockFilename=None): ''' Creates a lock file with the specified name. If no name is provided the calling action name will be used ''' basePath = os.getcwd() if (lockFilename == None): lockFilename = basePath + '/' + LOCKFILE_DIRECTORY + '/' + str(self.options.get('action')) + '.lock' print lockFilename lockFile = open(lockFilename,'w') return lockFilename def _createWorkDirectory(self, workDirectory): ''' Creates a work directory ''' if not os.path.exists(workDirectory): logging.warn("The work directory '%s' does not exist. Attempting to create it." % (workDirectory)) try: os.makedirs(workDirectory) logging.warn("Work Directory '%s' has been created" % (workDirectory)) except: raise IOError('not exist the work directory. (%s)' % workDirectory) def _removeWorkDirectory(self, workDirectory): ''' Removes a work directory which has directories or files ''' if os.path.exists(workDirectory): logging.warn("The work directory '%s' exists. Attempting to remove it." % (workDirectory)) try: shutil.rmtree(workDirectory) logging.warn("Work Directory '%s' has been removed" % (workDirectory)) except Exception, e: raise IOError('not remove the work directory. (%s)' % workDirectory)