def __init__(self):
self.options = Options()
self.config = SafeConfigParser()
self.config.read(os.path.join(etl_util.get_app_root_path(), 'conf/globals.cfg'))
env = Environment(self.config)
self.environment = env.environment
self.environmentId = env.environmentId
self._persistentParams = {}
def main():
options = Options()
# Read global configuration file
configParser = _readApplicationConfigurationFiles()
#####################
# Configure logging #
#####################
# Determine ML_JOBNET_NAME
jobnet_name = ''
if os.environ.has_key('ML_JOBNET_NAME'):
jobnet_name = os.environ['ML_JOBNET_NAME']
# Create log directory if it does not exist
base_log_directory = configParser.get('directories', 'LOG_DIRECTORY')
log_directory = "%s/%s" % (base_log_directory, jobnet_name)
if not os.path.exists(log_directory):
os.makedirs(log_directory)
# Determine the log file name
current_date = datetime.datetime.now().strftime('%Y%m%d%H%M%S')
if os.access(log_directory, os.W_OK):
logFilename = "%s/etl-%s.%s_%s.log" % (log_directory, options.get('action'), current_date, str(uuid.uuid4()))
else:
logFilename = "work/etl-%s.%s_%s.log" % (options.get('action'), current_date, str(uuid.uuid4()))
# define a handler which writes INFO messages or higher
console_log = logging.StreamHandler()
console_log.setLevel(logging.INFO)
# define a handler which writes DEBUG messages or higher
file_log = logging.FileHandler(filename=logFilename)
file_log.setLevel(logging.DEBUG)
file_format = logging.Formatter('%(asctime)s [%(levelname)-7s] (%(filename)s,%(lineno)d) %(message)s')
file_log.setFormatter(file_format)
logging.getLogger().addHandler(console_log)
logging.getLogger().addHandler(file_log)
logging.getLogger().setLevel(logging.DEBUG)
############################
# Dynamically Load Modules #
############################
actionLabels = {'actions.py':'General Actions'}
validActions = {}
modules = {}
for file in os.listdir('actions'):
if (file.strip().endswith('.py') and file.strip() != '__init__.py'):
# Dynamically Load the Module
moduleName = file.split('.py')[0]
className = "%sActions" % (_convertToCamelcase(moduleName))
modulePath = "actions.%s" % (moduleName)
_temp = __import__(modulePath, globals(), locals(), ['object'], -1)
# Dynamically get an instance of the given class
classInstance = getattr(_temp, className)()
# Find all executable actions
validActions = _findCallableActions(classInstance, moduleName, validActions)
# Validate if the action is valid. Print valid actions if no valid action is found
if (options.get('action') not in validActions):
logging.warn("%s is not a valid action. Please specify an action from the list below" % (options.get('action')))
actions = validActions.keys()
# Group All Actions by their class descriptions
actionsByClass = {}
for action in actions:
classDescription = validActions[action][1]
if (actionsByClass.has_key(classDescription) == True):
tmp = actionsByClass[classDescription]
tmp.append(action)
actionsByClass[classDescription] = tmp
else:
actionsByClass[classDescription] = [action]
for classDesc in actionsByClass:
print "\n%s" % (classDesc)
print "----------------------"
actions = actionsByClass[classDesc]
actions.sort()
for action in actions:
print action
sys.exit(197)
# Determine the JC_JOBID Environment Value
if (os.environ.has_key('JC_JOBID')):
jcJobId = os.environ['JC_JOBID']
else:
jcJobId = ''
# Check if a lock file exists, exit if it does
lockfile = Lockfile(options.get('action'))
if (lockfile.exists() == True):
logging.warn('Action is currently running!')
sys.exit(198)
# Log the start of the process
msg = "Starting the '%s' action" % (options.get('action'))
logging.debug(msg)
# Call the requested action, if an error occurs, log the error and send an email
try:
func = validActions[options.get('action')][0]
returnCode = func()
except FileConfigMissing as fcm:
logging.error(fcm)
returnCode = 101
except MandatoryOptionMissing as mom:
logging.error(mom)
returnCode = 102
except Exception,e:
logging.error(e)
exc_type, exc_value, exc_traceback = sys.exc_info()
tb = traceback.extract_tb(exc_traceback)
formattedTb = traceback.format_list(tb)
# All traceback concatenate for output
tbOutputLine = ''
for line in formattedTb:
tbOutputLine = tbOutputLine + line
logging.error(tbOutputLine)
if (lockfile.exists() == True):
lockfile.delete()
returnCode = 199 #TODO: make this look up the specific action's section and see if a different return code is defined
class EtlBase(object):
def __init__(self):
self.options = Options()
self.config = SafeConfigParser()
self.config.read(os.path.join(etl_util.get_app_root_path(), 'conf/globals.cfg'))
env = Environment(self.config)
self.environment = env.environment
self.environmentId = env.environmentId
self._persistentParams = {}
#####################
# Private Functions #
#####################
def _createDateString(self, dateFormat, returnDays = None, returnMonths = None):
currentTime = datetime.datetime.now()
if (returnMonths != None):
numWeeks = 4 * int(returnMonths) # TODO: Improve this part to ensure it always returns correctly
delta = datetime.timedelta(weeks = numWeeks)
requestedTime = currentTime - delta
elif (returnDays != None):
delta = datetime.timedelta(days = int(returnDays))
requestedTime = currentTime - delta
else:
requestedTime = currentTime
dateString = None
if (dateFormat == 'YYYYMMDDhhmiss'):
dateString = "%s%s%s" % (requestedTime.year , str(requestedTime.month).rjust(2,'0') , str(requestedTime.day).rjust(2,'0'))
dateString += "%s%s%s" % (str(requestedTime.hour).rjust(2,'0') , str(requestedTime.minute).rjust(2,'0') , str(requestedTime.second).rjust(2,'0'))
elif (dateFormat == 'YYYYMMDD'):
dateString = "%s%s%s" % (requestedTime.year , str(requestedTime.month).rjust(2,'0') , str(requestedTime.day).rjust(2,'0'))
elif (dateFormat == 'YYYY/MM/DD_hh:mi:ss'):
dateString = "%s/%s/%s_" % (requestedTime.year , str(requestedTime.month).rjust(2,'0') , str(requestedTime.day).rjust(2,'0'))
dateString = "%s:%s:%s" % (requestedTime.year , str(requestedTime.month).rjust(2,'0') , str(requestedTime.day).rjust(2,'0'))
elif (dateFormat == 'YYYY-MM-DD'):
dateString = "%s-%s-%s" % (requestedTime.year , str(requestedTime.month).rjust(2,'0') , str(requestedTime.day).rjust(2,'0'))
elif (dateFormat == 'YYYYMM'):
dateString = "%s%s" % (requestedTime.year , str(requestedTime.month).rjust(2,'0'))
elif (dateFormat == 'YYYY/MM'):
dateString = "%s/%s" % (requestedTime.year , str(requestedTime.month).rjust(2,'0'))
else:
msg = "The Date Format \"%s\" has not been implemented. Aborting Process" % (dateFormat)
logging.error(msg)
sys.exit(RC_ERROR)
return dateString
def _createLockFile(self, lockFilename=None):
'''
Creates a lock file with the specified name. If no name is provided the calling action name will be used
'''
basePath = os.getcwd()
if (lockFilename == None):
lockFilename = basePath + '/' + LOCKFILE_DIRECTORY + '/' + str(self.options.get('action')) + '.lock'
print lockFilename
lockFile = open(lockFilename,'w')
return lockFilename
def _createWorkDirectory(self, workDirectory):
'''
Creates a work directory
'''
if not os.path.exists(workDirectory):
logging.warn("The work directory '%s' does not exist. Attempting to create it." % (workDirectory))
try:
os.makedirs(workDirectory)
logging.warn("Work Directory '%s' has been created" % (workDirectory))
except:
raise IOError('not exist the work directory. (%s)' % workDirectory)
def _removeWorkDirectory(self, workDirectory):
'''
Removes a work directory which has directories or files
'''
if os.path.exists(workDirectory):
logging.warn("The work directory '%s' exists. Attempting to remove it." % (workDirectory))
try:
shutil.rmtree(workDirectory)
logging.warn("Work Directory '%s' has been removed" % (workDirectory))
except Exception, e:
raise IOError('not remove the work directory. (%s)' % workDirectory)
