def deleteLockfile(self):
'''
USAGE: etl.py --action=deleteLockfile --cfg=<config> --section=<section_pattern>
It will read the options of each section matched <section_pattern> in conf/actions/deleteLockfile/<config>.cfg
EXAMPLE: python2.7 etl.py --action=deleteLockfile --cfg=sample --section=sample
'''
sections = self.getSectionsForAction(DELETE_LOCKFILE_ACTION)
notExistedFiles = []
for section in sections:
logging.info("\nStart processing section: %s" % section)
# Read parameters for moving files
filename = self._readPatternMandatoryOption(section, FILENAME)
# Delete Lockfile
lock = Lockfile(filename)
if lock.exists():
try:
lock.delete()
except:
logging.error("Error happend when deleting lock file '%s'!" % (filename+'.lock'))
raise
logging.info("The lock file '%s.lock' was deleted successfully." % filename)
else:
notExistedFiles.append(filename+'.lock')
if not notExistedFiles:
logging.info("Deleted all lock files successfully.")
return RC_NO_ERROR
for notExistedFile in notExistedFiles:
logging.error("There is no lock file '%s' to be deleted." % notExistedFile)
return RC_ERROR
def createLockfile(self):
'''
USAGE: etl.py --action=createLockfile --cfg=<config> --section=<section_pattern>
It will read the options of each section matched <section_pattern> in conf/actions/createLockfile/<config>.cfg
EXAMPLE: python2.7 etl.py --action=createLockfile --cfg=sample --section=sample
'''
sections = self.getSectionsForAction(CREATE_LOCKFILE_ACTION)
lockedFiles = []
for section in sections:
logging.info("\nStart processing section: %s" % section)
# Read parameters for moving files
filename = self._readPatternMandatoryOption(section, FILENAME)
# Create Lockfile
lock = Lockfile(filename)
if lock.exists():
lockedFiles.append(filename+'.lock')
else:
try:
lock.create()
except:
logging.error("Error happend when creating lock file '%s'!" % (filename+'.lock'))
raise
logging.info("The lock file '%s.lock' was created successfully." % filename)
if not lockedFiles:
logging.info("Created all lock files successfully.")
return RC_NO_ERROR
for lockedFile in lockedFiles:
logging.warn("The lock file: '%s' was already existed." % lockedFile)
return RC_EXISTING_PROCESS_EXISTS
def main():
options = Options()
# Read global configuration file
configParser = _readApplicationConfigurationFiles()
#####################
# Configure logging #
#####################
# Determine ML_JOBNET_NAME
jobnet_name = ''
if os.environ.has_key('ML_JOBNET_NAME'):
jobnet_name = os.environ['ML_JOBNET_NAME']
# Create log directory if it does not exist
base_log_directory = configParser.get('directories', 'LOG_DIRECTORY')
log_directory = "%s/%s" % (base_log_directory, jobnet_name)
if not os.path.exists(log_directory):
os.makedirs(log_directory)
# Determine the log file name
current_date = datetime.datetime.now().strftime('%Y%m%d%H%M%S')
if os.access(log_directory, os.W_OK):
logFilename = "%s/etl-%s.%s_%s.log" % (log_directory, options.get('action'), current_date, str(uuid.uuid4()))
else:
logFilename = "work/etl-%s.%s_%s.log" % (options.get('action'), current_date, str(uuid.uuid4()))
# define a handler which writes INFO messages or higher
console_log = logging.StreamHandler()
console_log.setLevel(logging.INFO)
# define a handler which writes DEBUG messages or higher
file_log = logging.FileHandler(filename=logFilename)
file_log.setLevel(logging.DEBUG)
file_format = logging.Formatter('%(asctime)s [%(levelname)-7s] (%(filename)s,%(lineno)d) %(message)s')
file_log.setFormatter(file_format)
logging.getLogger().addHandler(console_log)
logging.getLogger().addHandler(file_log)
logging.getLogger().setLevel(logging.DEBUG)
############################
# Dynamically Load Modules #
############################
actionLabels = {'actions.py':'General Actions'}
validActions = {}
modules = {}
for file in os.listdir('actions'):
if (file.strip().endswith('.py') and file.strip() != '__init__.py'):
# Dynamically Load the Module
moduleName = file.split('.py')[0]
className = "%sActions" % (_convertToCamelcase(moduleName))
modulePath = "actions.%s" % (moduleName)
_temp = __import__(modulePath, globals(), locals(), ['object'], -1)
# Dynamically get an instance of the given class
classInstance = getattr(_temp, className)()
# Find all executable actions
validActions = _findCallableActions(classInstance, moduleName, validActions)
# Validate if the action is valid. Print valid actions if no valid action is found
if (options.get('action') not in validActions):
logging.warn("%s is not a valid action. Please specify an action from the list below" % (options.get('action')))
actions = validActions.keys()
# Group All Actions by their class descriptions
actionsByClass = {}
for action in actions:
classDescription = validActions[action][1]
if (actionsByClass.has_key(classDescription) == True):
tmp = actionsByClass[classDescription]
tmp.append(action)
actionsByClass[classDescription] = tmp
else:
actionsByClass[classDescription] = [action]
for classDesc in actionsByClass:
print "\n%s" % (classDesc)
print "----------------------"
actions = actionsByClass[classDesc]
actions.sort()
for action in actions:
print action
sys.exit(197)
# Determine the JC_JOBID Environment Value
if (os.environ.has_key('JC_JOBID')):
jcJobId = os.environ['JC_JOBID']
else:
jcJobId = ''
# Check if a lock file exists, exit if it does
lockfile = Lockfile(options.get('action'))
if (lockfile.exists() == True):
logging.warn('Action is currently running!')
sys.exit(198)
# Log the start of the process
msg = "Starting the '%s' action" % (options.get('action'))
logging.debug(msg)
# Call the requested action, if an error occurs, log the error and send an email
try:
func = validActions[options.get('action')][0]
returnCode = func()
except FileConfigMissing as fcm:
logging.error(fcm)
returnCode = 101
except MandatoryOptionMissing as mom:
logging.error(mom)
returnCode = 102
except Exception,e:
logging.error(e)
exc_type, exc_value, exc_traceback = sys.exc_info()
tb = traceback.extract_tb(exc_traceback)
formattedTb = traceback.format_list(tb)
# All traceback concatenate for output
tbOutputLine = ''
for line in formattedTb:
tbOutputLine = tbOutputLine + line
logging.error(tbOutputLine)
if (lockfile.exists() == True):
lockfile.delete()
returnCode = 199 #TODO: make this look up the specific action's section and see if a different return code is defined