Пример #1
0
    def deleteLockfile(self):
        '''
        USAGE: etl.py --action=deleteLockfile --cfg=<config> --section=<section_pattern>
        It will read the options of each section matched <section_pattern> in conf/actions/deleteLockfile/<config>.cfg
        EXAMPLE: python2.7 etl.py --action=deleteLockfile --cfg=sample --section=sample
        '''
        sections = self.getSectionsForAction(DELETE_LOCKFILE_ACTION)
        notExistedFiles = []
        for section in sections:
            logging.info("\nStart processing section: %s" % section)
            # Read parameters for moving files
            filename = self._readPatternMandatoryOption(section, FILENAME)

            # Delete Lockfile
            lock = Lockfile(filename)
            if lock.exists():
                try:
                    lock.delete()
                except:
                    logging.error("Error happend when deleting lock file '%s'!" % (filename+'.lock'))
                    raise
                logging.info("The lock file '%s.lock' was deleted successfully." % filename)
            else:
                notExistedFiles.append(filename+'.lock')
        if not notExistedFiles:
            logging.info("Deleted all lock files successfully.")
            return RC_NO_ERROR
        for notExistedFile in notExistedFiles:
            logging.error("There is no lock file '%s' to be deleted." % notExistedFile)
        return RC_ERROR
Пример #2
0
    def createLockfile(self):
        '''
        USAGE: etl.py --action=createLockfile --cfg=<config> --section=<section_pattern>
        It will read the options of each section matched <section_pattern> in conf/actions/createLockfile/<config>.cfg
        EXAMPLE: python2.7 etl.py --action=createLockfile --cfg=sample --section=sample
        '''
        sections = self.getSectionsForAction(CREATE_LOCKFILE_ACTION)
        lockedFiles = []
        for section in sections:
            logging.info("\nStart processing section: %s" % section)
            # Read parameters for moving files
            filename = self._readPatternMandatoryOption(section, FILENAME)

            # Create Lockfile
            lock = Lockfile(filename)
            if lock.exists():
                lockedFiles.append(filename+'.lock')
            else:
                try:
                    lock.create()
                except:
                    logging.error("Error happend when creating lock file '%s'!" % (filename+'.lock'))
                    raise
                logging.info("The lock file '%s.lock' was created successfully." % filename)
        if not lockedFiles:
            logging.info("Created all lock files successfully.")
            return RC_NO_ERROR
        for lockedFile in lockedFiles:
            logging.warn("The lock file: '%s' was already existed." % lockedFile)
        return RC_EXISTING_PROCESS_EXISTS
Пример #3
0
def main():
    options = Options()

    # Read global configuration file
    configParser    = _readApplicationConfigurationFiles()

    #####################
    # Configure logging #
    #####################
    # Determine ML_JOBNET_NAME
    jobnet_name = ''
    if os.environ.has_key('ML_JOBNET_NAME'):
        jobnet_name = os.environ['ML_JOBNET_NAME']

    # Create log directory if it does not exist
    base_log_directory = configParser.get('directories', 'LOG_DIRECTORY')
    log_directory = "%s/%s" % (base_log_directory, jobnet_name)
    if not os.path.exists(log_directory):
        os.makedirs(log_directory)

    # Determine the log file name
    current_date = datetime.datetime.now().strftime('%Y%m%d%H%M%S')
    if os.access(log_directory, os.W_OK):
        logFilename = "%s/etl-%s.%s_%s.log" % (log_directory, options.get('action'), current_date, str(uuid.uuid4()))
    else:
        logFilename = "work/etl-%s.%s_%s.log" % (options.get('action'), current_date, str(uuid.uuid4()))

    # define a handler which writes INFO messages or higher
    console_log = logging.StreamHandler()
    console_log.setLevel(logging.INFO)

    # define a handler which writes DEBUG messages or higher
    file_log = logging.FileHandler(filename=logFilename)
    file_log.setLevel(logging.DEBUG)
    file_format = logging.Formatter('%(asctime)s [%(levelname)-7s] (%(filename)s,%(lineno)d) %(message)s')
    file_log.setFormatter(file_format)

    logging.getLogger().addHandler(console_log)
    logging.getLogger().addHandler(file_log)
    logging.getLogger().setLevel(logging.DEBUG)

    ############################
    # Dynamically Load Modules #
    ############################
    actionLabels = {'actions.py':'General Actions'}
    validActions = {}
    modules      = {}
    for file in os.listdir('actions'):
        if (file.strip().endswith('.py') and file.strip() != '__init__.py'):
            # Dynamically Load the Module
            moduleName    = file.split('.py')[0]
            className     = "%sActions" % (_convertToCamelcase(moduleName))
            modulePath    = "actions.%s" % (moduleName)
            _temp = __import__(modulePath, globals(), locals(), ['object'], -1)

            # Dynamically get an instance of the given class
            classInstance = getattr(_temp, className)()

            # Find all executable actions
            validActions = _findCallableActions(classInstance, moduleName, validActions)

    # Validate if the action is valid. Print valid actions if no valid action is found
    if (options.get('action') not in validActions):
        logging.warn("%s is not a valid action. Please specify an action from the list below" % (options.get('action')))
        actions = validActions.keys()

        # Group All Actions by their class descriptions
        actionsByClass = {}
        for action in actions:
            classDescription = validActions[action][1]
            if (actionsByClass.has_key(classDescription) == True):
               tmp = actionsByClass[classDescription]
               tmp.append(action)
               actionsByClass[classDescription] = tmp
            else:
               actionsByClass[classDescription] = [action]

        for classDesc in actionsByClass:
            print "\n%s" % (classDesc)
            print "----------------------"
            actions = actionsByClass[classDesc]
            actions.sort()
            for action in actions:
               print action
        sys.exit(197)

    # Determine the JC_JOBID Environment Value
    if (os.environ.has_key('JC_JOBID')):
       jcJobId = os.environ['JC_JOBID']
    else:
       jcJobId = ''

    # Check if a lock file exists, exit if it does
    lockfile = Lockfile(options.get('action'))
    if (lockfile.exists() == True):
       logging.warn('Action is currently running!')
       sys.exit(198)

    # Log the start of the process
    msg = "Starting the '%s' action" % (options.get('action'))
    logging.debug(msg)

    # Call the requested action, if an error occurs, log the error and send an email
    try:
       func       = validActions[options.get('action')][0]
       returnCode = func()
    except FileConfigMissing as fcm:
       logging.error(fcm)
       returnCode = 101
    except MandatoryOptionMissing as mom:
       logging.error(mom)
       returnCode = 102
    except Exception,e:
       logging.error(e)
       exc_type, exc_value, exc_traceback = sys.exc_info()
       tb                                 = traceback.extract_tb(exc_traceback)
       formattedTb                        = traceback.format_list(tb)
       # All traceback concatenate for output
       tbOutputLine = ''
       for line in formattedTb:
           tbOutputLine = tbOutputLine + line
       logging.error(tbOutputLine)
       if (lockfile.exists() == True):
           lockfile.delete()
       returnCode = 199  #TODO: make this look up the specific action's section and see if a different return code is defined