Exemple #1
0
def generateUsageRecords(cfg, hostname, user_map, vo_map):
    """
    Starts the UR generation process.
    """

    torque_spool_dir = config.getConfigValue(cfg, config.SECTION_TORQUE,
                                             config.TORQUE_SPOOL_DIR, config.DEFAULT_TORQUE_SPOOL_DIR)
    torque_accounting_dir = os.path.join(torque_spool_dir, 'server_priv', 'accounting')

    torque_date_today = time.strftime(TORQUE_DATE_FORMAT, time.gmtime())
    job_id, torque_date = common.getGeneratorState(cfg, TORQUE_DATE_FORMAT)

    missing_user_mappings = {}

    while True:

        log_file = os.path.join(torque_accounting_dir, torque_date)
        tlp = TorqueLogParser(log_file)
        if job_id is not None:
            tlp.spoolToEntry(job_id)

        while True:

            try:
                log_entry = tlp.getNextLogEntry()
            except IOError:
                if torque_date == torque_date_today: # todays entry might not exist yet
                    #logging.info('Error opening log file for today')
                    break
                logging.error('Error opening log file at %s for date %s' % (log_file, torque_date))
                break

            if log_entry is None:
                break # no more log entries

            job_id = log_entry['jobid']

            ur = createUsageRecord(log_entry, hostname, user_map, vo_map, missing_user_mappings)
            log_dir = config.getConfigValue(cfg, config.SECTION_COMMON, config.LOGDIR, config.DEFAULT_LOG_DIR)
            ur_dir = os.path.join(log_dir, 'urs')
            if not os.path.exists(ur_dir):
                os.makedirs(ur_dir)

            ur_file = os.path.join(ur_dir, job_id)
            ur.writeXML(ur_file)
            common.writeGeneratorState(cfg, job_id, torque_date)
            logging.info('Wrote usage record to %s' % ur_file)

            job_id = None

        if torque_date == torque_date_today:
            break

        torque_date = common.getIncrementalDate(torque_date, TORQUE_DATE_FORMAT)
        job_id = None

    if missing_user_mappings:
        users = ','.join(missing_user_mappings)
        logging.info('Missing user mapping for the following users: %s' % users)
Exemple #2
0
def generateUsageRecords(cfg, hostname, user_map, vo_map):
    """
    Starts the UR generation process.
    """

    maui_spool_dir = config.getConfigValue(cfg, config.SECTION_MAUI, config.MAUI_SPOOL_DIR,
                                           config.DEFAULT_MAUI_SPOOL_DIR)
    maui_server_host = getMauiServer(maui_spool_dir)
    maui_date_today = time.strftime(MAUI_DATE_FORMAT, time.gmtime())
    job_id, maui_date = common.getGeneratorState(cfg, MAUI_DATE_FORMAT)

    missing_user_mappings = {}

    while True:

        log_file = os.path.join(maui_spool_dir, STATS_DIR, maui_date)
        mlp = MauiLogParser(log_file)
        if job_id is not None:
            mlp.spoolToEntry(job_id)

        while True:

            try:
                log_entry = mlp.getNextLogEntry()
            except IOError:
                if maui_date == maui_date_today: # todays entry might not exist yet
                    #logging.info('Error opening log file for today')
                    break
                logging.error('Error opening log file at %s for date %s' % (log_file, maui_date))
                break

            if log_entry is None:
                break # no more log entries

            if len(log_entry) != 44:
                logging.error('Read entry with an invalid number fields:')
                logging.error(' - File %s contains entry with %i fields. First field: %s' % (log_file, len(log_entry), log_entry[0]))
                logging.error(' - No usage record will be generated from this line')
                continue

            job_id = log_entry[0]
            if not shouldGenerateUR(log_entry, user_map):
                logging.debug('Job %s: No UR will be generated.' % job_id)
                continue

            ur = createUsageRecord(log_entry, hostname, user_map, vo_map, maui_server_host, missing_user_mappings)
            log_dir = config.getConfigValue(cfg, config.SECTION_COMMON, config.LOGDIR, config.DEFAULT_LOG_DIR)
            ur_dir = os.path.join(log_dir, 'urs')
            if not os.path.exists(ur_dir):
                os.makedirs(ur_dir)

            ur_file = os.path.join(ur_dir, job_id)
            ur.writeXML(ur_file)
            common.writeGeneratorState(cfg, job_id, maui_date)
            logging.info('Wrote usage record to %s' % ur_file)

            job_id = None

        if maui_date == maui_date_today:
            break

        maui_date = common.getIncrementalDate(maui_date, MAUI_DATE_FORMAT)
        job_id = None

    if missing_user_mappings:
        users = ','.join(missing_user_mappings)
        logging.info('Missing user mapping for the following users: %s' % users)