def generateUsageRecords(cfg, hostname, user_map, vo_map):
"""
Starts the UR generation process.
"""
torque_spool_dir = config.getConfigValue(cfg, config.SECTION_TORQUE,
config.TORQUE_SPOOL_DIR, config.DEFAULT_TORQUE_SPOOL_DIR)
torque_accounting_dir = os.path.join(torque_spool_dir, 'server_priv', 'accounting')
torque_date_today = time.strftime(TORQUE_DATE_FORMAT, time.gmtime())
job_id, torque_date = common.getGeneratorState(cfg, TORQUE_DATE_FORMAT)
missing_user_mappings = {}
while True:
log_file = os.path.join(torque_accounting_dir, torque_date)
tlp = TorqueLogParser(log_file)
if job_id is not None:
tlp.spoolToEntry(job_id)
while True:
try:
log_entry = tlp.getNextLogEntry()
except IOError:
if torque_date == torque_date_today: # todays entry might not exist yet
#logging.info('Error opening log file for today')
break
logging.error('Error opening log file at %s for date %s' % (log_file, torque_date))
break
if log_entry is None:
break # no more log entries
job_id = log_entry['jobid']
ur = createUsageRecord(log_entry, hostname, user_map, vo_map, missing_user_mappings)
log_dir = config.getConfigValue(cfg, config.SECTION_COMMON, config.LOGDIR, config.DEFAULT_LOG_DIR)
ur_dir = os.path.join(log_dir, 'urs')
if not os.path.exists(ur_dir):
os.makedirs(ur_dir)
ur_file = os.path.join(ur_dir, job_id)
ur.writeXML(ur_file)
common.writeGeneratorState(cfg, job_id, torque_date)
logging.info('Wrote usage record to %s' % ur_file)
job_id = None
if torque_date == torque_date_today:
break
torque_date = common.getIncrementalDate(torque_date, TORQUE_DATE_FORMAT)
job_id = None
if missing_user_mappings:
users = ','.join(missing_user_mappings)
logging.info('Missing user mapping for the following users: %s' % users)
def generateUsageRecords(cfg, hostname, user_map, vo_map):
"""
Starts the UR generation process.
"""
maui_spool_dir = config.getConfigValue(cfg, config.SECTION_MAUI, config.MAUI_SPOOL_DIR,
config.DEFAULT_MAUI_SPOOL_DIR)
maui_server_host = getMauiServer(maui_spool_dir)
maui_date_today = time.strftime(MAUI_DATE_FORMAT, time.gmtime())
job_id, maui_date = common.getGeneratorState(cfg, MAUI_DATE_FORMAT)
missing_user_mappings = {}
while True:
log_file = os.path.join(maui_spool_dir, STATS_DIR, maui_date)
mlp = MauiLogParser(log_file)
if job_id is not None:
mlp.spoolToEntry(job_id)
while True:
try:
log_entry = mlp.getNextLogEntry()
except IOError:
if maui_date == maui_date_today: # todays entry might not exist yet
#logging.info('Error opening log file for today')
break
logging.error('Error opening log file at %s for date %s' % (log_file, maui_date))
break
if log_entry is None:
break # no more log entries
if len(log_entry) != 44:
logging.error('Read entry with an invalid number fields:')
logging.error(' - File %s contains entry with %i fields. First field: %s' % (log_file, len(log_entry), log_entry[0]))
logging.error(' - No usage record will be generated from this line')
continue
job_id = log_entry[0]
if not shouldGenerateUR(log_entry, user_map):
logging.debug('Job %s: No UR will be generated.' % job_id)
continue
ur = createUsageRecord(log_entry, hostname, user_map, vo_map, maui_server_host, missing_user_mappings)
log_dir = config.getConfigValue(cfg, config.SECTION_COMMON, config.LOGDIR, config.DEFAULT_LOG_DIR)
ur_dir = os.path.join(log_dir, 'urs')
if not os.path.exists(ur_dir):
os.makedirs(ur_dir)
ur_file = os.path.join(ur_dir, job_id)
ur.writeXML(ur_file)
common.writeGeneratorState(cfg, job_id, maui_date)
logging.info('Wrote usage record to %s' % ur_file)
job_id = None
if maui_date == maui_date_today:
break
maui_date = common.getIncrementalDate(maui_date, MAUI_DATE_FORMAT)
job_id = None
if missing_user_mappings:
users = ','.join(missing_user_mappings)
logging.info('Missing user mapping for the following users: %s' % users)