def createUsageRecord(self, log_entry, hostname, user_map, project_map):
"""
Creates a Usage Record object given a slurm log entry.
"""
if log_entry[1] == '' or log_entry[2] == '':
return None
# extract data from the workload trace (log_entry)
job_id = str(log_entry[0])
user_name = getpwuid(int(log_entry[1]))[0]
queue = log_entry[2]
submit_time = time.mktime(common.datetimeFromIsoStr(log_entry[3]).timetuple())
start_time = time.mktime(common.datetimeFromIsoStr(log_entry[4]).timetuple())
end_time = time.mktime(common.datetimeFromIsoStr(log_entry[5]).timetuple())
account_name = log_entry[6]
utilized_cpu = common.getSeconds(log_entry[8])
wall_time = common.getSeconds(log_entry[7])
core_count = log_entry[9]
hosts = self.getNodes(log_entry[10])
# clean data and create various composite entries from the work load trace
job_identifier = job_id
fqdn_job_id = hostname + ':' + job_id
if self.idtimestamp:
record_id_timestamp = re.sub("[-:TZ]","",usagerecord.epoch2isoTime(start_time)) # remove characters
record_id = fqdn_job_id + ':' + record_id_timestamp
else:
record_id = fqdn_job_id
if not user_name in user_map.getMapping():
self.missing_user_mappings[user_name] = True
vo_info = []
if account_name is not None:
mapped_project = project_map.get(account_name)
if mapped_project is not None:
voi = usagerecord.VOInformation()
voi.type = 'lrmsurgen-projectmap'
voi.name = mapped_project
vo_info = [voi]
## fill in usage record fields
ur = usagerecord.UsageRecord()
ur.record_id = record_id
ur.local_job_id = job_identifier
ur.global_job_id = fqdn_job_id
ur.local_user_id = user_name
ur.global_user_name = user_map.get(user_name)
ur.machine_name = hostname
ur.queue = queue
ur.processors = core_count
ur.node_count = len(hosts)
ur.host = ','.join(hosts)
ur.submit_time = usagerecord.epoch2isoTime(submit_time)
ur.start_time = usagerecord.epoch2isoTime(start_time)
ur.end_time = usagerecord.epoch2isoTime(end_time)
ur.cpu_duration = utilized_cpu
ur.wall_duration = wall_time
ur.project_name = account_name
ur.vo_info += vo_info
return ur
def createUsageRecord(self, log_entry, hostname, user_map, project_map):
"""
Creates a Usage Record object given a slurm log entry.
"""
if log_entry[1] == '' or log_entry[2] == '':
return None
# Transforms a string 'billing=5,cpu=2,mem=24G,node=1' into a dict
# { 'billing': 5, 'cpu': 2, 'mem': '24G', 'node': 1 }
tres = log_entry[9]
tresdict = dict(
(k.strip(), v.strip())
for k, v in (item.split('=')
for item in tres.split(','))) if tres else dict()
# extract data from the workload trace (log_entry)
job_id = str(log_entry[0])
user_name = getpwuid(int(log_entry[1]))[0]
queue = log_entry[2]
submit_time = time.mktime(
common.datetimeFromIsoStr(log_entry[3]).timetuple())
start_time = time.mktime(
common.datetimeFromIsoStr(log_entry[4]).timetuple())
end_time = time.mktime(
common.datetimeFromIsoStr(log_entry[5]).timetuple())
account_name = log_entry[6]
utilized_cpu = common.getSeconds(log_entry[8])
wall_time = common.getSeconds(log_entry[7])
processors = self.getProcessors(tresdict)
charge = self.getCharge(tresdict, wall_time)
hosts = self.getNodes(log_entry[10])
nnodes = int(log_entry[11])
# clean data and create various composite entries from the work load trace
job_identifier = job_id
fqdn_job_id = hostname + ':' + job_id
if self.idtimestamp:
record_id_timestamp = re.sub(
"[-:TZ]", "",
usagerecord.epoch2isoTime(start_time)) # remove characters
record_id = fqdn_job_id + ':' + record_id_timestamp
else:
record_id = fqdn_job_id
if not user_name in user_map.getMapping():
self.missing_user_mappings[user_name] = True
vo_info = []
if account_name is not None:
mapped_project = project_map.get(account_name)
if mapped_project is not None:
voi = usagerecord.VOInformation()
voi.type = 'lrmsurgen-projectmap'
voi.name = mapped_project
vo_info = [voi]
## fill in usage record fields
ur = usagerecord.UsageRecord()
ur.record_id = record_id
ur.local_job_id = job_identifier
ur.global_job_id = fqdn_job_id
ur.local_user_id = user_name
ur.global_user_name = user_map.get(user_name)
ur.machine_name = hostname
ur.queue = queue
ur.processors = processors
ur.node_count = nnodes
ur.host = ','.join(hosts)
ur.submit_time = usagerecord.epoch2isoTime(submit_time)
ur.start_time = usagerecord.epoch2isoTime(start_time)
ur.end_time = usagerecord.epoch2isoTime(end_time)
ur.cpu_duration = utilized_cpu
ur.wall_duration = wall_time
ur.project_name = account_name
ur.vo_info += vo_info
# Optional field:
if charge is not None:
ur.charge = charge
return ur
def createUsageRecord(self, log_entry, hostname, user_map, project_map):
"""
Creates a Usage Record object given a slurm log entry.
"""
if log_entry[1] == '' or log_entry[2] == '':
return None
# extract data from the workload trace (log_entry)
job_id = str(log_entry[0])
user_name = getpwuid(int(log_entry[1]))[0]
queue = log_entry[2]
submit_time = time.mktime(
common.datetimeFromIsoStr(log_entry[3]).timetuple())
start_time = time.mktime(
common.datetimeFromIsoStr(log_entry[4]).timetuple())
end_time = time.mktime(
common.datetimeFromIsoStr(log_entry[5]).timetuple())
account_name = log_entry[6]
utilized_cpu = common.getSeconds(log_entry[8])
wall_time = common.getSeconds(log_entry[7])
core_count = self.extractBillingUnit(log_entry[9])
hosts = self.getNodes(log_entry[10])
# clean data and create various composite entries from the work load trace
job_identifier = job_id
fqdn_job_id = hostname + ':' + job_id
if self.idtimestamp:
record_id_timestamp = re.sub(
"[-:TZ]", "",
usagerecord.epoch2isoTime(start_time)) # remove characters
record_id = fqdn_job_id + ':' + record_id_timestamp
else:
record_id = fqdn_job_id
if not user_name in user_map.getMapping():
self.missing_user_mappings[user_name] = True
vo_info = []
if account_name is not None:
mapped_project = project_map.get(account_name)
if mapped_project is not None:
voi = usagerecord.VOInformation()
voi.type = 'lrmsurgen-projectmap'
voi.name = mapped_project
vo_info = [voi]
## fill in usage record fields
ur = usagerecord.UsageRecord()
ur.record_id = record_id
ur.local_job_id = job_identifier
ur.global_job_id = fqdn_job_id
ur.local_user_id = user_name
ur.global_user_name = user_map.get(user_name)
ur.machine_name = hostname
ur.queue = queue
ur.processors = core_count
ur.node_count = len(hosts)
ur.host = ','.join(hosts)
ur.submit_time = usagerecord.epoch2isoTime(submit_time)
ur.start_time = usagerecord.epoch2isoTime(start_time)
ur.end_time = usagerecord.epoch2isoTime(end_time)
ur.cpu_duration = utilized_cpu
ur.wall_duration = wall_time
ur.project_name = account_name
ur.vo_info += vo_info
return ur
def test_getSeconds(self):
print "Testing getSeconds()"
self.assertEqual(common.getSeconds("1:0"), 1 * 60 + 0,
"Wrong number of seconds..")
self.assertEqual(common.getSeconds("1:1:0"), 1 * 3600 + 1 * 60 + 0,
"Wrong number of seconds..")
self.assertEqual(common.getSeconds("1-1:1:0"),
1 * 86400 + 1 * 3600 + 1 * 60 + 0,
"Wrong number of seconds..")
self.assertEqual(common.getSeconds("2:3"), 2 * 60 + 3,
"Wrong number of seconds..")
self.assertEqual(common.getSeconds("1:2:3"), 1 * 3600 + 2 * 60 + 3,
"Wrong number of seconds..")
self.assertEqual(common.getSeconds("1-2:3:4"),
1 * 86400 + 2 * 3600 + 3 * 60 + 4,
"Wrong number of seconds..")
self.assertEqual(common.getSeconds("2:3.1"), 2 * 60 + 3,
"Wrong number of seconds..")
self.assertEqual(common.getSeconds("1:2:3.2"), 1 * 3600 + 2 * 60 + 3,
"Wrong number of seconds..")
self.assertEqual(common.getSeconds("4-3:2:1.33"),
4 * 86400 + 3 * 3600 + 2 * 60 + 1,
"Wrong number of seconds..")
self.assertEqual(common.getSeconds("Smurf"), -1,
"Failed to detect error...")
self.assertEqual(common.getSeconds("1:a:3"), -1,
"Failed to detect error...")
self.assertEqual(common.getSeconds(""), -1,
"Failed to detect error...")
try:
self.assertEqual(common.getSeconds("-1:2:3"), -1,
"Failed to detect error...")
except:
pass
def test_getSeconds(self):
print "Testing getSeconds()"
self.assertEqual(common.getSeconds("1:0"), 1*60+0, "Wrong number of seconds..")
self.assertEqual(common.getSeconds("1:1:0"), 1*3600+1*60+0, "Wrong number of seconds..")
self.assertEqual(common.getSeconds("1-1:1:0"), 1*86400+1*3600+1*60+0, "Wrong number of seconds..")
self.assertEqual(common.getSeconds("2:3"), 2*60+3, "Wrong number of seconds..")
self.assertEqual(common.getSeconds("1:2:3"), 1*3600+2*60+3, "Wrong number of seconds..")
self.assertEqual(common.getSeconds("1-2:3:4"), 1*86400+2*3600+3*60+4, "Wrong number of seconds..")
self.assertEqual(common.getSeconds("2:3.1"), 2*60+3, "Wrong number of seconds..")
self.assertEqual(common.getSeconds("1:2:3.2"), 1*3600+2*60+3, "Wrong number of seconds..")
self.assertEqual(common.getSeconds("4-3:2:1.33"), 4*86400+3*3600+2*60+1, "Wrong number of seconds..")
self.assertEqual(common.getSeconds("Smurf"), -1, "Failed to detect error...")
self.assertEqual(common.getSeconds("1:a:3"), -1, "Failed to detect error...")
self.assertEqual(common.getSeconds(""), -1, "Failed to detect error...")
try:
self.assertEqual(common.getSeconds("-1:2:3"), -1, "Failed to detect error...")
except:
pass
