def generateUsageRecords(self, hostname, user_map, project_map):
"""
Starts the UR generation process.
"""
self.missing_user_mappings = {}
# Creates 5 Usage Record object
for count in [1,2,3,4,5]:
self.state = self.state + 1
# create some data at random...
job_id = str(self.state)
account_name = 'default'
user_name = 'default'
submit_time = time.mktime(common.datetimeFromIsoStr('2012-01-01T00:00:00').timetuple())
start_time = time.mktime(common.datetimeFromIsoStr('2012-01-02T01:23:45').timetuple())
end_time = time.mktime(common.datetimeFromIsoStr('2012-01-02T02:34:56').timetuple())
# clean data and create various composite entries from the work load trace
fqdn_job_id = hostname + ':' + job_id
if self.idtimestamp:
record_id_timestamp = re.sub("[-:TZ]","",usagerecord.epoch2isoTime(start_time)) # remove characters
record_id = fqdn_job_id + ':' + record_id_timestamp
else:
record_id = fqdn_job_id
if not user_name in user_map.getMapping():
self.missing_user_mappings[user_name] = True
vo_info = []
if account_name is not None:
mapped_project = project_map.get(account_name)
if mapped_project is not None:
voi = usagerecord.VOInformation()
voi.type = 'lrmsurgen-projectmap'
voi.name = mapped_project
vo_info = [voi]
## fill in usage record fields
ur = usagerecord.UsageRecord()
ur.record_id = record_id
ur.local_job_id = job_id
ur.global_job_id = fqdn_job_id
ur.local_user_id = user_name
ur.global_user_name = user_map.get(user_name)
ur.machine_name = hostname
ur.queue = 'default'
ur.processors = 1
ur.node_count = 1
ur.host = hostname
ur.submit_time = usagerecord.epoch2isoTime(submit_time)
ur.start_time = usagerecord.epoch2isoTime(start_time)
ur.end_time = usagerecord.epoch2isoTime(end_time)
ur.cpu_duration = 90
ur.wall_duration = 100
ur.project_name = account_name
ur.vo_info += vo_info
common.writeUr(ur,self.cfg)
def test_datetimeFromIsoStr(self):
print "Testing datetimeFromIsoStr"
self.assertEqual(
common.datetimeFromIsoStr("2012-01-02T08:09:10").__str__(),
"2012-01-02 08:09:10", "Not Equal..")
self.assertEqual(
common.datetimeFromIsoStr("2010-10-12T18:19:20").__str__(),
"2010-10-12 18:19:20", "Not Equal..")
def createUsageRecord(self, log_entry, hostname, user_map, project_map):
"""
Creates a Usage Record object given a slurm log entry.
"""
if log_entry[1] == '' or log_entry[2] == '':
return None
# extract data from the workload trace (log_entry)
job_id = str(log_entry[0])
user_name = getpwuid(int(log_entry[1]))[0]
queue = log_entry[2]
submit_time = time.mktime(common.datetimeFromIsoStr(log_entry[3]).timetuple())
start_time = time.mktime(common.datetimeFromIsoStr(log_entry[4]).timetuple())
end_time = time.mktime(common.datetimeFromIsoStr(log_entry[5]).timetuple())
account_name = log_entry[6]
utilized_cpu = common.getSeconds(log_entry[8])
wall_time = common.getSeconds(log_entry[7])
core_count = log_entry[9]
hosts = self.getNodes(log_entry[10])
# clean data and create various composite entries from the work load trace
job_identifier = job_id
fqdn_job_id = hostname + ':' + job_id
if self.idtimestamp:
record_id_timestamp = re.sub("[-:TZ]","",usagerecord.epoch2isoTime(start_time)) # remove characters
record_id = fqdn_job_id + ':' + record_id_timestamp
else:
record_id = fqdn_job_id
if not user_name in user_map.getMapping():
self.missing_user_mappings[user_name] = True
vo_info = []
if account_name is not None:
mapped_project = project_map.get(account_name)
if mapped_project is not None:
voi = usagerecord.VOInformation()
voi.type = 'lrmsurgen-projectmap'
voi.name = mapped_project
vo_info = [voi]
## fill in usage record fields
ur = usagerecord.UsageRecord()
ur.record_id = record_id
ur.local_job_id = job_identifier
ur.global_job_id = fqdn_job_id
ur.local_user_id = user_name
ur.global_user_name = user_map.get(user_name)
ur.machine_name = hostname
ur.queue = queue
ur.processors = core_count
ur.node_count = len(hosts)
ur.host = ','.join(hosts)
ur.submit_time = usagerecord.epoch2isoTime(submit_time)
ur.start_time = usagerecord.epoch2isoTime(start_time)
ur.end_time = usagerecord.epoch2isoTime(end_time)
ur.cpu_duration = utilized_cpu
ur.wall_duration = wall_time
ur.project_name = account_name
ur.vo_info += vo_info
return ur
def createUsageRecord(self, log_entry, hostname, user_map, project_map):
"""
Creates a Usage Record object given a slurm log entry.
"""
if log_entry[1] == '' or log_entry[2] == '':
return None
# Transforms a string 'billing=5,cpu=2,mem=24G,node=1' into a dict
# { 'billing': 5, 'cpu': 2, 'mem': '24G', 'node': 1 }
tres = log_entry[9]
tresdict = dict(
(k.strip(), v.strip())
for k, v in (item.split('=')
for item in tres.split(','))) if tres else dict()
# extract data from the workload trace (log_entry)
job_id = str(log_entry[0])
user_name = getpwuid(int(log_entry[1]))[0]
queue = log_entry[2]
submit_time = time.mktime(
common.datetimeFromIsoStr(log_entry[3]).timetuple())
start_time = time.mktime(
common.datetimeFromIsoStr(log_entry[4]).timetuple())
end_time = time.mktime(
common.datetimeFromIsoStr(log_entry[5]).timetuple())
account_name = log_entry[6]
utilized_cpu = common.getSeconds(log_entry[8])
wall_time = common.getSeconds(log_entry[7])
processors = self.getProcessors(tresdict)
charge = self.getCharge(tresdict, wall_time)
hosts = self.getNodes(log_entry[10])
nnodes = int(log_entry[11])
# clean data and create various composite entries from the work load trace
job_identifier = job_id
fqdn_job_id = hostname + ':' + job_id
if self.idtimestamp:
record_id_timestamp = re.sub(
"[-:TZ]", "",
usagerecord.epoch2isoTime(start_time)) # remove characters
record_id = fqdn_job_id + ':' + record_id_timestamp
else:
record_id = fqdn_job_id
if not user_name in user_map.getMapping():
self.missing_user_mappings[user_name] = True
vo_info = []
if account_name is not None:
mapped_project = project_map.get(account_name)
if mapped_project is not None:
voi = usagerecord.VOInformation()
voi.type = 'lrmsurgen-projectmap'
voi.name = mapped_project
vo_info = [voi]
## fill in usage record fields
ur = usagerecord.UsageRecord()
ur.record_id = record_id
ur.local_job_id = job_identifier
ur.global_job_id = fqdn_job_id
ur.local_user_id = user_name
ur.global_user_name = user_map.get(user_name)
ur.machine_name = hostname
ur.queue = queue
ur.processors = processors
ur.node_count = nnodes
ur.host = ','.join(hosts)
ur.submit_time = usagerecord.epoch2isoTime(submit_time)
ur.start_time = usagerecord.epoch2isoTime(start_time)
ur.end_time = usagerecord.epoch2isoTime(end_time)
ur.cpu_duration = utilized_cpu
ur.wall_duration = wall_time
ur.project_name = account_name
ur.vo_info += vo_info
# Optional field:
if charge is not None:
ur.charge = charge
return ur
def createUsageRecord(self, log_entry, hostname, user_map, project_map):
"""
Creates a Usage Record object given a slurm log entry.
"""
if log_entry[1] == '' or log_entry[2] == '':
return None
# extract data from the workload trace (log_entry)
job_id = str(log_entry[0])
user_name = getpwuid(int(log_entry[1]))[0]
queue = log_entry[2]
submit_time = time.mktime(
common.datetimeFromIsoStr(log_entry[3]).timetuple())
start_time = time.mktime(
common.datetimeFromIsoStr(log_entry[4]).timetuple())
end_time = time.mktime(
common.datetimeFromIsoStr(log_entry[5]).timetuple())
account_name = log_entry[6]
utilized_cpu = common.getSeconds(log_entry[8])
wall_time = common.getSeconds(log_entry[7])
core_count = self.extractBillingUnit(log_entry[9])
hosts = self.getNodes(log_entry[10])
# clean data and create various composite entries from the work load trace
job_identifier = job_id
fqdn_job_id = hostname + ':' + job_id
if self.idtimestamp:
record_id_timestamp = re.sub(
"[-:TZ]", "",
usagerecord.epoch2isoTime(start_time)) # remove characters
record_id = fqdn_job_id + ':' + record_id_timestamp
else:
record_id = fqdn_job_id
if not user_name in user_map.getMapping():
self.missing_user_mappings[user_name] = True
vo_info = []
if account_name is not None:
mapped_project = project_map.get(account_name)
if mapped_project is not None:
voi = usagerecord.VOInformation()
voi.type = 'lrmsurgen-projectmap'
voi.name = mapped_project
vo_info = [voi]
## fill in usage record fields
ur = usagerecord.UsageRecord()
ur.record_id = record_id
ur.local_job_id = job_identifier
ur.global_job_id = fqdn_job_id
ur.local_user_id = user_name
ur.global_user_name = user_map.get(user_name)
ur.machine_name = hostname
ur.queue = queue
ur.processors = core_count
ur.node_count = len(hosts)
ur.host = ','.join(hosts)
ur.submit_time = usagerecord.epoch2isoTime(submit_time)
ur.start_time = usagerecord.epoch2isoTime(start_time)
ur.end_time = usagerecord.epoch2isoTime(end_time)
ur.cpu_duration = utilized_cpu
ur.wall_duration = wall_time
ur.project_name = account_name
ur.vo_info += vo_info
return ur
def generateUsageRecords(self, hostname, user_map, project_map):
"""
Starts the UR generation process.
"""
self.missing_user_mappings = {}
# Creates 5 Usage Record object
for count in [1, 2, 3, 4, 5]:
self.state = self.state + 1
# create some data at random...
job_id = str(self.state)
account_name = 'default'
user_name = 'default'
submit_time = time.mktime(
common.datetimeFromIsoStr('2012-01-01T00:00:00').timetuple())
start_time = time.mktime(
common.datetimeFromIsoStr('2012-01-02T01:23:45').timetuple())
end_time = time.mktime(
common.datetimeFromIsoStr('2012-01-02T02:34:56').timetuple())
# clean data and create various composite entries from the work load trace
fqdn_job_id = hostname + ':' + job_id
if self.idtimestamp:
record_id_timestamp = re.sub(
"[-:TZ]", "",
usagerecord.epoch2isoTime(start_time)) # remove characters
record_id = fqdn_job_id + ':' + record_id_timestamp
else:
record_id = fqdn_job_id
if not user_name in user_map.getMapping():
self.missing_user_mappings[user_name] = True
vo_info = []
if account_name is not None:
mapped_project = project_map.get(account_name)
if mapped_project is not None:
voi = usagerecord.VOInformation()
voi.type = 'lrmsurgen-projectmap'
voi.name = mapped_project
vo_info = [voi]
## fill in usage record fields
ur = usagerecord.UsageRecord()
ur.record_id = record_id
ur.local_job_id = job_id
ur.global_job_id = fqdn_job_id
ur.local_user_id = user_name
ur.global_user_name = user_map.get(user_name)
ur.machine_name = hostname
ur.queue = 'default'
ur.processors = 1
ur.node_count = 1
ur.host = hostname
ur.submit_time = usagerecord.epoch2isoTime(submit_time)
ur.start_time = usagerecord.epoch2isoTime(start_time)
ur.end_time = usagerecord.epoch2isoTime(end_time)
ur.cpu_duration = 90
ur.wall_duration = 100
ur.project_name = account_name
ur.vo_info += vo_info
common.writeUr(ur, self.cfg)
def test_datetimeFromIsoStr(self):
print "Testing datetimeFromIsoStr"
self.assertEqual(common.datetimeFromIsoStr("2012-01-02T08:09:10").__str__(), "2012-01-02 08:09:10", "Not Equal..")
self.assertEqual(common.datetimeFromIsoStr("2010-10-12T18:19:20").__str__(), "2010-10-12 18:19:20", "Not Equal..")