def createUsageRecord(self, log_entry, hostname, user_map, project_map): """ Creates a Usage Record object given a slurm log entry. """ if log_entry[1] == '' or log_entry[2] == '': return None # extract data from the workload trace (log_entry) job_id = str(log_entry[0]) user_name = getpwuid(int(log_entry[1]))[0] queue = log_entry[2] submit_time = time.mktime(common.datetimeFromIsoStr(log_entry[3]).timetuple()) start_time = time.mktime(common.datetimeFromIsoStr(log_entry[4]).timetuple()) end_time = time.mktime(common.datetimeFromIsoStr(log_entry[5]).timetuple()) account_name = log_entry[6] utilized_cpu = common.getSeconds(log_entry[8]) wall_time = common.getSeconds(log_entry[7]) core_count = log_entry[9] hosts = self.getNodes(log_entry[10]) # clean data and create various composite entries from the work load trace job_identifier = job_id fqdn_job_id = hostname + ':' + job_id if self.idtimestamp: record_id_timestamp = re.sub("[-:TZ]","",usagerecord.epoch2isoTime(start_time)) # remove characters record_id = fqdn_job_id + ':' + record_id_timestamp else: record_id = fqdn_job_id if not user_name in user_map.getMapping(): self.missing_user_mappings[user_name] = True vo_info = [] if account_name is not None: mapped_project = project_map.get(account_name) if mapped_project is not None: voi = usagerecord.VOInformation() voi.type = 'lrmsurgen-projectmap' voi.name = mapped_project vo_info = [voi] ## fill in usage record fields ur = usagerecord.UsageRecord() ur.record_id = record_id ur.local_job_id = job_identifier ur.global_job_id = fqdn_job_id ur.local_user_id = user_name ur.global_user_name = user_map.get(user_name) ur.machine_name = hostname ur.queue = queue ur.processors = core_count ur.node_count = len(hosts) ur.host = ','.join(hosts) ur.submit_time = usagerecord.epoch2isoTime(submit_time) ur.start_time = usagerecord.epoch2isoTime(start_time) ur.end_time = usagerecord.epoch2isoTime(end_time) ur.cpu_duration = utilized_cpu ur.wall_duration = wall_time ur.project_name = account_name ur.vo_info += vo_info return ur
def createUsageRecord(self, log_entry, hostname, user_map, project_map): """ Creates a Usage Record object given a slurm log entry. """ if log_entry[1] == '' or log_entry[2] == '': return None # Transforms a string 'billing=5,cpu=2,mem=24G,node=1' into a dict # { 'billing': 5, 'cpu': 2, 'mem': '24G', 'node': 1 } tres = log_entry[9] tresdict = dict( (k.strip(), v.strip()) for k, v in (item.split('=') for item in tres.split(','))) if tres else dict() # extract data from the workload trace (log_entry) job_id = str(log_entry[0]) user_name = getpwuid(int(log_entry[1]))[0] queue = log_entry[2] submit_time = time.mktime( common.datetimeFromIsoStr(log_entry[3]).timetuple()) start_time = time.mktime( common.datetimeFromIsoStr(log_entry[4]).timetuple()) end_time = time.mktime( common.datetimeFromIsoStr(log_entry[5]).timetuple()) account_name = log_entry[6] utilized_cpu = common.getSeconds(log_entry[8]) wall_time = common.getSeconds(log_entry[7]) processors = self.getProcessors(tresdict) charge = self.getCharge(tresdict, wall_time) hosts = self.getNodes(log_entry[10]) nnodes = int(log_entry[11]) # clean data and create various composite entries from the work load trace job_identifier = job_id fqdn_job_id = hostname + ':' + job_id if self.idtimestamp: record_id_timestamp = re.sub( "[-:TZ]", "", usagerecord.epoch2isoTime(start_time)) # remove characters record_id = fqdn_job_id + ':' + record_id_timestamp else: record_id = fqdn_job_id if not user_name in user_map.getMapping(): self.missing_user_mappings[user_name] = True vo_info = [] if account_name is not None: mapped_project = project_map.get(account_name) if mapped_project is not None: voi = usagerecord.VOInformation() voi.type = 'lrmsurgen-projectmap' voi.name = mapped_project vo_info = [voi] ## fill in usage record fields ur = usagerecord.UsageRecord() ur.record_id = record_id ur.local_job_id = job_identifier ur.global_job_id = fqdn_job_id ur.local_user_id = user_name ur.global_user_name = user_map.get(user_name) ur.machine_name = hostname ur.queue = queue ur.processors = processors ur.node_count = nnodes ur.host = ','.join(hosts) ur.submit_time = usagerecord.epoch2isoTime(submit_time) ur.start_time = usagerecord.epoch2isoTime(start_time) ur.end_time = usagerecord.epoch2isoTime(end_time) ur.cpu_duration = utilized_cpu ur.wall_duration = wall_time ur.project_name = account_name ur.vo_info += vo_info # Optional field: if charge is not None: ur.charge = charge return ur
def createUsageRecord(self, log_entry, hostname, user_map, project_map): """ Creates a Usage Record object given a slurm log entry. """ if log_entry[1] == '' or log_entry[2] == '': return None # extract data from the workload trace (log_entry) job_id = str(log_entry[0]) user_name = getpwuid(int(log_entry[1]))[0] queue = log_entry[2] submit_time = time.mktime( common.datetimeFromIsoStr(log_entry[3]).timetuple()) start_time = time.mktime( common.datetimeFromIsoStr(log_entry[4]).timetuple()) end_time = time.mktime( common.datetimeFromIsoStr(log_entry[5]).timetuple()) account_name = log_entry[6] utilized_cpu = common.getSeconds(log_entry[8]) wall_time = common.getSeconds(log_entry[7]) core_count = self.extractBillingUnit(log_entry[9]) hosts = self.getNodes(log_entry[10]) # clean data and create various composite entries from the work load trace job_identifier = job_id fqdn_job_id = hostname + ':' + job_id if self.idtimestamp: record_id_timestamp = re.sub( "[-:TZ]", "", usagerecord.epoch2isoTime(start_time)) # remove characters record_id = fqdn_job_id + ':' + record_id_timestamp else: record_id = fqdn_job_id if not user_name in user_map.getMapping(): self.missing_user_mappings[user_name] = True vo_info = [] if account_name is not None: mapped_project = project_map.get(account_name) if mapped_project is not None: voi = usagerecord.VOInformation() voi.type = 'lrmsurgen-projectmap' voi.name = mapped_project vo_info = [voi] ## fill in usage record fields ur = usagerecord.UsageRecord() ur.record_id = record_id ur.local_job_id = job_identifier ur.global_job_id = fqdn_job_id ur.local_user_id = user_name ur.global_user_name = user_map.get(user_name) ur.machine_name = hostname ur.queue = queue ur.processors = core_count ur.node_count = len(hosts) ur.host = ','.join(hosts) ur.submit_time = usagerecord.epoch2isoTime(submit_time) ur.start_time = usagerecord.epoch2isoTime(start_time) ur.end_time = usagerecord.epoch2isoTime(end_time) ur.cpu_duration = utilized_cpu ur.wall_duration = wall_time ur.project_name = account_name ur.vo_info += vo_info return ur
def test_getSeconds(self): print "Testing getSeconds()" self.assertEqual(common.getSeconds("1:0"), 1 * 60 + 0, "Wrong number of seconds..") self.assertEqual(common.getSeconds("1:1:0"), 1 * 3600 + 1 * 60 + 0, "Wrong number of seconds..") self.assertEqual(common.getSeconds("1-1:1:0"), 1 * 86400 + 1 * 3600 + 1 * 60 + 0, "Wrong number of seconds..") self.assertEqual(common.getSeconds("2:3"), 2 * 60 + 3, "Wrong number of seconds..") self.assertEqual(common.getSeconds("1:2:3"), 1 * 3600 + 2 * 60 + 3, "Wrong number of seconds..") self.assertEqual(common.getSeconds("1-2:3:4"), 1 * 86400 + 2 * 3600 + 3 * 60 + 4, "Wrong number of seconds..") self.assertEqual(common.getSeconds("2:3.1"), 2 * 60 + 3, "Wrong number of seconds..") self.assertEqual(common.getSeconds("1:2:3.2"), 1 * 3600 + 2 * 60 + 3, "Wrong number of seconds..") self.assertEqual(common.getSeconds("4-3:2:1.33"), 4 * 86400 + 3 * 3600 + 2 * 60 + 1, "Wrong number of seconds..") self.assertEqual(common.getSeconds("Smurf"), -1, "Failed to detect error...") self.assertEqual(common.getSeconds("1:a:3"), -1, "Failed to detect error...") self.assertEqual(common.getSeconds(""), -1, "Failed to detect error...") try: self.assertEqual(common.getSeconds("-1:2:3"), -1, "Failed to detect error...") except: pass
def test_getSeconds(self): print "Testing getSeconds()" self.assertEqual(common.getSeconds("1:0"), 1*60+0, "Wrong number of seconds..") self.assertEqual(common.getSeconds("1:1:0"), 1*3600+1*60+0, "Wrong number of seconds..") self.assertEqual(common.getSeconds("1-1:1:0"), 1*86400+1*3600+1*60+0, "Wrong number of seconds..") self.assertEqual(common.getSeconds("2:3"), 2*60+3, "Wrong number of seconds..") self.assertEqual(common.getSeconds("1:2:3"), 1*3600+2*60+3, "Wrong number of seconds..") self.assertEqual(common.getSeconds("1-2:3:4"), 1*86400+2*3600+3*60+4, "Wrong number of seconds..") self.assertEqual(common.getSeconds("2:3.1"), 2*60+3, "Wrong number of seconds..") self.assertEqual(common.getSeconds("1:2:3.2"), 1*3600+2*60+3, "Wrong number of seconds..") self.assertEqual(common.getSeconds("4-3:2:1.33"), 4*86400+3*3600+2*60+1, "Wrong number of seconds..") self.assertEqual(common.getSeconds("Smurf"), -1, "Failed to detect error...") self.assertEqual(common.getSeconds("1:a:3"), -1, "Failed to detect error...") self.assertEqual(common.getSeconds(""), -1, "Failed to detect error...") try: self.assertEqual(common.getSeconds("-1:2:3"), -1, "Failed to detect error...") except: pass