def generateUsageRecords(self, hostname, user_map, project_map): """ Starts the UR generation process. """ self.missing_user_mappings = {} # Creates 5 Usage Record object for count in [1,2,3,4,5]: self.state = self.state + 1 # create some data at random... job_id = str(self.state) account_name = 'default' user_name = 'default' submit_time = time.mktime(common.datetimeFromIsoStr('2012-01-01T00:00:00').timetuple()) start_time = time.mktime(common.datetimeFromIsoStr('2012-01-02T01:23:45').timetuple()) end_time = time.mktime(common.datetimeFromIsoStr('2012-01-02T02:34:56').timetuple()) # clean data and create various composite entries from the work load trace fqdn_job_id = hostname + ':' + job_id if self.idtimestamp: record_id_timestamp = re.sub("[-:TZ]","",usagerecord.epoch2isoTime(start_time)) # remove characters record_id = fqdn_job_id + ':' + record_id_timestamp else: record_id = fqdn_job_id if not user_name in user_map.getMapping(): self.missing_user_mappings[user_name] = True vo_info = [] if account_name is not None: mapped_project = project_map.get(account_name) if mapped_project is not None: voi = usagerecord.VOInformation() voi.type = 'lrmsurgen-projectmap' voi.name = mapped_project vo_info = [voi] ## fill in usage record fields ur = usagerecord.UsageRecord() ur.record_id = record_id ur.local_job_id = job_id ur.global_job_id = fqdn_job_id ur.local_user_id = user_name ur.global_user_name = user_map.get(user_name) ur.machine_name = hostname ur.queue = 'default' ur.processors = 1 ur.node_count = 1 ur.host = hostname ur.submit_time = usagerecord.epoch2isoTime(submit_time) ur.start_time = usagerecord.epoch2isoTime(start_time) ur.end_time = usagerecord.epoch2isoTime(end_time) ur.cpu_duration = 90 ur.wall_duration = 100 ur.project_name = account_name ur.vo_info += vo_info common.writeUr(ur,self.cfg)
def test_datetimeFromIsoStr(self): print "Testing datetimeFromIsoStr" self.assertEqual( common.datetimeFromIsoStr("2012-01-02T08:09:10").__str__(), "2012-01-02 08:09:10", "Not Equal..") self.assertEqual( common.datetimeFromIsoStr("2010-10-12T18:19:20").__str__(), "2010-10-12 18:19:20", "Not Equal..")
def createUsageRecord(self, log_entry, hostname, user_map, project_map): """ Creates a Usage Record object given a slurm log entry. """ if log_entry[1] == '' or log_entry[2] == '': return None # extract data from the workload trace (log_entry) job_id = str(log_entry[0]) user_name = getpwuid(int(log_entry[1]))[0] queue = log_entry[2] submit_time = time.mktime(common.datetimeFromIsoStr(log_entry[3]).timetuple()) start_time = time.mktime(common.datetimeFromIsoStr(log_entry[4]).timetuple()) end_time = time.mktime(common.datetimeFromIsoStr(log_entry[5]).timetuple()) account_name = log_entry[6] utilized_cpu = common.getSeconds(log_entry[8]) wall_time = common.getSeconds(log_entry[7]) core_count = log_entry[9] hosts = self.getNodes(log_entry[10]) # clean data and create various composite entries from the work load trace job_identifier = job_id fqdn_job_id = hostname + ':' + job_id if self.idtimestamp: record_id_timestamp = re.sub("[-:TZ]","",usagerecord.epoch2isoTime(start_time)) # remove characters record_id = fqdn_job_id + ':' + record_id_timestamp else: record_id = fqdn_job_id if not user_name in user_map.getMapping(): self.missing_user_mappings[user_name] = True vo_info = [] if account_name is not None: mapped_project = project_map.get(account_name) if mapped_project is not None: voi = usagerecord.VOInformation() voi.type = 'lrmsurgen-projectmap' voi.name = mapped_project vo_info = [voi] ## fill in usage record fields ur = usagerecord.UsageRecord() ur.record_id = record_id ur.local_job_id = job_identifier ur.global_job_id = fqdn_job_id ur.local_user_id = user_name ur.global_user_name = user_map.get(user_name) ur.machine_name = hostname ur.queue = queue ur.processors = core_count ur.node_count = len(hosts) ur.host = ','.join(hosts) ur.submit_time = usagerecord.epoch2isoTime(submit_time) ur.start_time = usagerecord.epoch2isoTime(start_time) ur.end_time = usagerecord.epoch2isoTime(end_time) ur.cpu_duration = utilized_cpu ur.wall_duration = wall_time ur.project_name = account_name ur.vo_info += vo_info return ur
def createUsageRecord(self, log_entry, hostname, user_map, project_map): """ Creates a Usage Record object given a slurm log entry. """ if log_entry[1] == '' or log_entry[2] == '': return None # Transforms a string 'billing=5,cpu=2,mem=24G,node=1' into a dict # { 'billing': 5, 'cpu': 2, 'mem': '24G', 'node': 1 } tres = log_entry[9] tresdict = dict( (k.strip(), v.strip()) for k, v in (item.split('=') for item in tres.split(','))) if tres else dict() # extract data from the workload trace (log_entry) job_id = str(log_entry[0]) user_name = getpwuid(int(log_entry[1]))[0] queue = log_entry[2] submit_time = time.mktime( common.datetimeFromIsoStr(log_entry[3]).timetuple()) start_time = time.mktime( common.datetimeFromIsoStr(log_entry[4]).timetuple()) end_time = time.mktime( common.datetimeFromIsoStr(log_entry[5]).timetuple()) account_name = log_entry[6] utilized_cpu = common.getSeconds(log_entry[8]) wall_time = common.getSeconds(log_entry[7]) processors = self.getProcessors(tresdict) charge = self.getCharge(tresdict, wall_time) hosts = self.getNodes(log_entry[10]) nnodes = int(log_entry[11]) # clean data and create various composite entries from the work load trace job_identifier = job_id fqdn_job_id = hostname + ':' + job_id if self.idtimestamp: record_id_timestamp = re.sub( "[-:TZ]", "", usagerecord.epoch2isoTime(start_time)) # remove characters record_id = fqdn_job_id + ':' + record_id_timestamp else: record_id = fqdn_job_id if not user_name in user_map.getMapping(): self.missing_user_mappings[user_name] = True vo_info = [] if account_name is not None: mapped_project = project_map.get(account_name) if mapped_project is not None: voi = usagerecord.VOInformation() voi.type = 'lrmsurgen-projectmap' voi.name = mapped_project vo_info = [voi] ## fill in usage record fields ur = usagerecord.UsageRecord() ur.record_id = record_id ur.local_job_id = job_identifier ur.global_job_id = fqdn_job_id ur.local_user_id = user_name ur.global_user_name = user_map.get(user_name) ur.machine_name = hostname ur.queue = queue ur.processors = processors ur.node_count = nnodes ur.host = ','.join(hosts) ur.submit_time = usagerecord.epoch2isoTime(submit_time) ur.start_time = usagerecord.epoch2isoTime(start_time) ur.end_time = usagerecord.epoch2isoTime(end_time) ur.cpu_duration = utilized_cpu ur.wall_duration = wall_time ur.project_name = account_name ur.vo_info += vo_info # Optional field: if charge is not None: ur.charge = charge return ur
def createUsageRecord(self, log_entry, hostname, user_map, project_map): """ Creates a Usage Record object given a slurm log entry. """ if log_entry[1] == '' or log_entry[2] == '': return None # extract data from the workload trace (log_entry) job_id = str(log_entry[0]) user_name = getpwuid(int(log_entry[1]))[0] queue = log_entry[2] submit_time = time.mktime( common.datetimeFromIsoStr(log_entry[3]).timetuple()) start_time = time.mktime( common.datetimeFromIsoStr(log_entry[4]).timetuple()) end_time = time.mktime( common.datetimeFromIsoStr(log_entry[5]).timetuple()) account_name = log_entry[6] utilized_cpu = common.getSeconds(log_entry[8]) wall_time = common.getSeconds(log_entry[7]) core_count = self.extractBillingUnit(log_entry[9]) hosts = self.getNodes(log_entry[10]) # clean data and create various composite entries from the work load trace job_identifier = job_id fqdn_job_id = hostname + ':' + job_id if self.idtimestamp: record_id_timestamp = re.sub( "[-:TZ]", "", usagerecord.epoch2isoTime(start_time)) # remove characters record_id = fqdn_job_id + ':' + record_id_timestamp else: record_id = fqdn_job_id if not user_name in user_map.getMapping(): self.missing_user_mappings[user_name] = True vo_info = [] if account_name is not None: mapped_project = project_map.get(account_name) if mapped_project is not None: voi = usagerecord.VOInformation() voi.type = 'lrmsurgen-projectmap' voi.name = mapped_project vo_info = [voi] ## fill in usage record fields ur = usagerecord.UsageRecord() ur.record_id = record_id ur.local_job_id = job_identifier ur.global_job_id = fqdn_job_id ur.local_user_id = user_name ur.global_user_name = user_map.get(user_name) ur.machine_name = hostname ur.queue = queue ur.processors = core_count ur.node_count = len(hosts) ur.host = ','.join(hosts) ur.submit_time = usagerecord.epoch2isoTime(submit_time) ur.start_time = usagerecord.epoch2isoTime(start_time) ur.end_time = usagerecord.epoch2isoTime(end_time) ur.cpu_duration = utilized_cpu ur.wall_duration = wall_time ur.project_name = account_name ur.vo_info += vo_info return ur
def generateUsageRecords(self, hostname, user_map, project_map): """ Starts the UR generation process. """ self.missing_user_mappings = {} # Creates 5 Usage Record object for count in [1, 2, 3, 4, 5]: self.state = self.state + 1 # create some data at random... job_id = str(self.state) account_name = 'default' user_name = 'default' submit_time = time.mktime( common.datetimeFromIsoStr('2012-01-01T00:00:00').timetuple()) start_time = time.mktime( common.datetimeFromIsoStr('2012-01-02T01:23:45').timetuple()) end_time = time.mktime( common.datetimeFromIsoStr('2012-01-02T02:34:56').timetuple()) # clean data and create various composite entries from the work load trace fqdn_job_id = hostname + ':' + job_id if self.idtimestamp: record_id_timestamp = re.sub( "[-:TZ]", "", usagerecord.epoch2isoTime(start_time)) # remove characters record_id = fqdn_job_id + ':' + record_id_timestamp else: record_id = fqdn_job_id if not user_name in user_map.getMapping(): self.missing_user_mappings[user_name] = True vo_info = [] if account_name is not None: mapped_project = project_map.get(account_name) if mapped_project is not None: voi = usagerecord.VOInformation() voi.type = 'lrmsurgen-projectmap' voi.name = mapped_project vo_info = [voi] ## fill in usage record fields ur = usagerecord.UsageRecord() ur.record_id = record_id ur.local_job_id = job_id ur.global_job_id = fqdn_job_id ur.local_user_id = user_name ur.global_user_name = user_map.get(user_name) ur.machine_name = hostname ur.queue = 'default' ur.processors = 1 ur.node_count = 1 ur.host = hostname ur.submit_time = usagerecord.epoch2isoTime(submit_time) ur.start_time = usagerecord.epoch2isoTime(start_time) ur.end_time = usagerecord.epoch2isoTime(end_time) ur.cpu_duration = 90 ur.wall_duration = 100 ur.project_name = account_name ur.vo_info += vo_info common.writeUr(ur, self.cfg)
def test_datetimeFromIsoStr(self): print "Testing datetimeFromIsoStr" self.assertEqual(common.datetimeFromIsoStr("2012-01-02T08:09:10").__str__(), "2012-01-02 08:09:10", "Not Equal..") self.assertEqual(common.datetimeFromIsoStr("2010-10-12T18:19:20").__str__(), "2010-10-12 18:19:20", "Not Equal..")