예제 #1
0
파일: example.py 프로젝트: hpc2n/bart
    def generateUsageRecords(self, hostname, user_map, project_map):
        """
        Starts the UR generation process.
        """
        self.missing_user_mappings = {}
        
        # Creates 5 Usage Record object
        for count in [1,2,3,4,5]:
            self.state = self.state + 1
            
            # create some data at random...
            job_id       = str(self.state)      
            account_name = 'default'
            user_name    = 'default'
            submit_time  = time.mktime(common.datetimeFromIsoStr('2012-01-01T00:00:00').timetuple())
            start_time   = time.mktime(common.datetimeFromIsoStr('2012-01-02T01:23:45').timetuple())
            end_time     = time.mktime(common.datetimeFromIsoStr('2012-01-02T02:34:56').timetuple())

            # clean data and create various composite entries from the work load trace
            fqdn_job_id = hostname + ':' + job_id
            if self.idtimestamp:
                record_id_timestamp = re.sub("[-:TZ]","",usagerecord.epoch2isoTime(start_time)) # remove characters
                record_id = fqdn_job_id + ':' + record_id_timestamp
            else:
                record_id = fqdn_job_id

            if not user_name in user_map.getMapping():
                self.missing_user_mappings[user_name] = True

            vo_info = []
            if account_name is not None:
                mapped_project = project_map.get(account_name)
                if mapped_project is not None:
                    voi = usagerecord.VOInformation()
                    voi.type = 'lrmsurgen-projectmap'
                    voi.name = mapped_project
                    vo_info = [voi]

            ## fill in usage record fields
            ur = usagerecord.UsageRecord()
            ur.record_id        = record_id
            ur.local_job_id     = job_id
            ur.global_job_id    = fqdn_job_id
            ur.local_user_id    = user_name
            ur.global_user_name = user_map.get(user_name)
            ur.machine_name     = hostname
            ur.queue            = 'default'
            ur.processors       = 1
            ur.node_count       = 1
            ur.host             = hostname
            ur.submit_time      = usagerecord.epoch2isoTime(submit_time)
            ur.start_time       = usagerecord.epoch2isoTime(start_time)
            ur.end_time         = usagerecord.epoch2isoTime(end_time)
            ur.cpu_duration     = 90
            ur.wall_duration    = 100
            ur.project_name     = account_name
            ur.vo_info         += vo_info

            common.writeUr(ur,self.cfg)
예제 #2
0
 def test_datetimeFromIsoStr(self):
     print "Testing datetimeFromIsoStr"
     self.assertEqual(
         common.datetimeFromIsoStr("2012-01-02T08:09:10").__str__(),
         "2012-01-02 08:09:10", "Not Equal..")
     self.assertEqual(
         common.datetimeFromIsoStr("2010-10-12T18:19:20").__str__(),
         "2010-10-12 18:19:20", "Not Equal..")
예제 #3
0
파일: slurm.py 프로젝트: paran1/bart
    def createUsageRecord(self, log_entry, hostname, user_map, project_map):
        """
        Creates a Usage Record object given a slurm log entry.
        """
        
        if log_entry[1] == '' or log_entry[2] == '':
            return None

        # extract data from the workload trace (log_entry)
        job_id       = str(log_entry[0])
        user_name    = getpwuid(int(log_entry[1]))[0]
        queue        = log_entry[2]
        submit_time  = time.mktime(common.datetimeFromIsoStr(log_entry[3]).timetuple())
        start_time   = time.mktime(common.datetimeFromIsoStr(log_entry[4]).timetuple())
        end_time     = time.mktime(common.datetimeFromIsoStr(log_entry[5]).timetuple())
        account_name = log_entry[6]
        utilized_cpu = common.getSeconds(log_entry[8])
        wall_time    = common.getSeconds(log_entry[7])
        core_count   = log_entry[9]
        hosts        = self.getNodes(log_entry[10])

        # clean data and create various composite entries from the work load trace
        job_identifier = job_id
        fqdn_job_id = hostname + ':' + job_id
        if self.idtimestamp:
            record_id_timestamp = re.sub("[-:TZ]","",usagerecord.epoch2isoTime(start_time)) # remove characters
            record_id = fqdn_job_id + ':' + record_id_timestamp
        else:
            record_id = fqdn_job_id

        if not user_name in user_map.getMapping():
            self.missing_user_mappings[user_name] = True

        vo_info = []
        if account_name is not None:
            mapped_project = project_map.get(account_name)
            if mapped_project is not None:
                voi = usagerecord.VOInformation()
                voi.type = 'lrmsurgen-projectmap'
                voi.name = mapped_project
                vo_info = [voi]

        ## fill in usage record fields
        ur = usagerecord.UsageRecord()
        ur.record_id        = record_id
        ur.local_job_id     = job_identifier
        ur.global_job_id    = fqdn_job_id
        ur.local_user_id    = user_name
        ur.global_user_name = user_map.get(user_name)
        ur.machine_name     = hostname
        ur.queue            = queue
        ur.processors       = core_count
        ur.node_count       = len(hosts)
        ur.host             = ','.join(hosts)
        ur.submit_time      = usagerecord.epoch2isoTime(submit_time)
        ur.start_time       = usagerecord.epoch2isoTime(start_time)
        ur.end_time         = usagerecord.epoch2isoTime(end_time)
        ur.cpu_duration     = utilized_cpu
        ur.wall_duration    = wall_time
        ur.project_name     = account_name
        ur.vo_info         += vo_info

        return ur
예제 #4
0
    def createUsageRecord(self, log_entry, hostname, user_map, project_map):
        """
        Creates a Usage Record object given a slurm log entry.
        """

        if log_entry[1] == '' or log_entry[2] == '':
            return None

        # Transforms a string 'billing=5,cpu=2,mem=24G,node=1' into a dict
        # { 'billing': 5, 'cpu': 2, 'mem': '24G', 'node': 1 }
        tres = log_entry[9]
        tresdict = dict(
            (k.strip(), v.strip())
            for k, v in (item.split('=')
                         for item in tres.split(','))) if tres else dict()

        # extract data from the workload trace (log_entry)
        job_id = str(log_entry[0])
        user_name = getpwuid(int(log_entry[1]))[0]
        queue = log_entry[2]
        submit_time = time.mktime(
            common.datetimeFromIsoStr(log_entry[3]).timetuple())
        start_time = time.mktime(
            common.datetimeFromIsoStr(log_entry[4]).timetuple())
        end_time = time.mktime(
            common.datetimeFromIsoStr(log_entry[5]).timetuple())
        account_name = log_entry[6]
        utilized_cpu = common.getSeconds(log_entry[8])
        wall_time = common.getSeconds(log_entry[7])
        processors = self.getProcessors(tresdict)
        charge = self.getCharge(tresdict, wall_time)
        hosts = self.getNodes(log_entry[10])
        nnodes = int(log_entry[11])

        # clean data and create various composite entries from the work load trace
        job_identifier = job_id
        fqdn_job_id = hostname + ':' + job_id
        if self.idtimestamp:
            record_id_timestamp = re.sub(
                "[-:TZ]", "",
                usagerecord.epoch2isoTime(start_time))  # remove characters
            record_id = fqdn_job_id + ':' + record_id_timestamp
        else:
            record_id = fqdn_job_id

        if not user_name in user_map.getMapping():
            self.missing_user_mappings[user_name] = True

        vo_info = []
        if account_name is not None:
            mapped_project = project_map.get(account_name)
            if mapped_project is not None:
                voi = usagerecord.VOInformation()
                voi.type = 'lrmsurgen-projectmap'
                voi.name = mapped_project
                vo_info = [voi]

        ## fill in usage record fields
        ur = usagerecord.UsageRecord()
        ur.record_id = record_id
        ur.local_job_id = job_identifier
        ur.global_job_id = fqdn_job_id
        ur.local_user_id = user_name
        ur.global_user_name = user_map.get(user_name)
        ur.machine_name = hostname
        ur.queue = queue
        ur.processors = processors
        ur.node_count = nnodes
        ur.host = ','.join(hosts)
        ur.submit_time = usagerecord.epoch2isoTime(submit_time)
        ur.start_time = usagerecord.epoch2isoTime(start_time)
        ur.end_time = usagerecord.epoch2isoTime(end_time)
        ur.cpu_duration = utilized_cpu
        ur.wall_duration = wall_time
        ur.project_name = account_name
        ur.vo_info += vo_info

        # Optional field:
        if charge is not None:
            ur.charge = charge

        return ur
예제 #5
0
파일: slurm.py 프로젝트: kmjonsson/bart
    def createUsageRecord(self, log_entry, hostname, user_map, project_map):
        """
        Creates a Usage Record object given a slurm log entry.
        """

        if log_entry[1] == '' or log_entry[2] == '':
            return None

        # extract data from the workload trace (log_entry)
        job_id = str(log_entry[0])
        user_name = getpwuid(int(log_entry[1]))[0]
        queue = log_entry[2]
        submit_time = time.mktime(
            common.datetimeFromIsoStr(log_entry[3]).timetuple())
        start_time = time.mktime(
            common.datetimeFromIsoStr(log_entry[4]).timetuple())
        end_time = time.mktime(
            common.datetimeFromIsoStr(log_entry[5]).timetuple())
        account_name = log_entry[6]
        utilized_cpu = common.getSeconds(log_entry[8])
        wall_time = common.getSeconds(log_entry[7])
        core_count = self.extractBillingUnit(log_entry[9])
        hosts = self.getNodes(log_entry[10])

        # clean data and create various composite entries from the work load trace
        job_identifier = job_id
        fqdn_job_id = hostname + ':' + job_id
        if self.idtimestamp:
            record_id_timestamp = re.sub(
                "[-:TZ]", "",
                usagerecord.epoch2isoTime(start_time))  # remove characters
            record_id = fqdn_job_id + ':' + record_id_timestamp
        else:
            record_id = fqdn_job_id

        if not user_name in user_map.getMapping():
            self.missing_user_mappings[user_name] = True

        vo_info = []
        if account_name is not None:
            mapped_project = project_map.get(account_name)
            if mapped_project is not None:
                voi = usagerecord.VOInformation()
                voi.type = 'lrmsurgen-projectmap'
                voi.name = mapped_project
                vo_info = [voi]

        ## fill in usage record fields
        ur = usagerecord.UsageRecord()
        ur.record_id = record_id
        ur.local_job_id = job_identifier
        ur.global_job_id = fqdn_job_id
        ur.local_user_id = user_name
        ur.global_user_name = user_map.get(user_name)
        ur.machine_name = hostname
        ur.queue = queue
        ur.processors = core_count
        ur.node_count = len(hosts)
        ur.host = ','.join(hosts)
        ur.submit_time = usagerecord.epoch2isoTime(submit_time)
        ur.start_time = usagerecord.epoch2isoTime(start_time)
        ur.end_time = usagerecord.epoch2isoTime(end_time)
        ur.cpu_duration = utilized_cpu
        ur.wall_duration = wall_time
        ur.project_name = account_name
        ur.vo_info += vo_info

        return ur
예제 #6
0
    def generateUsageRecords(self, hostname, user_map, project_map):
        """
        Starts the UR generation process.
        """
        self.missing_user_mappings = {}

        # Creates 5 Usage Record object
        for count in [1, 2, 3, 4, 5]:
            self.state = self.state + 1

            # create some data at random...
            job_id = str(self.state)
            account_name = 'default'
            user_name = 'default'
            submit_time = time.mktime(
                common.datetimeFromIsoStr('2012-01-01T00:00:00').timetuple())
            start_time = time.mktime(
                common.datetimeFromIsoStr('2012-01-02T01:23:45').timetuple())
            end_time = time.mktime(
                common.datetimeFromIsoStr('2012-01-02T02:34:56').timetuple())

            # clean data and create various composite entries from the work load trace
            fqdn_job_id = hostname + ':' + job_id
            if self.idtimestamp:
                record_id_timestamp = re.sub(
                    "[-:TZ]", "",
                    usagerecord.epoch2isoTime(start_time))  # remove characters
                record_id = fqdn_job_id + ':' + record_id_timestamp
            else:
                record_id = fqdn_job_id

            if not user_name in user_map.getMapping():
                self.missing_user_mappings[user_name] = True

            vo_info = []
            if account_name is not None:
                mapped_project = project_map.get(account_name)
                if mapped_project is not None:
                    voi = usagerecord.VOInformation()
                    voi.type = 'lrmsurgen-projectmap'
                    voi.name = mapped_project
                    vo_info = [voi]

            ## fill in usage record fields
            ur = usagerecord.UsageRecord()
            ur.record_id = record_id
            ur.local_job_id = job_id
            ur.global_job_id = fqdn_job_id
            ur.local_user_id = user_name
            ur.global_user_name = user_map.get(user_name)
            ur.machine_name = hostname
            ur.queue = 'default'
            ur.processors = 1
            ur.node_count = 1
            ur.host = hostname
            ur.submit_time = usagerecord.epoch2isoTime(submit_time)
            ur.start_time = usagerecord.epoch2isoTime(start_time)
            ur.end_time = usagerecord.epoch2isoTime(end_time)
            ur.cpu_duration = 90
            ur.wall_duration = 100
            ur.project_name = account_name
            ur.vo_info += vo_info

            common.writeUr(ur, self.cfg)
예제 #7
0
파일: common.py 프로젝트: hpc2n/bart
 def test_datetimeFromIsoStr(self):
     print "Testing datetimeFromIsoStr"
     self.assertEqual(common.datetimeFromIsoStr("2012-01-02T08:09:10").__str__(), "2012-01-02 08:09:10", "Not Equal..")
     self.assertEqual(common.datetimeFromIsoStr("2010-10-12T18:19:20").__str__(), "2010-10-12 18:19:20", "Not Equal..")