Exemple #1
0
    def parse(self, line):
        '''
        Parses single line from accounting log file.
        '''
        # condor_history -constraint "JobStartDate > 0" -format "%s|" GlobalJobId -format "%s|" Owner -format "%d|" RemoteWallClockTime -format "%d|" RemoteUserCpu -format "%d|" RemoteSysCpu -format "%d|" JobStartDate -format "%d|" EnteredCurrentStatus -format "%d|" ResidentSetSize_RAW -format "%d|" ImageSize_RAW -format "%d|" RequestCpus
        # arcce.rl.ac.uk#2376.0#71589|tatls011|287|107|11|1435671643|1435671930|26636|26832|1|1

        values = line.strip().split('|')

        mapping = {
            'Site': lambda x: self.site_name,
            'MachineName': lambda x: self.machine_name,
            'Infrastructure': lambda x: "APEL-CREAM-HTCONDOR",
            'JobName': lambda x: x[0],
            'LocalUserID': lambda x: x[1],
            'LocalUserGroup': lambda x: "",
            'WallDuration': lambda x: int(x[2]),
            'CpuDuration': lambda x: int(x[3]) + int(x[4]),
            'StartTime': lambda x: x[5],
            'StopTime': lambda x: x[6],
            'MemoryReal': lambda x: int(x[7]),
            'MemoryVirtual': lambda x: int(x[8]),
            'Processors': lambda x: int(x[9]),
            'NodeCount': lambda x: 0
        }

        rc = {}

        for key in mapping:
            rc[key] = mapping[key](values)

        record = EventRecord()
        record.set_all(rc)
        return record
Exemple #2
0
    def parse(self, line):
        '''
        Parses single line from accounting log file.
        '''
        # condor_history -constraint "JobStartDate > 0" -format "%s|" GlobalJobId -format "%s|" Owner -format "%d|" RemoteWallClockTime -format "%d|" RemoteUserCpu -format "%d|" RemoteSysCpu -format "%d|" JobStartDate -format "%d|" EnteredCurrentStatus -format "%d|" ResidentSetSize_RAW -format "%d|" ImageSize_RAW -format "%d|" RequestCpus
        # arcce.rl.ac.uk#2376.0#71589|tatls011|287|107|11|1435671643|1435671930|26636|26832|1|1

        values = line.strip().split('|')

        mapping = {'Site'            : lambda x: self.site_name,
                   'MachineName'     : lambda x: self.machine_name,
                   'Infrastructure'  : lambda x: "APEL-CREAM-HTCONDOR",
                   'JobName'         : lambda x: x[0],
                   'LocalUserID'     : lambda x: x[1],
                   'LocalUserGroup'  : lambda x: "",
                   'WallDuration'    : lambda x: int(x[2]),
                   'CpuDuration'     : lambda x: int(x[3])+int(x[4]),
                   'StartTime'       : lambda x: x[5],
                   'StopTime'        : lambda x: x[6],
                   'MemoryReal'      : lambda x: int(x[7]),
                   'MemoryVirtual'   : lambda x: int(x[8]),
                   'Processors'      : lambda x: int(x[9]),
                   'NodeCount'       : lambda x: 0
                  }

        rc = {}

        for key in mapping:
            rc[key] = mapping[key](values)

        record = EventRecord()
        record.set_all(rc)
        return record
Exemple #3
0
    def parse(self, line):
        '''
        Parses single line from accounting log file.
        '''
        # /usr/local/bin/sacct -P -n --format=JobID,JobName,User,Group,Start,End,Elapsed,CPUTimeRAW,Partition,NCPUS,NNodes,NodeList,MaxRSS,MaxVMSize,State -j $JOBID >> /var/log/apel/slurm_acc.20130311
        # 1007|cream_612883006|dteam005|dteam|2013-03-27T17:13:41|2013-03-27T17:13:44|00:00:03|3|prod|1|1|cert-40|||COMPLETED
        
        # log.info('line: %s' % (line));
        values = line.strip().split('|')
        
        if values[14] != 'COMPLETED':
            return None

        rmem = None
        if values[12]:
            # remove 'K' string from the end
            rmem = int(values[12][:-1])

        vmem = None
        if values[13]:
            # remove 'K' string from the end
            vmem = int(values[13][:-1])

        mapping = {
                   'Site'            : lambda x: self.site_name,
                   'MachineName'     : lambda x: self.machine_name,
                   'Infrastructure'  : lambda x: "APEL-CREAM-SLURM",
                   'JobName'         : lambda x: x[0],
                   'LocalUserID'     : lambda x: x[2],
                   'LocalUserGroup'  : lambda x: x[3],
                   'WallDuration'    : lambda x: parse_time(x[6]),
                   'CpuDuration'     : lambda x: int(x[7]), 
                   # SLURM gives timestamps which are in system time.
                   'StartTime'       : lambda x: parse_local_timestamp(x[4]),
                   'StopTime'        : lambda x: parse_local_timestamp(x[5]),
                   'Queue'           : lambda x: x[9],
                   'MemoryReal'      : lambda x: rmem, # KB
                   'MemoryVirtual'   : lambda x: vmem, # KB
                   'Processors'      : lambda x: int(x[9]),
                   'NodeCount'       : lambda x: int(x[10])
        }

        rc = {}

        for key in mapping:
            rc[key] = mapping[key](values)

        assert rc['CpuDuration'] >= 0, 'Negative CpuDuration value'
        assert rc['WallDuration'] >= 0, 'Negative WallDuration value'

        record = EventRecord()
        record.set_all(rc)
        return record      
Exemple #4
0
 def parse(self, line):
     '''
     Parses single line from PBS log file.
     
     Please notice, that we use two different separators: ';' and ' '
     '''
     data = {}
     unused_date, status, jobName, rest = line.split(';')
     
     # we accept only 'E' status
     # be careful!: this parse can return None, and this is _valid_ situation
     if status != 'E':
         return None
     
     for item in rest.split():
         key, value = item.split('=', 1)
         data[key] = value
         
     if self._mpi:
         nodes, cores = _parse_mpi(data['exec_host'])
     else:
         nodes, cores = 0, 0
     
     # map each field to functions which will extract them
     mapping = {'Site'           : lambda x: self.site_name, 
                'JobName'        : lambda x: jobName, 
                'LocalUserID'    : lambda x: x['user'],
                'LocalUserGroup' : lambda x: x['group'],
                'WallDuration'   : lambda x: parse_time(x['resources_used.walltime']),
                'CpuDuration'    : lambda x: parse_time(x['resources_used.cput']),
                'StartTime'      : lambda x: int(x['start']),
                'StopTime'       : lambda x: int(x['end']),
                'Infrastructure' : lambda x: "APEL-CREAM-PBS",
                'MachineName'    : lambda x: self.machine_name,
                # remove 'kb' string from the end
                'MemoryReal'     : lambda x: int(x['resources_used.mem'][:-2]),
                'MemoryVirtual'  : lambda x: int(x['resources_used.vmem'][:-2]),
                'NodeCount'      : lambda x: nodes,
                'Processors'     : lambda x: cores}
     
     rc = {}
             
     for key in mapping:
         rc[key] = mapping[key](data)
     
     assert rc['CpuDuration'] >= 0, 'Negative CpuDuration value'
     assert rc['WallDuration'] >= 0, 'Negative WallDuration value'
     
     record = EventRecord()
     record.set_all(rc)
     return record
Exemple #5
0
    def parse(self, line):
        '''
        Parses single line from accounting log file.
        '''
        # /usr/local/bin/sacct -P -n --format=JobID,JobName,User,Group,Start,End,Elapsed,CPUTimeRAW,Partition,NCPUS,NNodes,NodeList,MaxRSS,MaxVMSize -j $JOBID >> /var/log/apel/slurm_acc.20130311
        # 667|sleep|root|root|2013-03-11T12:47:37|2013-03-11T12:47:40|00:00:03|12|debug|4|2|cloud-vm-[03-04]|560K|100904K

        # log.info('line: %s' % (line));
        values = line.strip().split('|')

        rmem = 0
        if values[12]:
            # remove 'K' string from the end
            rmem = float(values[12][:-1])

        vmem = 0
        if values[13]:
            # remove 'K' string from the end
            vmem = float(values[13][:-1])

        mapping = {
                   'Site'            : lambda x: self.site_name,
                   'MachineName'     : lambda x: self.machine_name,
                   'Infrastructure'  : lambda x: "APEL-CREAM-SLURM",
                   'JobName'         : lambda x: x[0],
                   'LocalUserID'     : lambda x: x[2],
                   'LocalUserGroup'  : lambda x: x[3],
                   'WallDuration'    : lambda x: parse_time(x[6]),
                   'CpuDuration'     : lambda x: int(float(x[7])), 
                   # need to check timezones
                   'StartTime'       : lambda x: parse_timestamp(x[4]),
                   'StopTime'        : lambda x: parse_timestamp(x[5]),
                   'Queue'           : lambda x: x[9],
                   'MemoryReal'      : lambda x: int(rmem), # KB
                   'MemoryVirtual'   : lambda x: int(vmem), # KB
                   'Processors'      : lambda x: int(x[9]),
                   'NodeCount'       : lambda x: int(x[10])
        }

        rc = {}

        for key in mapping:
            rc[key] = mapping[key](values)

        assert rc['CpuDuration'] >= 0, 'Negative CpuDuration value'
        assert rc['WallDuration'] >= 0, 'Negative WallDuration value'

        record = EventRecord()
        record.set_all(rc)
        return record      
Exemple #6
0
Fichier : sge.py Projet : apel/apel
    def parse(self, line):
        '''
        Parses single line from accounting log file.
        '''
        values = line.split(':')

        if self._mpi:
            procs = int(values[34])
        else:
            procs = 0

        # If a version of GE that uses millisecond timestamps is being used then
        # set the divisor to convert back to seconds.
        if self._ms_timestamps:
            divisor = 1000
        else:
            divisor = 1

        mapping = {'Site'           : lambda x: self.site_name,
                  'JobName'         : lambda x: x[5],
                  'LocalUserID'     : lambda x: x[3],
                  'LocalUserGroup'  : lambda x: x[2],
                  # int() can't parse strings like '1.000'
                  'WallDuration'    : lambda x: int(round(float(x[13]))*self._get_wall_multiplier(x[1])),
                  'CpuDuration'     : lambda x: int(round(float(x[36]))*self._get_cpu_multiplier(x[1])),
                  'StartTime'       : lambda x: int(round(float(x[9])/divisor)),
                  'StopTime'        : lambda x: int(round(float(x[10])/divisor)),
                  'Infrastructure'  : lambda x: "APEL-CREAM-SGE",
                  'MachineName'     : lambda x: self.machine_name,
                  'MemoryReal'      : lambda x: int(float(x[37])*1024*1024),  # is this correct?
                  'MemoryVirtual'   : lambda x: int(float(x[42])),
                  'Processors'      : lambda x: procs,
                  # Apparently can't get the number of WNs.
                  'NodeCount'       : lambda x: 0}

        record = EventRecord()
        data = {}

        for key in mapping:
            data[key] = mapping[key](values)

        assert data['CpuDuration'] >= 0, 'Negative CpuDuration value'
        assert data['WallDuration'] >= 0, 'Negative WallDuration value'
        assert data['StopTime'] > 0, 'Zero epoch time for field StopTime'


        record.set_all(data)

        return record
Exemple #7
0
    def parse(self, line):
        '''
        Parses single line from accounting log file.
        '''
        values = line.split(':')

        if self._mpi:
            procs = int(values[34])
        else:
            procs = 0

        mapping = {
            'Site': lambda x: self.site_name,
            'JobName': lambda x: x[5],
            'LocalUserID': lambda x: x[3],
            'LocalUserGroup': lambda x: x[2],
            # int() can't parse strings like '1.000'
            'WallDuration': lambda x: int(round(float(x[13]))),
            'CpuDuration': lambda x: int(round(float(x[36]))),
            'StartTime': lambda x: int(x[9]),
            'StopTime': lambda x: int(x[10]),
            'Infrastructure': lambda x: "APEL-CREAM-SGE",
            'MachineName': lambda x: self.machine_name,
            'MemoryReal':
            lambda x: int(float(x[37]) * 1024 * 1024),  # is this correct?
            'MemoryVirtual': lambda x: int(float(x[42])),
            'Processors': lambda x: procs,
            # Apparently can't get the number of WNs.
            'NodeCount': lambda x: 0
        }

        record = EventRecord()
        data = {}

        for key in mapping:
            data[key] = mapping[key](values)

        assert data['CpuDuration'] >= 0, 'Negative CpuDuration value'
        assert data['WallDuration'] >= 0, 'Negative WallDuration value'
        assert data['StopTime'] > 0, 'Zero epoch time for field StopTime'

        record.set_all(data)

        return record
Exemple #8
0
 def parse(self, line):
     '''
     Parses single line from accounting log file.
     '''
     values = line.split(':')
     
     if self._mpi:
         procs = int(values[34])
     else:
         procs = 0
         
     mapping = {'Site'           : lambda x: self.site_name,
               'JobName'         : lambda x: x[5],
               'LocalUserID'     : lambda x: x[3],
               'LocalUserGroup'  : lambda x: x[2],
               'WallDuration'    : lambda x: int(x[13]),
               # some kind of hack - int() can't parse strings like '1.000'
               'CpuDuration'     : lambda x: int(float(x[36])), 
               'StartTime'       : lambda x: int(x[9]),
               'StopTime'        : lambda x: int(x[10]),
               'Infrastructure'  : lambda x: "APEL-CREAM-SGE",
               'MachineName'     : lambda x: self.machine_name,
               'MemoryReal'      : lambda x: int(float(x[37])*1024*1024),  # is this correct?
               'MemoryVirtual'   : lambda x: int(float(x[42])),
               'Processors'      : lambda x: procs,
               # Apparently can't get the number of WNs.
               'NodeCount'       : lambda x: 0}
 
     record = EventRecord()
     data = {}
     
     for key in mapping:
         data[key] = mapping[key](values)
     
     assert data['CpuDuration'] >= 0, 'Negative CpuDuration value'
     assert data['WallDuration'] >= 0, 'Negative WallDuration value'
     assert data['StopTime'] > 0, 'Zero epoch time for field StopTime'
     
     
     record.set_all(data)
     
     return record
Exemple #9
0
    def parse(self, line):
        '''
        Parses single line from accounting log file.
        '''
        # Put here command that extracts log line

        data = {}

        for item in line.split("; ") :
            key, value = item.split('=', 1)
            data[key] = value

        mapping = {'Site' : lambda x: self.site_name,
            'JobName'       : lambda x: x['clusterid'] + "_" + self.machine_name,
            'LocalUserID'   : lambda x: x['owner'],
            'LocalUserGroup': lambda x: x['VO'],
            'WallDuration'  : lambda x: float(x['cputmult'])*
              (float(x['walltime+suspensiontime'])-float(x['suspensiontime'])),
            'CpuDuration'   : lambda x: float(x['cputmult'])*(float(x['cputime'])+float(x['syscputime'])),
            'StartTime'     : lambda x: int(x['startdate']),
            'StopTime'      : lambda x: int(x['enddate']),
            'Infrastructure': lambda x: "APEL-HTCONDOR",
            'MachineName'   : lambda x: self.machine_name,
            # remove 'kb' string from the end
            'MemoryReal'    : lambda x: int(x['pmem']),
            'MemoryVirtual' : lambda x: int(x['vmem']),
            'NodeCount'     : lambda x: 1,
            'Processors'    : lambda x: int(x['request_cpus'])
        }

        rc = {}

        for key in mapping:
            rc[key] = mapping[key](data)

        record = EventRecord()
        record.set_all(rc)
        return record
Exemple #10
0
    def parse(self, line):
        '''
        Parses single line from PBS log file.

        Please notice, that we use two different separators: ';' and ' '
        '''
        data = {}
        unused_date, status, jobName, rest = line.split(';')

        # we accept only 'E' status
        # be careful!: this parse can return None, and this is _valid_ situation
        if status != 'E':
            return None

        for item in rest.split():
            key, value = item.split('=', 1)
            data[key] = value

        if self._mpi:
            nodes, cores = _parse_mpi(data['exec_host'])
        else:
            nodes, cores = 0, 0

        wall_function = parse_time
        cput_function = parse_time

        # Different versions Torque use different time formats for for cput and
        # walltime (either seconds or hh:mm:ss) so check for that here.
        if ':' not in data['resources_used.walltime']:
            # Although the duration doesn't need converting if it's already in
            # seconds, this needs to be a function to work with later code.
            wall_function = lambda y: y
        if ':' not in data['resources_used.cput']:
            cput_function = lambda y: y

        # map each field to functions which will extract them
        mapping = {
            'Site': lambda x: self.site_name,
            'JobName': lambda x: jobName,
            'LocalUserID': lambda x: x['user'],
            'LocalUserGroup': lambda x: x['group'],
            'WallDuration':
            lambda x: wall_function(x['resources_used.walltime']),
            'CpuDuration': lambda x: cput_function(x['resources_used.cput']),
            'StartTime': lambda x: int(x['start']),
            'StopTime': lambda x: int(x['end']),
            'Infrastructure': lambda x: "APEL-CREAM-PBS",
            'MachineName': lambda x: self.machine_name,
            # remove 'kb' string from the end
            'MemoryReal': lambda x: int(x['resources_used.mem'][:-2]),
            'MemoryVirtual': lambda x: int(x['resources_used.vmem'][:-2]),
            'NodeCount': lambda x: nodes,
            'Processors': lambda x: cores
        }

        rc = {}

        for key in mapping:
            rc[key] = mapping[key](data)

        # Input checking
        if rc['CpuDuration'] < 0:
            raise ValueError("Negative 'cput' value")
        if rc['WallDuration'] < 0:
            raise ValueError("Negative 'walltime' value")
        if rc['StopTime'] < rc['StartTime']:
            raise ValueError("'end' time less than 'start' time")

        record = EventRecord()
        record.set_all(rc)
        return record
Exemple #11
0
    def parse(self, line):

        # correct handling of double quotes
        # expression <condition and expr1 or expr2> in Python
        # means the same as <condition ? expr1 : expr2> in C
        # the later implementations of Python introduced syntax:
        # <expr1 if condition else expr2>
        items = [x[0].startswith('"') and x[0][1:-1].replace('""', '"') or x[0]
                for x in self.EXPR.findall(line)]

        if items[0] != 'JOB_FINISH':
            return None


        num_asked = int(items[22])
        num_exec = int(items[23 + num_asked])
        offset = num_asked + num_exec

        # scale by host factor if option is chosen
        if self._scale_hf:
            host_factor = float(items[25 + num_asked + num_exec])
        else:
            host_factor = 1

        if self._mpi:
            # get unique values for the different hosts listed after num_exec
            nnodes = len(set(items[24 + num_asked:24 + offset]))
            ncores = num_exec
        else:
            nnodes = 0
            ncores = 0

        mapping = {'Site'          : lambda x: self.site_name,
                   'JobName'       : lambda x: x[3],
                   'LocalUserID'   : lambda x: x[11],
                   'LocalUserGroup': lambda x: "",
                   'WallDuration'  : lambda x: int(host_factor * (int(x[2]) - int(x[10]))),
                   'CpuDuration'   : lambda x: int(round(host_factor * (float(x[28+offset]) + float(x[29+offset])))),
                   'StartTime'     : lambda x: int(x[10]),
                   'StopTime'      : lambda x: int(x[2]),
                   'Infrastructure': lambda x: "APEL-CREAM-LSF",
                   'Queue'         : lambda x: x[12],
                   'MachineName'   : lambda x: self.machine_name,
                   'MemoryReal'    : lambda x: int(x[54+offset]) > 0 and int(x[54+offset]) or 0,
                   'MemoryVirtual' : lambda x: int(x[55+offset]) > 0 and int(x[55+offset]) or 0,
                   'Processors'    : lambda x: ncores,
                   'NodeCount'     : lambda x: nnodes}

        data = {}

        for key in mapping:
            data[key] = mapping[key](items)

        # Input checking
        if data['CpuDuration'] < 0:
            raise ValueError('Negative CpuDuration value')
        if data['WallDuration'] < 0:
            raise ValueError('Negative WallDuration value')
        if data['StopTime'] < data['StartTime']:
            raise ValueError('StopTime less than StartTime')

        record = EventRecord()
        record.set_all(data)
        return record
Exemple #12
0
Fichier : pbs.py Projet : apel/apel
    def parse(self, line):
        '''
        Parses single line from PBS log file.

        Please notice, that we use two different separators: ';' and ' '
        '''
        data = {}
        unused_date, status, jobName, rest = line.split(';')

        # we accept only 'E' status
        # be careful!: this parse can return None, and this is _valid_ situation
        if status != 'E':
            return None

        for item in rest.split():
            key, value = item.split('=', 1)
            data[key] = value

        if self._mpi:
            nodes, cores = _parse_mpi(data['exec_host'])
        else:
            nodes, cores = 0, 0

        wall_function = parse_time
        cput_function = parse_time

        # Different versions Torque use different time formats for for cput and
        # walltime (either seconds or hh:mm:ss) so check for that here.
        if ':' not in data['resources_used.walltime']:
            # Although the duration doesn't need converting if it's already in
            # seconds, this needs to be a function to work with later code.
            wall_function = lambda y: y
        if ':' not in data['resources_used.cput']:
            cput_function = lambda y: y

        # map each field to functions which will extract them
        mapping = {'Site'          : lambda x: self.site_name,
                   'JobName'       : lambda x: jobName,
                   'LocalUserID'   : lambda x: x['user'],
                   'LocalUserGroup': lambda x: x['group'],
                   'WallDuration'  : lambda x: wall_function(x['resources_used.walltime']),
                   'CpuDuration'   : lambda x: cput_function(x['resources_used.cput']),
                   'StartTime'     : lambda x: int(x['start']),
                   'StopTime'      : lambda x: int(x['end']),
                   'Infrastructure': lambda x: "APEL-CREAM-PBS",
                   'MachineName'   : lambda x: self.machine_name,
                   # remove 'kb' string from the end
                   'MemoryReal'    : lambda x: int(x['resources_used.mem'][:-2]),
                   'MemoryVirtual' : lambda x: int(x['resources_used.vmem'][:-2]),
                   'NodeCount'     : lambda x: nodes,
                   'Processors'    : lambda x: cores
                   }

        rc = {}

        for key in mapping:
            rc[key] = mapping[key](data)

        # Input checking
        if rc['CpuDuration'] < 0:
            raise ValueError("Negative 'cput' value")
        if rc['WallDuration'] < 0:
            raise ValueError("Negative 'walltime' value")
        if rc['StopTime'] < rc['StartTime']:
            raise ValueError("'end' time less than 'start' time")

        record = EventRecord()
        record.set_all(rc)
        return record
Exemple #13
0
    def parse(self, line):

        # correct handling of double quotes
        # expression <condition and expr1 or expr2> in Python
        # means the same as <condition ? expr1 : expr2> in C
        # the later implementations of Python introduced syntax:
        # <expr1 if condition else expr2>
        items = [x[0].startswith('"') and x[0][1:-1].replace('""', '"') or x[0] 
                for x in self.EXPR.findall(line)]

        if items[0] != 'JOB_FINISH':
            return None
        

        num_asked = int(items[22])
        num_exec = int(items[23 + num_asked])
        offset = num_asked + num_exec
        
        # scale by host factor if option is chosen
        if self._scale_hf:
            host_factor = float(items[25 + num_asked + num_exec])
        else:
            host_factor = 1
            
        if self._mpi:
            # get unique values for the different hosts listed after num_exec
            nnodes = len(set(items[24 + num_asked:24 + offset]))
            ncores = num_exec
        else:
            nnodes = 0
            ncores = 0
        
        mapping_lsf_5678 = {
                   'Site'           : lambda x: self.site_name,
                   'JobName'        : lambda x: x[3],
                   'LocalUserID'    : lambda x: x[11],
                   'LocalUserGroup' : lambda x: "",
                   'WallDuration'   : lambda x: int(host_factor * (int(x[2]) - int(x[10]))),
                   'CpuDuration'    : lambda x: int(round(host_factor * (float(x[28+offset]) + float(x[29+offset])))),
                   'StartTime'      : lambda x: int(x[10]),
                   'StopTime'       : lambda x: int(x[2]),
                   'Infrastructure' : lambda x: "APEL-CREAM-LSF",
                   'Queue'          : lambda x: x[12],
                   'MachineName'    : lambda x: self.machine_name,
                   'MemoryReal'     : lambda x: int(x[54+offset]) > 0 and int(x[54+offset]) or 0,
                   'MemoryVirtual'  : lambda x: int(x[55+offset]) > 0 and int(x[55+offset]) or 0,
                   'Processors'     : lambda x: ncores,
                   'NodeCount'      : lambda x: nnodes
                   }
        
        mapping = {
                   '5' : mapping_lsf_5678,
                   '6' : mapping_lsf_5678,
                   '7' : mapping_lsf_5678,
                   '8' : mapping_lsf_5678
                  }
        
        version = items[1][0]
        data = {}
    
        for key in mapping[version]:
            data[key] = mapping[version][key](items)
        
        assert data['CpuDuration'] >= 0, 'Negative CpuDuration value'
        assert data['WallDuration'] >= 0, 'Negative WallDuration value'
        
        record = EventRecord()
        record.set_all(data)
        return record
Exemple #14
0
    def parse(self, line):
        """Parse single line from accounting log file."""
        # Some sites will use TotalCPU rather than CPUTimeRAW

        # sacct -P -n --format=JobID,JobName,User,Group,Start,End,Elapsed,
        # CPUTimeRAW,Partition,NCPUS,NNodes,NodeList,MaxRSS,MaxVMSize,State -j
        #  $JOBID >> /var/log/apel/slurm_acc.20130311

        # 1007|cream_612883006|dteam005|dteam|2013-03-27T17:13:41|2013-03-27T17:13:44|00:00:03|3|prod|1|1|cert-40|||COMPLETED

        # log.info('line: %s' % (line));
        values = line.strip().split('|')

        # These statuses indicate the job has stopped and resources were used.
        if values[14] not in ('CANCELLED', 'COMPLETED', 'FAILED',
                              'NODE_FAIL', 'PREEMPTED', 'TIMEOUT'):
            return None

        # Select CPU time parsing function based on field used.
        if ':' not in values[7]:
            # CPUTimeRAW used which is a plain integer (as a string).
            cput_function = int
        else:
            # TotalCPU used which has the form d-h:m:s, h:m:s or m:s.s.
            cput_function = parse_time

        rmem = self._normalise_memory(values[12])

        vmem = self._normalise_memory(values[13])

        mapping = {'Site'            : lambda x: self.site_name,
                   'MachineName'     : lambda x: self.machine_name,
                   'Infrastructure'  : lambda x: "APEL-CREAM-SLURM",
                   'JobName'         : lambda x: x[0],
                   'LocalUserID'     : lambda x: x[2],
                   'LocalUserGroup'  : lambda x: x[3],
                   'WallDuration'    : lambda x: parse_time(x[6]),
                   'CpuDuration'     : lambda x: cput_function(x[7]),
                   # SLURM gives timestamps which are in system time.
                   'StartTime'       : lambda x: parse_local_timestamp(x[4]),
                   'StopTime'        : lambda x: parse_local_timestamp(x[5]),
                   'Queue'           : lambda x: x[8],
                   'MemoryReal'      : lambda x: rmem,  # KB
                   'MemoryVirtual'   : lambda x: vmem,  # KB
                   'Processors'      : lambda x: int(x[9]),
                   'NodeCount'       : lambda x: int(x[10])
                   }

        rc = {}

        for key in mapping:
            rc[key] = mapping[key](values)

        # Delete the Queue key if empty and let the Record class handle it
        # (usually by inserting the string 'None').
        if rc['Queue'] == '':
            del rc['Queue']

        # Input checking
        if rc['CpuDuration'] < 0:
            raise ValueError('Negative CpuDuration value')
        # No negative WallDuration test as parse_time prevents that.

        if rc['StopTime'] < rc['StartTime']:
            raise ValueError('StopTime less than StartTime')

        record = EventRecord()
        record.set_all(rc)
        return record
Exemple #15
0
    def parse(self, line):
        '''
        Parses single line from accounting log file.
        '''
        # /usr/local/bin/sacct -P -n --format=JobID,JobName,User,Group,Start,End
        # ,Elapsed,CPUTimeRAW,Partition,NCPUS,NNodes,NodeList,MaxRSS,MaxVMSize,S
        # tate -j $JOBID >> /var/log/apel/slurm_acc.20130311

        # 1007|cream_612883006|dteam005|dteam|2013-03-27T17:13:41|2013-03-27T17:13:44|00:00:03|3|prod|1|1|cert-40|||COMPLETED

        # log.info('line: %s' % (line));
        values = line.strip().split('|')

        if values[14] != 'COMPLETED':
            return None

        rmem = self._normalise_memory(values[12])

        vmem = self._normalise_memory(values[13])

        mapping = {'Site'            : lambda x: self.site_name,
                   'MachineName'     : lambda x: self.machine_name,
                   'Infrastructure'  : lambda x: "APEL-CREAM-SLURM",
                   'JobName'         : lambda x: x[0],
                   'LocalUserID'     : lambda x: x[2],
                   'LocalUserGroup'  : lambda x: x[3],
                   'WallDuration'    : lambda x: parse_time(x[6]),
                   'CpuDuration'     : lambda x: int(x[7]), 
                   # SLURM gives timestamps which are in system time.
                   'StartTime'       : lambda x: parse_local_timestamp(x[4]),
                   'StopTime'        : lambda x: parse_local_timestamp(x[5]),
                   'Queue'           : lambda x: x[8],
                   'MemoryReal'      : lambda x: rmem,  # KB
                   'MemoryVirtual'   : lambda x: vmem,  # KB
                   'Processors'      : lambda x: int(x[9]),
                   'NodeCount'       : lambda x: int(x[10])
                   }

        rc = {}

        for key in mapping:
            rc[key] = mapping[key](values)

        # Delete the Queue key if empty and let the Record class handle it
        # (usually by inserting the string 'None').
        if rc['Queue'] == '':
            del rc['Queue']

        # Input checking
        if rc['CpuDuration'] < 0:
            raise ValueError('Negative CpuDuration value')

        if rc['WallDuration'] < 0:
            raise ValueError('Negative WallDuration value')

        if rc['StopTime'] < rc['StartTime']:
            raise ValueError('StopTime less than StartTime')

        record = EventRecord()
        record.set_all(rc)
        return record
Exemple #16
0
    def parse(self, line):
        """Parse single line from accounting log file."""
        # Some sites will use TotalCPU rather than CPUTimeRAW

        # sacct -P -n --format=JobID,JobName,User,Group,Start,End,Elapsed,
        # CPUTimeRAW,Partition,NCPUS,NNodes,NodeList,MaxRSS,MaxVMSize,State -j
        #  $JOBID >> /var/log/apel/slurm_acc.20130311

        # 1007|cream_612883006|dteam005|dteam|2013-03-27T17:13:41|2013-03-27T17:13:44|00:00:03|3|prod|1|1|cert-40|||COMPLETED

        # log.info('line: %s' % (line));
        values = line.strip().split('|')

        # These statuses indicate the job has stopped and resources were used.
        if values[14] not in ('CANCELLED', 'COMPLETED', 'FAILED', 'NODE_FAIL',
                              'PREEMPTED', 'TIMEOUT'):
            return None

        # Select CPU time parsing function based on field used.
        if ':' not in values[7]:
            # CPUTimeRAW used which is a plain integer (as a string).
            cput_function = int
        else:
            # TotalCPU used which has the form d-h:m:s, h:m:s or m:s.s.
            cput_function = parse_time

        rmem = self._normalise_memory(values[12])

        vmem = self._normalise_memory(values[13])

        mapping = {
            'Site': lambda x: self.site_name,
            'MachineName': lambda x: self.machine_name,
            'Infrastructure': lambda x: "APEL-CREAM-SLURM",
            'JobName': lambda x: x[0],
            'LocalUserID': lambda x: x[2],
            'LocalUserGroup': lambda x: x[3],
            'WallDuration': lambda x: parse_time(x[6]),
            'CpuDuration': lambda x: cput_function(x[7]),
            # SLURM gives timestamps which are in system time.
            'StartTime': lambda x: parse_local_timestamp(x[4]),
            'StopTime': lambda x: parse_local_timestamp(x[5]),
            'Queue': lambda x: x[8],
            'MemoryReal': lambda x: rmem,  # KB
            'MemoryVirtual': lambda x: vmem,  # KB
            'Processors': lambda x: int(x[9]),
            'NodeCount': lambda x: int(x[10])
        }

        rc = {}

        for key in mapping:
            rc[key] = mapping[key](values)

        # Delete the Queue key if empty and let the Record class handle it
        # (usually by inserting the string 'None').
        if rc['Queue'] == '':
            del rc['Queue']

        # Input checking
        if rc['CpuDuration'] < 0:
            raise ValueError('Negative CpuDuration value')
        # No negative WallDuration test as parse_time prevents that.

        if rc['StopTime'] < rc['StartTime']:
            raise ValueError('StopTime less than StartTime')

        record = EventRecord()
        record.set_all(rc)
        return record