def parse(self, line):
'''
Parses single line from accounting log file.
'''
# condor_history -constraint "JobStartDate > 0" -format "%s|" GlobalJobId -format "%s|" Owner -format "%d|" RemoteWallClockTime -format "%d|" RemoteUserCpu -format "%d|" RemoteSysCpu -format "%d|" JobStartDate -format "%d|" EnteredCurrentStatus -format "%d|" ResidentSetSize_RAW -format "%d|" ImageSize_RAW -format "%d|" RequestCpus
# arcce.rl.ac.uk#2376.0#71589|tatls011|287|107|11|1435671643|1435671930|26636|26832|1|1
values = line.strip().split('|')
mapping = {
'Site': lambda x: self.site_name,
'MachineName': lambda x: self.machine_name,
'Infrastructure': lambda x: "APEL-CREAM-HTCONDOR",
'JobName': lambda x: x[0],
'LocalUserID': lambda x: x[1],
'LocalUserGroup': lambda x: "",
'WallDuration': lambda x: int(x[2]),
'CpuDuration': lambda x: int(x[3]) + int(x[4]),
'StartTime': lambda x: x[5],
'StopTime': lambda x: x[6],
'MemoryReal': lambda x: int(x[7]),
'MemoryVirtual': lambda x: int(x[8]),
'Processors': lambda x: int(x[9]),
'NodeCount': lambda x: 0
}
rc = {}
for key in mapping:
rc[key] = mapping[key](values)
record = EventRecord()
record.set_all(rc)
return record
def parse(self, line):
'''
Parses single line from accounting log file.
'''
# condor_history -constraint "JobStartDate > 0" -format "%s|" GlobalJobId -format "%s|" Owner -format "%d|" RemoteWallClockTime -format "%d|" RemoteUserCpu -format "%d|" RemoteSysCpu -format "%d|" JobStartDate -format "%d|" EnteredCurrentStatus -format "%d|" ResidentSetSize_RAW -format "%d|" ImageSize_RAW -format "%d|" RequestCpus
# arcce.rl.ac.uk#2376.0#71589|tatls011|287|107|11|1435671643|1435671930|26636|26832|1|1
values = line.strip().split('|')
mapping = {'Site' : lambda x: self.site_name,
'MachineName' : lambda x: self.machine_name,
'Infrastructure' : lambda x: "APEL-CREAM-HTCONDOR",
'JobName' : lambda x: x[0],
'LocalUserID' : lambda x: x[1],
'LocalUserGroup' : lambda x: "",
'WallDuration' : lambda x: int(x[2]),
'CpuDuration' : lambda x: int(x[3])+int(x[4]),
'StartTime' : lambda x: x[5],
'StopTime' : lambda x: x[6],
'MemoryReal' : lambda x: int(x[7]),
'MemoryVirtual' : lambda x: int(x[8]),
'Processors' : lambda x: int(x[9]),
'NodeCount' : lambda x: 0
}
rc = {}
for key in mapping:
rc[key] = mapping[key](values)
record = EventRecord()
record.set_all(rc)
return record
def parse(self, line):
'''
Parses single line from accounting log file.
'''
# /usr/local/bin/sacct -P -n --format=JobID,JobName,User,Group,Start,End,Elapsed,CPUTimeRAW,Partition,NCPUS,NNodes,NodeList,MaxRSS,MaxVMSize,State -j $JOBID >> /var/log/apel/slurm_acc.20130311
# 1007|cream_612883006|dteam005|dteam|2013-03-27T17:13:41|2013-03-27T17:13:44|00:00:03|3|prod|1|1|cert-40|||COMPLETED
# log.info('line: %s' % (line));
values = line.strip().split('|')
if values[14] != 'COMPLETED':
return None
rmem = None
if values[12]:
# remove 'K' string from the end
rmem = int(values[12][:-1])
vmem = None
if values[13]:
# remove 'K' string from the end
vmem = int(values[13][:-1])
mapping = {
'Site' : lambda x: self.site_name,
'MachineName' : lambda x: self.machine_name,
'Infrastructure' : lambda x: "APEL-CREAM-SLURM",
'JobName' : lambda x: x[0],
'LocalUserID' : lambda x: x[2],
'LocalUserGroup' : lambda x: x[3],
'WallDuration' : lambda x: parse_time(x[6]),
'CpuDuration' : lambda x: int(x[7]),
# SLURM gives timestamps which are in system time.
'StartTime' : lambda x: parse_local_timestamp(x[4]),
'StopTime' : lambda x: parse_local_timestamp(x[5]),
'Queue' : lambda x: x[9],
'MemoryReal' : lambda x: rmem, # KB
'MemoryVirtual' : lambda x: vmem, # KB
'Processors' : lambda x: int(x[9]),
'NodeCount' : lambda x: int(x[10])
}
rc = {}
for key in mapping:
rc[key] = mapping[key](values)
assert rc['CpuDuration'] >= 0, 'Negative CpuDuration value'
assert rc['WallDuration'] >= 0, 'Negative WallDuration value'
record = EventRecord()
record.set_all(rc)
return record
def parse(self, line):
'''
Parses single line from PBS log file.
Please notice, that we use two different separators: ';' and ' '
'''
data = {}
unused_date, status, jobName, rest = line.split(';')
# we accept only 'E' status
# be careful!: this parse can return None, and this is _valid_ situation
if status != 'E':
return None
for item in rest.split():
key, value = item.split('=', 1)
data[key] = value
if self._mpi:
nodes, cores = _parse_mpi(data['exec_host'])
else:
nodes, cores = 0, 0
# map each field to functions which will extract them
mapping = {'Site' : lambda x: self.site_name,
'JobName' : lambda x: jobName,
'LocalUserID' : lambda x: x['user'],
'LocalUserGroup' : lambda x: x['group'],
'WallDuration' : lambda x: parse_time(x['resources_used.walltime']),
'CpuDuration' : lambda x: parse_time(x['resources_used.cput']),
'StartTime' : lambda x: int(x['start']),
'StopTime' : lambda x: int(x['end']),
'Infrastructure' : lambda x: "APEL-CREAM-PBS",
'MachineName' : lambda x: self.machine_name,
# remove 'kb' string from the end
'MemoryReal' : lambda x: int(x['resources_used.mem'][:-2]),
'MemoryVirtual' : lambda x: int(x['resources_used.vmem'][:-2]),
'NodeCount' : lambda x: nodes,
'Processors' : lambda x: cores}
rc = {}
for key in mapping:
rc[key] = mapping[key](data)
assert rc['CpuDuration'] >= 0, 'Negative CpuDuration value'
assert rc['WallDuration'] >= 0, 'Negative WallDuration value'
record = EventRecord()
record.set_all(rc)
return record
def parse(self, line):
'''
Parses single line from accounting log file.
'''
# /usr/local/bin/sacct -P -n --format=JobID,JobName,User,Group,Start,End,Elapsed,CPUTimeRAW,Partition,NCPUS,NNodes,NodeList,MaxRSS,MaxVMSize -j $JOBID >> /var/log/apel/slurm_acc.20130311
# 667|sleep|root|root|2013-03-11T12:47:37|2013-03-11T12:47:40|00:00:03|12|debug|4|2|cloud-vm-[03-04]|560K|100904K
# log.info('line: %s' % (line));
values = line.strip().split('|')
rmem = 0
if values[12]:
# remove 'K' string from the end
rmem = float(values[12][:-1])
vmem = 0
if values[13]:
# remove 'K' string from the end
vmem = float(values[13][:-1])
mapping = {
'Site' : lambda x: self.site_name,
'MachineName' : lambda x: self.machine_name,
'Infrastructure' : lambda x: "APEL-CREAM-SLURM",
'JobName' : lambda x: x[0],
'LocalUserID' : lambda x: x[2],
'LocalUserGroup' : lambda x: x[3],
'WallDuration' : lambda x: parse_time(x[6]),
'CpuDuration' : lambda x: int(float(x[7])),
# need to check timezones
'StartTime' : lambda x: parse_timestamp(x[4]),
'StopTime' : lambda x: parse_timestamp(x[5]),
'Queue' : lambda x: x[9],
'MemoryReal' : lambda x: int(rmem), # KB
'MemoryVirtual' : lambda x: int(vmem), # KB
'Processors' : lambda x: int(x[9]),
'NodeCount' : lambda x: int(x[10])
}
rc = {}
for key in mapping:
rc[key] = mapping[key](values)
assert rc['CpuDuration'] >= 0, 'Negative CpuDuration value'
assert rc['WallDuration'] >= 0, 'Negative WallDuration value'
record = EventRecord()
record.set_all(rc)
return record
def parse(self, line):
'''
Parses single line from accounting log file.
'''
values = line.split(':')
if self._mpi:
procs = int(values[34])
else:
procs = 0
# If a version of GE that uses millisecond timestamps is being used then
# set the divisor to convert back to seconds.
if self._ms_timestamps:
divisor = 1000
else:
divisor = 1
mapping = {'Site' : lambda x: self.site_name,
'JobName' : lambda x: x[5],
'LocalUserID' : lambda x: x[3],
'LocalUserGroup' : lambda x: x[2],
# int() can't parse strings like '1.000'
'WallDuration' : lambda x: int(round(float(x[13]))*self._get_wall_multiplier(x[1])),
'CpuDuration' : lambda x: int(round(float(x[36]))*self._get_cpu_multiplier(x[1])),
'StartTime' : lambda x: int(round(float(x[9])/divisor)),
'StopTime' : lambda x: int(round(float(x[10])/divisor)),
'Infrastructure' : lambda x: "APEL-CREAM-SGE",
'MachineName' : lambda x: self.machine_name,
'MemoryReal' : lambda x: int(float(x[37])*1024*1024), # is this correct?
'MemoryVirtual' : lambda x: int(float(x[42])),
'Processors' : lambda x: procs,
# Apparently can't get the number of WNs.
'NodeCount' : lambda x: 0}
record = EventRecord()
data = {}
for key in mapping:
data[key] = mapping[key](values)
assert data['CpuDuration'] >= 0, 'Negative CpuDuration value'
assert data['WallDuration'] >= 0, 'Negative WallDuration value'
assert data['StopTime'] > 0, 'Zero epoch time for field StopTime'
record.set_all(data)
return record
def parse(self, line):
'''
Parses single line from accounting log file.
'''
values = line.split(':')
if self._mpi:
procs = int(values[34])
else:
procs = 0
mapping = {
'Site': lambda x: self.site_name,
'JobName': lambda x: x[5],
'LocalUserID': lambda x: x[3],
'LocalUserGroup': lambda x: x[2],
# int() can't parse strings like '1.000'
'WallDuration': lambda x: int(round(float(x[13]))),
'CpuDuration': lambda x: int(round(float(x[36]))),
'StartTime': lambda x: int(x[9]),
'StopTime': lambda x: int(x[10]),
'Infrastructure': lambda x: "APEL-CREAM-SGE",
'MachineName': lambda x: self.machine_name,
'MemoryReal':
lambda x: int(float(x[37]) * 1024 * 1024), # is this correct?
'MemoryVirtual': lambda x: int(float(x[42])),
'Processors': lambda x: procs,
# Apparently can't get the number of WNs.
'NodeCount': lambda x: 0
}
record = EventRecord()
data = {}
for key in mapping:
data[key] = mapping[key](values)
assert data['CpuDuration'] >= 0, 'Negative CpuDuration value'
assert data['WallDuration'] >= 0, 'Negative WallDuration value'
assert data['StopTime'] > 0, 'Zero epoch time for field StopTime'
record.set_all(data)
return record
def parse(self, line):
'''
Parses single line from accounting log file.
'''
values = line.split(':')
if self._mpi:
procs = int(values[34])
else:
procs = 0
mapping = {'Site' : lambda x: self.site_name,
'JobName' : lambda x: x[5],
'LocalUserID' : lambda x: x[3],
'LocalUserGroup' : lambda x: x[2],
'WallDuration' : lambda x: int(x[13]),
# some kind of hack - int() can't parse strings like '1.000'
'CpuDuration' : lambda x: int(float(x[36])),
'StartTime' : lambda x: int(x[9]),
'StopTime' : lambda x: int(x[10]),
'Infrastructure' : lambda x: "APEL-CREAM-SGE",
'MachineName' : lambda x: self.machine_name,
'MemoryReal' : lambda x: int(float(x[37])*1024*1024), # is this correct?
'MemoryVirtual' : lambda x: int(float(x[42])),
'Processors' : lambda x: procs,
# Apparently can't get the number of WNs.
'NodeCount' : lambda x: 0}
record = EventRecord()
data = {}
for key in mapping:
data[key] = mapping[key](values)
assert data['CpuDuration'] >= 0, 'Negative CpuDuration value'
assert data['WallDuration'] >= 0, 'Negative WallDuration value'
assert data['StopTime'] > 0, 'Zero epoch time for field StopTime'
record.set_all(data)
return record
def parse(self, line):
'''
Parses single line from accounting log file.
'''
# Put here command that extracts log line
data = {}
for item in line.split("; ") :
key, value = item.split('=', 1)
data[key] = value
mapping = {'Site' : lambda x: self.site_name,
'JobName' : lambda x: x['clusterid'] + "_" + self.machine_name,
'LocalUserID' : lambda x: x['owner'],
'LocalUserGroup': lambda x: x['VO'],
'WallDuration' : lambda x: float(x['cputmult'])*
(float(x['walltime+suspensiontime'])-float(x['suspensiontime'])),
'CpuDuration' : lambda x: float(x['cputmult'])*(float(x['cputime'])+float(x['syscputime'])),
'StartTime' : lambda x: int(x['startdate']),
'StopTime' : lambda x: int(x['enddate']),
'Infrastructure': lambda x: "APEL-HTCONDOR",
'MachineName' : lambda x: self.machine_name,
# remove 'kb' string from the end
'MemoryReal' : lambda x: int(x['pmem']),
'MemoryVirtual' : lambda x: int(x['vmem']),
'NodeCount' : lambda x: 1,
'Processors' : lambda x: int(x['request_cpus'])
}
rc = {}
for key in mapping:
rc[key] = mapping[key](data)
record = EventRecord()
record.set_all(rc)
return record
def parse(self, line):
'''
Parses single line from PBS log file.
Please notice, that we use two different separators: ';' and ' '
'''
data = {}
unused_date, status, jobName, rest = line.split(';')
# we accept only 'E' status
# be careful!: this parse can return None, and this is _valid_ situation
if status != 'E':
return None
for item in rest.split():
key, value = item.split('=', 1)
data[key] = value
if self._mpi:
nodes, cores = _parse_mpi(data['exec_host'])
else:
nodes, cores = 0, 0
wall_function = parse_time
cput_function = parse_time
# Different versions Torque use different time formats for for cput and
# walltime (either seconds or hh:mm:ss) so check for that here.
if ':' not in data['resources_used.walltime']:
# Although the duration doesn't need converting if it's already in
# seconds, this needs to be a function to work with later code.
wall_function = lambda y: y
if ':' not in data['resources_used.cput']:
cput_function = lambda y: y
# map each field to functions which will extract them
mapping = {
'Site': lambda x: self.site_name,
'JobName': lambda x: jobName,
'LocalUserID': lambda x: x['user'],
'LocalUserGroup': lambda x: x['group'],
'WallDuration':
lambda x: wall_function(x['resources_used.walltime']),
'CpuDuration': lambda x: cput_function(x['resources_used.cput']),
'StartTime': lambda x: int(x['start']),
'StopTime': lambda x: int(x['end']),
'Infrastructure': lambda x: "APEL-CREAM-PBS",
'MachineName': lambda x: self.machine_name,
# remove 'kb' string from the end
'MemoryReal': lambda x: int(x['resources_used.mem'][:-2]),
'MemoryVirtual': lambda x: int(x['resources_used.vmem'][:-2]),
'NodeCount': lambda x: nodes,
'Processors': lambda x: cores
}
rc = {}
for key in mapping:
rc[key] = mapping[key](data)
# Input checking
if rc['CpuDuration'] < 0:
raise ValueError("Negative 'cput' value")
if rc['WallDuration'] < 0:
raise ValueError("Negative 'walltime' value")
if rc['StopTime'] < rc['StartTime']:
raise ValueError("'end' time less than 'start' time")
record = EventRecord()
record.set_all(rc)
return record
def parse(self, line):
# correct handling of double quotes
# expression <condition and expr1 or expr2> in Python
# means the same as <condition ? expr1 : expr2> in C
# the later implementations of Python introduced syntax:
# <expr1 if condition else expr2>
items = [x[0].startswith('"') and x[0][1:-1].replace('""', '"') or x[0]
for x in self.EXPR.findall(line)]
if items[0] != 'JOB_FINISH':
return None
num_asked = int(items[22])
num_exec = int(items[23 + num_asked])
offset = num_asked + num_exec
# scale by host factor if option is chosen
if self._scale_hf:
host_factor = float(items[25 + num_asked + num_exec])
else:
host_factor = 1
if self._mpi:
# get unique values for the different hosts listed after num_exec
nnodes = len(set(items[24 + num_asked:24 + offset]))
ncores = num_exec
else:
nnodes = 0
ncores = 0
mapping = {'Site' : lambda x: self.site_name,
'JobName' : lambda x: x[3],
'LocalUserID' : lambda x: x[11],
'LocalUserGroup': lambda x: "",
'WallDuration' : lambda x: int(host_factor * (int(x[2]) - int(x[10]))),
'CpuDuration' : lambda x: int(round(host_factor * (float(x[28+offset]) + float(x[29+offset])))),
'StartTime' : lambda x: int(x[10]),
'StopTime' : lambda x: int(x[2]),
'Infrastructure': lambda x: "APEL-CREAM-LSF",
'Queue' : lambda x: x[12],
'MachineName' : lambda x: self.machine_name,
'MemoryReal' : lambda x: int(x[54+offset]) > 0 and int(x[54+offset]) or 0,
'MemoryVirtual' : lambda x: int(x[55+offset]) > 0 and int(x[55+offset]) or 0,
'Processors' : lambda x: ncores,
'NodeCount' : lambda x: nnodes}
data = {}
for key in mapping:
data[key] = mapping[key](items)
# Input checking
if data['CpuDuration'] < 0:
raise ValueError('Negative CpuDuration value')
if data['WallDuration'] < 0:
raise ValueError('Negative WallDuration value')
if data['StopTime'] < data['StartTime']:
raise ValueError('StopTime less than StartTime')
record = EventRecord()
record.set_all(data)
return record
def parse(self, line):
'''
Parses single line from PBS log file.
Please notice, that we use two different separators: ';' and ' '
'''
data = {}
unused_date, status, jobName, rest = line.split(';')
# we accept only 'E' status
# be careful!: this parse can return None, and this is _valid_ situation
if status != 'E':
return None
for item in rest.split():
key, value = item.split('=', 1)
data[key] = value
if self._mpi:
nodes, cores = _parse_mpi(data['exec_host'])
else:
nodes, cores = 0, 0
wall_function = parse_time
cput_function = parse_time
# Different versions Torque use different time formats for for cput and
# walltime (either seconds or hh:mm:ss) so check for that here.
if ':' not in data['resources_used.walltime']:
# Although the duration doesn't need converting if it's already in
# seconds, this needs to be a function to work with later code.
wall_function = lambda y: y
if ':' not in data['resources_used.cput']:
cput_function = lambda y: y
# map each field to functions which will extract them
mapping = {'Site' : lambda x: self.site_name,
'JobName' : lambda x: jobName,
'LocalUserID' : lambda x: x['user'],
'LocalUserGroup': lambda x: x['group'],
'WallDuration' : lambda x: wall_function(x['resources_used.walltime']),
'CpuDuration' : lambda x: cput_function(x['resources_used.cput']),
'StartTime' : lambda x: int(x['start']),
'StopTime' : lambda x: int(x['end']),
'Infrastructure': lambda x: "APEL-CREAM-PBS",
'MachineName' : lambda x: self.machine_name,
# remove 'kb' string from the end
'MemoryReal' : lambda x: int(x['resources_used.mem'][:-2]),
'MemoryVirtual' : lambda x: int(x['resources_used.vmem'][:-2]),
'NodeCount' : lambda x: nodes,
'Processors' : lambda x: cores
}
rc = {}
for key in mapping:
rc[key] = mapping[key](data)
# Input checking
if rc['CpuDuration'] < 0:
raise ValueError("Negative 'cput' value")
if rc['WallDuration'] < 0:
raise ValueError("Negative 'walltime' value")
if rc['StopTime'] < rc['StartTime']:
raise ValueError("'end' time less than 'start' time")
record = EventRecord()
record.set_all(rc)
return record
def parse(self, line):
# correct handling of double quotes
# expression <condition and expr1 or expr2> in Python
# means the same as <condition ? expr1 : expr2> in C
# the later implementations of Python introduced syntax:
# <expr1 if condition else expr2>
items = [x[0].startswith('"') and x[0][1:-1].replace('""', '"') or x[0]
for x in self.EXPR.findall(line)]
if items[0] != 'JOB_FINISH':
return None
num_asked = int(items[22])
num_exec = int(items[23 + num_asked])
offset = num_asked + num_exec
# scale by host factor if option is chosen
if self._scale_hf:
host_factor = float(items[25 + num_asked + num_exec])
else:
host_factor = 1
if self._mpi:
# get unique values for the different hosts listed after num_exec
nnodes = len(set(items[24 + num_asked:24 + offset]))
ncores = num_exec
else:
nnodes = 0
ncores = 0
mapping_lsf_5678 = {
'Site' : lambda x: self.site_name,
'JobName' : lambda x: x[3],
'LocalUserID' : lambda x: x[11],
'LocalUserGroup' : lambda x: "",
'WallDuration' : lambda x: int(host_factor * (int(x[2]) - int(x[10]))),
'CpuDuration' : lambda x: int(round(host_factor * (float(x[28+offset]) + float(x[29+offset])))),
'StartTime' : lambda x: int(x[10]),
'StopTime' : lambda x: int(x[2]),
'Infrastructure' : lambda x: "APEL-CREAM-LSF",
'Queue' : lambda x: x[12],
'MachineName' : lambda x: self.machine_name,
'MemoryReal' : lambda x: int(x[54+offset]) > 0 and int(x[54+offset]) or 0,
'MemoryVirtual' : lambda x: int(x[55+offset]) > 0 and int(x[55+offset]) or 0,
'Processors' : lambda x: ncores,
'NodeCount' : lambda x: nnodes
}
mapping = {
'5' : mapping_lsf_5678,
'6' : mapping_lsf_5678,
'7' : mapping_lsf_5678,
'8' : mapping_lsf_5678
}
version = items[1][0]
data = {}
for key in mapping[version]:
data[key] = mapping[version][key](items)
assert data['CpuDuration'] >= 0, 'Negative CpuDuration value'
assert data['WallDuration'] >= 0, 'Negative WallDuration value'
record = EventRecord()
record.set_all(data)
return record
def parse(self, line):
"""Parse single line from accounting log file."""
# Some sites will use TotalCPU rather than CPUTimeRAW
# sacct -P -n --format=JobID,JobName,User,Group,Start,End,Elapsed,
# CPUTimeRAW,Partition,NCPUS,NNodes,NodeList,MaxRSS,MaxVMSize,State -j
# $JOBID >> /var/log/apel/slurm_acc.20130311
# 1007|cream_612883006|dteam005|dteam|2013-03-27T17:13:41|2013-03-27T17:13:44|00:00:03|3|prod|1|1|cert-40|||COMPLETED
# log.info('line: %s' % (line));
values = line.strip().split('|')
# These statuses indicate the job has stopped and resources were used.
if values[14] not in ('CANCELLED', 'COMPLETED', 'FAILED',
'NODE_FAIL', 'PREEMPTED', 'TIMEOUT'):
return None
# Select CPU time parsing function based on field used.
if ':' not in values[7]:
# CPUTimeRAW used which is a plain integer (as a string).
cput_function = int
else:
# TotalCPU used which has the form d-h:m:s, h:m:s or m:s.s.
cput_function = parse_time
rmem = self._normalise_memory(values[12])
vmem = self._normalise_memory(values[13])
mapping = {'Site' : lambda x: self.site_name,
'MachineName' : lambda x: self.machine_name,
'Infrastructure' : lambda x: "APEL-CREAM-SLURM",
'JobName' : lambda x: x[0],
'LocalUserID' : lambda x: x[2],
'LocalUserGroup' : lambda x: x[3],
'WallDuration' : lambda x: parse_time(x[6]),
'CpuDuration' : lambda x: cput_function(x[7]),
# SLURM gives timestamps which are in system time.
'StartTime' : lambda x: parse_local_timestamp(x[4]),
'StopTime' : lambda x: parse_local_timestamp(x[5]),
'Queue' : lambda x: x[8],
'MemoryReal' : lambda x: rmem, # KB
'MemoryVirtual' : lambda x: vmem, # KB
'Processors' : lambda x: int(x[9]),
'NodeCount' : lambda x: int(x[10])
}
rc = {}
for key in mapping:
rc[key] = mapping[key](values)
# Delete the Queue key if empty and let the Record class handle it
# (usually by inserting the string 'None').
if rc['Queue'] == '':
del rc['Queue']
# Input checking
if rc['CpuDuration'] < 0:
raise ValueError('Negative CpuDuration value')
# No negative WallDuration test as parse_time prevents that.
if rc['StopTime'] < rc['StartTime']:
raise ValueError('StopTime less than StartTime')
record = EventRecord()
record.set_all(rc)
return record
def parse(self, line):
'''
Parses single line from accounting log file.
'''
# /usr/local/bin/sacct -P -n --format=JobID,JobName,User,Group,Start,End
# ,Elapsed,CPUTimeRAW,Partition,NCPUS,NNodes,NodeList,MaxRSS,MaxVMSize,S
# tate -j $JOBID >> /var/log/apel/slurm_acc.20130311
# 1007|cream_612883006|dteam005|dteam|2013-03-27T17:13:41|2013-03-27T17:13:44|00:00:03|3|prod|1|1|cert-40|||COMPLETED
# log.info('line: %s' % (line));
values = line.strip().split('|')
if values[14] != 'COMPLETED':
return None
rmem = self._normalise_memory(values[12])
vmem = self._normalise_memory(values[13])
mapping = {'Site' : lambda x: self.site_name,
'MachineName' : lambda x: self.machine_name,
'Infrastructure' : lambda x: "APEL-CREAM-SLURM",
'JobName' : lambda x: x[0],
'LocalUserID' : lambda x: x[2],
'LocalUserGroup' : lambda x: x[3],
'WallDuration' : lambda x: parse_time(x[6]),
'CpuDuration' : lambda x: int(x[7]),
# SLURM gives timestamps which are in system time.
'StartTime' : lambda x: parse_local_timestamp(x[4]),
'StopTime' : lambda x: parse_local_timestamp(x[5]),
'Queue' : lambda x: x[8],
'MemoryReal' : lambda x: rmem, # KB
'MemoryVirtual' : lambda x: vmem, # KB
'Processors' : lambda x: int(x[9]),
'NodeCount' : lambda x: int(x[10])
}
rc = {}
for key in mapping:
rc[key] = mapping[key](values)
# Delete the Queue key if empty and let the Record class handle it
# (usually by inserting the string 'None').
if rc['Queue'] == '':
del rc['Queue']
# Input checking
if rc['CpuDuration'] < 0:
raise ValueError('Negative CpuDuration value')
if rc['WallDuration'] < 0:
raise ValueError('Negative WallDuration value')
if rc['StopTime'] < rc['StartTime']:
raise ValueError('StopTime less than StartTime')
record = EventRecord()
record.set_all(rc)
return record
def parse(self, line):
"""Parse single line from accounting log file."""
# Some sites will use TotalCPU rather than CPUTimeRAW
# sacct -P -n --format=JobID,JobName,User,Group,Start,End,Elapsed,
# CPUTimeRAW,Partition,NCPUS,NNodes,NodeList,MaxRSS,MaxVMSize,State -j
# $JOBID >> /var/log/apel/slurm_acc.20130311
# 1007|cream_612883006|dteam005|dteam|2013-03-27T17:13:41|2013-03-27T17:13:44|00:00:03|3|prod|1|1|cert-40|||COMPLETED
# log.info('line: %s' % (line));
values = line.strip().split('|')
# These statuses indicate the job has stopped and resources were used.
if values[14] not in ('CANCELLED', 'COMPLETED', 'FAILED', 'NODE_FAIL',
'PREEMPTED', 'TIMEOUT'):
return None
# Select CPU time parsing function based on field used.
if ':' not in values[7]:
# CPUTimeRAW used which is a plain integer (as a string).
cput_function = int
else:
# TotalCPU used which has the form d-h:m:s, h:m:s or m:s.s.
cput_function = parse_time
rmem = self._normalise_memory(values[12])
vmem = self._normalise_memory(values[13])
mapping = {
'Site': lambda x: self.site_name,
'MachineName': lambda x: self.machine_name,
'Infrastructure': lambda x: "APEL-CREAM-SLURM",
'JobName': lambda x: x[0],
'LocalUserID': lambda x: x[2],
'LocalUserGroup': lambda x: x[3],
'WallDuration': lambda x: parse_time(x[6]),
'CpuDuration': lambda x: cput_function(x[7]),
# SLURM gives timestamps which are in system time.
'StartTime': lambda x: parse_local_timestamp(x[4]),
'StopTime': lambda x: parse_local_timestamp(x[5]),
'Queue': lambda x: x[8],
'MemoryReal': lambda x: rmem, # KB
'MemoryVirtual': lambda x: vmem, # KB
'Processors': lambda x: int(x[9]),
'NodeCount': lambda x: int(x[10])
}
rc = {}
for key in mapping:
rc[key] = mapping[key](values)
# Delete the Queue key if empty and let the Record class handle it
# (usually by inserting the string 'None').
if rc['Queue'] == '':
del rc['Queue']
# Input checking
if rc['CpuDuration'] < 0:
raise ValueError('Negative CpuDuration value')
# No negative WallDuration test as parse_time prevents that.
if rc['StopTime'] < rc['StartTime']:
raise ValueError('StopTime less than StartTime')
record = EventRecord()
record.set_all(rc)
return record