def getBenchmarkDictionary(self, workdir):
""" Return the benchmarks dictionary """
if self.__benchmarks:
return self.__benchmarks
else:
return getJSONDictionary(self.getBenchmarkFileName(workdir))
def executeBenchmark(self):
""" Interface method for benchmark test """
# Use this method to interface with benchmark code
# The method should return a dictionary containing the results of the test
dictionary = {}
cmd = "/afs/cern.ch/user/w/walkerr/public/kflops.sh"
timeout = 120
tolog("Executing benchmark test: %s" % (cmd))
exitcode, output = timedCommand(cmd, timeout=timeout)
if exitcode != 0:
tolog("!!WARNING!!3434!! Encountered a problem with benchmark test: %s" % (output))
else:
tolog("Benchmark finished")
filename = "kflops.json"
if not os.path.exists(filename):
tolog("!!WARNING!!3435!! Benchmark did not produce expected output file: %s" % (filename))
else:
tolog("Parsing benchmark output file: %s" % (filename))
dictionary = getJSONDictionary(filename)
if dictionary == {}:
tolog("!!WARNING!!3436!! Empty benchmark dictionary - nothing to report")
return dictionary
def executeBenchmark(self):
""" Interface method for benchmark test """
# Use this method to interface with benchmark code
# The method should return a dictionary containing the results of the test
dictionary = {}
cmd = "export BMK_ROOTDIR='/cvmfs/atlas.cern.ch/repo/benchmarks/cern/current;"
cmd += "$BMK_ROOTDIR/cern-benchmark --benchmarks=\"fastBmk\" --queue_host=dashb-test-mb.cern.ch --queue_port=61113 --username=vmspecprod --topic=/topic/vm.spec --cloud=cloud-name --vo=ATLAS"
timeout = 120
tolog("Executing benchmark test: %s" % (cmd))
exitcode, output = timedCommand(cmd, timeout=timeout)
if exitcode != 0:
tolog("!!WARNING!!3434!! Encountered a problem with benchmark test: %s" % (output))
else:
tolog("Benchmark finished")
filename = "kflops.json"
if not os.path.exists(filename):
tolog("!!WARNING!!3435!! Benchmark did not produce expected output file: %s" % (filename))
else:
tolog("Parsing benchmark output file: %s" % (filename))
dictionary = getJSONDictionary(filename)
if dictionary == {}:
tolog("!!WARNING!!3436!! Empty benchmark dictionary - nothing to report")
return dictionary
def executeBenchmark(self):
""" Interface method for benchmark test """
# Use this method to interface with benchmark code
# The method should return a dictionary containing the results of the test
dictionary = {}
cmd = "/afs/cern.ch/user/w/walkerr/public/kflops.sh"
timeout = 120
tolog("Executing benchmark test: %s" % (cmd))
exitcode, output = timedCommand(cmd, timeout=timeout)
if exitcode != 0:
tolog(
"!!WARNING!!3434!! Encountered a problem with benchmark test: %s"
% (output))
else:
tolog("Benchmark finished")
filename = "kflops.json"
if not os.path.exists(filename):
tolog(
"!!WARNING!!3435!! Benchmark did not produce expected output file: %s"
% (filename))
else:
tolog("Parsing benchmark output file: %s" % (filename))
dictionary = getJSONDictionary(filename)
if dictionary == {}:
tolog(
"!!WARNING!!3436!! Empty benchmark dictionary - nothing to report"
)
return dictionary
def getBenchmarkDictionary(self, workdir):
""" Return the benchmarks dictionary """
if self.__benchmarks:
return self.__benchmarks
else:
return getJSONDictionary(self.getBenchmarkFileName(workdir))
def getUtilityInfo(self, node, experiment, workdir):
""" Add the utility info to the node structure if available """
# Get the experiment object and check if the special utility (e.g. a memory monitor) was used
thisExperiment = getExperiment(experiment)
if thisExperiment.shouldExecuteUtility():
# Try to get the memory monitor info from the workdir first
path = os.path.join(workdir, thisExperiment.getUtilityJSONFilename())
init_path = os.path.join(self.__pilot_initdir, thisExperiment.getUtilityJSONFilename())
if not os.path.exists(path):
tolog("File does not exist: %s" % (path))
if os.path.exists(init_path):
path = init_path
else:
tolog("File does not exist either: %s" % (path))
path = ""
if path != "":
tolog("Reading memory monitoring info from: %s" % (path))
# Get the dictionary
d = getJSONDictionary(path)
if d and d != {}:
try:
node['maxRSS'] = d['Max']['maxRSS']
node['maxVMEM'] = d['Max']['maxVMEM']
node['maxSWAP'] = d['Max']['maxSwap']
node['maxPSS'] = d['Max']['maxPSS']
node['avgRSS'] = d['Avg']['avgRSS']
node['avgVMEM'] = d['Avg']['avgVMEM']
node['avgSWAP'] = d['Avg']['avgSwap']
node['avgPSS'] = d['Avg']['avgPSS']
except Exception, e:
tolog("!!WARNING!!54541! Exception caught while parsing memory monitor JSON: %s" % (e))
else:
tolog("Extracted info from memory monitor JSON")
# Done with the memory monitor for this job (if the file is read from the pilots' init dir), remove the file in case there are other jobs to be run
if os.path.exists(init_path):
try:
os.system("rm -rf %s" % (init_path))
except Exception, e:
tolog("!!WARNING!!4343!! Failed to remove %s: %s" % (init_path), e)
else:
tolog("Removed %s" % (init_path))
def getUtilityInfo(self, node, experiment, workdir):
""" Add the utility info to the node structure if available """
# Get the experiment object and check if the special utility (e.g. a memory monitor) was used
thisExperiment = getExperiment(experiment)
if thisExperiment.shouldExecuteUtility():
# Try to get the memory monitor info from the workdir first
path = os.path.join(workdir, thisExperiment.getUtilityJSONFilename())
init_path = os.path.join(self.__pilot_initdir, thisExperiment.getUtilityJSONFilename())
primary_location = False
if not os.path.exists(path):
tolog("File does not exist: %s" % (path))
if os.path.exists(init_path):
path = init_path
else:
tolog("File does not exist either: %s" % (path))
path = ""
primary_location = False
else:
primary_location = True
if path != "":
tolog("Reading memory monitoring info from: %s" % (path))
# If the file is the primary one (ie the one in the workdir and not the initdir, then also check the modification time)
read_from_file = True
if primary_location:
# Get the modification time
mod_time = None
max_time = 120
try:
file_modification_time = os.path.getmtime(path)
current_time = int(time())
mod_time = current_time - file_modification_time
tolog("File %s was modified %d seconds ago" % (mod_time))
except:
tolog("!!WARNING!!2323!! Could not read the modification time of %s" % (path))
tolog("!!WARNING!!2324!! Will add -1 values for the memory info")
node['maxRSS'] = -1
node['maxVMEM'] = -1
node['maxSWAP'] = -1
node['maxPSS'] = -1
node['avgRSS'] = -1
node['avgVMEM'] = -1
node['avgSWAP'] = -1
node['avgPSS'] = -1
read_from_file = False
else:
if mod_time > max_time:
tolog("!!WARNING!!2325!! File %s was modified over %d s ago, will add -1 values for the memory info" % (path, max_time))
node['maxRSS'] = -1
node['maxVMEM'] = -1
node['maxSWAP'] = -1
node['maxPSS'] = -1
node['avgRSS'] = -1
node['avgVMEM'] = -1
node['avgSWAP'] = -1
node['avgPSS'] = -1
read_from_file = False
if read_from_file:
# Get the dictionary
d = getJSONDictionary(path)
if d and d != {}:
try:
# Move to experiment class?
node['maxRSS'] = d['Max']['maxRSS']
node['maxVMEM'] = d['Max']['maxVMEM']
node['maxSWAP'] = d['Max']['maxSwap']
node['maxPSS'] = d['Max']['maxPSS']
node['avgRSS'] = d['Avg']['avgRSS']
node['avgVMEM'] = d['Avg']['avgVMEM']
node['avgSWAP'] = d['Avg']['avgSwap']
node['avgPSS'] = d['Avg']['avgPSS']
except Exception, e:
tolog("!!WARNING!!54541! Exception caught while parsing memory monitor JSON: %s" % (e))
else:
tolog("Extracted info from memory monitor JSON")
# Done with the memory monitor for this job (if the file is read from the pilots' init dir), remove the file in case there are other jobs to be run
if os.path.exists(init_path):
try:
os.system("rm -rf %s" % (init_path))
except Exception, e:
tolog("!!WARNING!!4343!! Failed to remove %s: %s" % (init_path), e)
else:
tolog("Removed %s" % (init_path))
