def process_appker_output(appstdout=None,
stdout=None,
stderr=None,
geninfo=None,
resource_appker_vars=None):
# set App Kernel Description
parser = AppKerOutputParser(
name='wrf',
version=1,
description="Weather Research and Forecasting Model",
url='http://www.wrf-model.org',
measurement_name='WRF')
# set obligatory parameters and statistics
# set common parameters and statistics
parser.add_common_must_have_params_and_stats()
# set app kernel custom sets
parser.add_must_have_parameter('App:Version')
parser.add_must_have_parameter('Input:Grid Resolution')
parser.add_must_have_parameter('Input:Simulation Length')
parser.add_must_have_parameter('Input:Simulation Start Date')
parser.add_must_have_parameter('Input:Timestep')
parser.add_must_have_parameter('RunEnv:Nodes')
parser.add_must_have_parameter('WRF Dynamical Solver')
# parser.add_must_have_statistic('Average Floating-Point Performance')
parser.add_must_have_statistic('Average Simulation Speed')
parser.add_must_have_statistic('Mean Time To Simulate One Timestep')
parser.add_must_have_statistic('Output Data Size')
# parser.add_must_have_statistic('Peak Floating-Point Performance')
parser.add_must_have_statistic('Peak Simulation Speed')
parser.add_must_have_statistic('Time Spent on I/O')
parser.add_must_have_statistic('Wall Clock Time')
# parse common parameters and statistics
parser.parse_common_params_and_stats(appstdout, stdout, stderr, geninfo,
resource_appker_vars)
# read output
lines = []
if os.path.isfile(appstdout):
fin = open(appstdout, "rt")
lines = fin.readlines()
fin.close()
# process the output
io_size = None
wall_clock_time = None
iteration_wall_clock_time = []
sim_time_per_iteration = None
dx = None
dy = None
flops_conversion = None
j = 0
while j < len(lines):
m = re.search(r'XDMOD\*\*\*SIZE OF CURRENT DIR BEFORE WRF RUN\s*(\d+)',
lines[j])
if m:
io_size = int(m.group(1).strip())
m = re.search(r'XDMOD\*\*\*SIZE OF CURRENT DIR AFTER WRF RUN\s*(\d+)',
lines[j])
if m and io_size:
parser.set_statistic("Output Data Size",
(int(m.group(1).strip()) - io_size) / 1024.0 /
1024.0, "MByte")
m = re.search(r'XDMOD\*\*\*WRF RUN BEGINS HERE --(.+)', lines[j])
if m:
wall_clock_time = parser.get_datetime_local(m.group(1).strip())
m = re.search(r'XDMOD\*\*\*WRF RUN HAS FINISHED --(.+)', lines[j])
if m and wall_clock_time:
wall_clock_time = parser.get_datetime_local(
m.group(1).strip()) - wall_clock_time
parser.set_statistic("Wall Clock Time",
wall_clock_time.total_seconds(), "Second")
if lines[j].find('XDMOD***RESULT OF rsl.out.0000 BEGINS') >= 0:
# the output from MPI rank #0
io_time = None
while j < len(lines):
if lines[j].find('XDMOD***RESULT OF rsl.out.0000 ENDS') >= 0:
break
m = re.search(
r'Timing for processing restart file.+?:\s+(\d\S+)',
lines[j], re.I)
if m:
if io_time is None:
io_time = 0.0
io_time += float(m.group(1).strip())
m = re.search(r'Timing for Writing.+?:\s+(\d\S+)', lines[j],
re.I)
if m:
if io_time is None:
io_time = 0.0
io_time += float(m.group(1).strip())
m = re.search(
r'Timing for main: time.+?on domain.+?:\s+(\d\S+)',
lines[j], re.I)
if m:
iteration_wall_clock_time.append(float(m.group(1).strip()))
m = re.search(r'WRF NUMBER OF TILES.+?(\d+)', lines[j])
if m:
omp_threads = int(m.group(1).strip())
if omp_threads > 1:
parser.set_parameter("Number of OpenMP Threads",
omp_threads)
m = re.match(r'^\s+WRF V(\S+) MODEL', lines[j])
if m:
parser.set_parameter("App:Version", m.group(1).strip())
j += 1
parser.set_statistic("Time Spent on I/O", io_time, "Second")
if re.search('XDMOD\*\*\*RESULT OF wrfout.+?BEGINS',
lines[j]) is not None:
# the output file's header (netCDF dump)
io_time = None
while j < len(lines):
if re.search('XDMOD\*\*\*RESULT OF wrfout.+?ENDS',
lines[j]) is not None:
break
m = re.search(r':DX = (\d+)', lines[j], re.I)
if m:
dx = float(m.group(1).strip()) * 0.001 # in meters
m = re.search(r':DY = (\d+)', lines[j], re.I)
if m:
dy = float(m.group(1).strip()) * 0.001 # in meters
m = re.search(r':DT = (\d+)', lines[j], re.I)
if m:
sim_time_per_iteration = float(
m.group(1).strip()) # in seconds
parser.set_parameter("Input:Timestep",
sim_time_per_iteration,
"Second per Step")
m = re.search(r':SIMULATION_START_DATE = "(.+?)"', lines[j],
re.I)
if m:
parser.set_parameter("Input:Simulation Start Date",
(m.group(1).strip()))
m = re.search(r':GRIDTYPE = "(.+?)"', lines[j], re.I)
if m:
solver = m.group(1).strip()
if solver == 'C':
solver = 'Advanced Research WRF (ARW)'
if solver == 'E':
solver = 'Nonhydrostatic Mesoscale Model (NMM)'
parser.set_parameter("WRF Dynamical Solver", solver)
m = re.search(r'Timing for Writing.+?:\s+(\d\S+)', lines[j],
re.I)
if m:
if io_time is None:
io_time = 0.0
io_time += float(m.group(1).strip())
m = re.search(
r'Timing for main: time.+?on domain.+?:\s+(\d\S+)',
lines[j], re.I)
if m:
iteration_wall_clock_time.append(float(m.group(1).strip()))
m = re.search(r'WRF NUMBER OF TILES.+?(\d+)', lines[j])
if m:
omp_threads = int(m.group(1).strip())
if omp_threads > 1:
parser.set_parameter("Number of OpenMP Threads",
omp_threads)
m = re.match(r'^\s+WRF V(\S+) MODEL', lines[j])
if m:
parser.set_parameter("App:Version", m.group(1).strip())
j += 1
if dx and dy:
if (dx - int(dx)) * 1000 < 0.1 and (
dy - int(dy)
) * 1000 < 0.1: # back compatibility with output format
parser.set_parameter("Input:Grid Resolution",
"%.0f x %.0f" % (dx, dy), "km^2")
else:
parser.set_parameter("Input:Grid Resolution",
str(dx) + " x " + str(dy), "km^2")
m = re.search(r'XDMOD\*\*\*FLOATING-POINT PERFORMANCE CONVERSION',
lines[j])
if m:
flops_conversion = lines[j + 1].strip()
j += 1
if wall_clock_time:
parser.successfulRun = True
else:
parser.successfulRun = False
if len(iteration_wall_clock_time) > 0 and sim_time_per_iteration:
parser.set_parameter("Input:Simulation Length",
(len(iteration_wall_clock_time)) *
sim_time_per_iteration / 3600.0, "Hour")
iteration_wall_clock_time = sorted(iteration_wall_clock_time)
iteration_wall_clock_time.pop()
t = 0.0
min_t = iteration_wall_clock_time[0]
for tt in iteration_wall_clock_time:
t += tt
t = t / len(iteration_wall_clock_time)
parser.set_statistic("Mean Time To Simulate One Timestep", t, "Second")
parser.set_statistic("Average Simulation Speed",
sim_time_per_iteration / t,
"Simulated Second per Second")
parser.set_statistic("Peak Simulation Speed",
sim_time_per_iteration / min_t,
"Simulated Second per Second")
if flops_conversion:
flops_conversion = flops_conversion.replace("$", "").replace(
"gflops=", "")
gflops = eval(flops_conversion, {'T': t})
parser.set_statistic("Average Floating-Point Performance",
1000.0 * gflops, "MFLOP per Second")
gflops = eval(flops_conversion, {'T': min_t})
parser.set_statistic("Peak Floating-Point Performance",
1000.0 * gflops, "MFLOP per Second")
if __name__ == "__main__":
# output for testing purpose
parsing_complete = parser.parsing_complete(True)
print("parsing complete:", parsing_complete)
if hasattr(parser, 'successfulRun'):
print("successfulRun", parser.successfulRun)
parser.print_params_stats_as_must_have()
print(parser.get_xml())
# return complete XML overwize return None
return parser.get_xml()
def process_appker_output(appstdout=None,
stdout=None,
stderr=None,
geninfo=None,
proclog=None,
resource_appker_vars=None):
# set App Kernel Description
parser = AppKerOutputParser(
name='gamess',
version=1,
description=
"Gamess: General Atomic and Molecular Electronic Structure System",
url='http://www.msg.ameslab.gov',
measurement_name='Gamess')
# set obligatory parameters and statistics
# set common parameters and statistics
parser.add_common_must_have_params_and_stats()
# set app kernel custom sets
parser.add_must_have_parameter('App:Version')
parser.add_must_have_statistic('Wall Clock Time')
parser.add_must_have_statistic('User Time')
parser.add_must_have_statistic('Time Spent in MP2 Energy Calculation')
parser.add_must_have_statistic(
'Time Spent in Restricted Hartree-Fock Calculation')
# parse common parameters and statistics
parser.parse_common_params_and_stats(appstdout, stdout, stderr, geninfo,
resource_appker_vars)
# read output
lines = []
if os.path.isfile(appstdout):
fin = open(appstdout, "rt")
lines = fin.readlines()
fin.close()
# process the output
start_time = None
end_time = None
mp2_energy_calculation_time = 0.0
rhf_calculation_time = 0.0
efficiency = None
j = 0
while j < len(lines):
m = re.search(r'GAMESS VERSION = ([^*]+)', lines[j])
if m:
parser.set_parameter("App:Version", m.group(1).strip())
m = re.search(
r'PARALLEL VERSION RUNNING ON\s*([\d.]+) PROCESSORS IN\s*([\d.]+) NODE',
lines[j])
if m:
parser.set_parameter("App:NCores", m.group(1).strip())
parser.set_parameter("App:NNodes", m.group(2).strip())
m = re.search(r'EXECUTION OF GAMESS BEGUN (.+)', lines[j])
if m:
start_time = parser.get_datetime_local(m.group(1).strip())
m = re.search(r'EXECUTION OF GAMESS TERMINATED NORMALLY (.+)',
lines[j])
if m:
end_time = parser.get_datetime_local(m.group(1).strip())
if re.search(r'DONE WITH MP2 ENERGY', lines[j]):
j += 1
m = re.search(r'STEP CPU TIME=\s*([\d.]+)', lines[j])
if m:
mp2_energy_calculation_time += float(m.group(1).strip())
if re.search(r'END OF RHF CALCULATION', lines[j]):
j += 1
m = re.search(r'STEP CPU TIME=\s*([\d.]+)', lines[j])
if m:
rhf_calculation_time += float(m.group(1).strip())
m = re.search(r'TOTAL WALL CLOCK TIME.+CPU UTILIZATION IS\s+([\d.]+)',
lines[j])
if m:
efficiency = float(m.group(1).strip())
j += 1
if start_time and end_time:
walltime = total_seconds(end_time - start_time)
if walltime >= 0.0:
parser.set_statistic('Wall Clock Time', str(walltime), "Second")
if efficiency:
parser.set_statistic("User Time",
str((0.01 * efficiency * walltime)),
"Second")
parser.set_statistic("Time Spent in MP2 Energy Calculation",
str(mp2_energy_calculation_time), "Second")
parser.set_statistic("Time Spent in Restricted Hartree-Fock Calculation",
str(rhf_calculation_time), "Second")
if "attemptsToLaunch" in parser.geninfo:
parser.set_statistic("Attempts to Launch",
parser.geninfo['attemptsToLaunch'])
else:
parser.set_statistic("Attempts to Launch", 1)
if __name__ == "__main__":
# output for testing purpose
print(("parsing complete:", parser.parsing_complete()))
parser.print_params_stats_as_must_have()
print((parser.get_xml()))
# return complete XML overwize return None
return parser.get_xml()