def run_suite(case, config, summary):
""" Run the full suite of verification tests """
config["name"] = case
model_dir = os.path.join(livvkit.model_dir, config['data_dir'], case)
bench_dir = os.path.join(livvkit.bench_dir, config['data_dir'], case)
tabs = []
case_summary = LIVVDict()
model_cases = functions.collect_cases(model_dir)
bench_cases = functions.collect_cases(bench_dir)
for subcase in sorted(six.iterkeys(model_cases)):
bench_subcases = bench_cases[subcase] if subcase in bench_cases else []
case_sections = []
for mcase in sorted(model_cases[subcase], key=functions.sort_processor_counts):
bpath = (os.path.join(bench_dir, subcase, mcase.replace("-", os.path.sep))
if mcase in bench_subcases else "")
mpath = os.path.join(model_dir, subcase, mcase.replace("-", os.path.sep))
case_result = _analyze_case(mpath, bpath, config)
case_sections.append(elements.section(mcase, case_result))
case_summary[subcase] = _summarize_result(case_result,
case_summary[subcase])
tabs.append(elements.tab(subcase, section_list=case_sections))
result = elements.page(case, config["description"], tab_list=tabs)
summary[case] = case_summary
_print_summary(case, summary[case])
functions.create_page_from_template("verification.html",
os.path.join(livvkit.index_dir,
"verification",
case + ".html")
)
functions.write_json(result, os.path.join(livvkit.output_dir, "verification"), case+".json")
def run(name, config):
"""
Runs the analysis of the coverage of the ice sheet over the land mass.
Produces both an overall coverage percentage metric and a coverage plot.
Args:
name: The name of the test
config: A dictionary representation of the configuration file
Returns:
An elements.page with the list of elements to display
"""
greenland_data = os.path.join(livvkit.__path__[0], config['data_dir'],
config['gl_data'])
velocity_data = os.path.join(livvkit.__path__[0], config['data_dir'],
config['vel_data'])
if not (os.path.exists(greenland_data) and os.path.exists(velocity_data)):
# Add more handling here -- what do we want to return for failed tests
return elements.error("lvargo13",
"Could not find necessary data for validation!")
# Generate the script
output_dir = os.path.join(livvkit.index_dir, 'validation', 'imgs')
output_file_base = os.path.join(output_dir, 'lvargo13')
functions.mkdir_p(output_dir)
ncl_command = 'ncl \'gl_data = addfile("' + greenland_data + '", "r")\' ' \
+ '\'vel_data = addfile("' + velocity_data + '", "r")\' ' \
+ '\'model_prefix = "' \
+ os.path.join(livvkit.__path__[0], config['data_dir'], config['model_prefix']) \
+ '"\' ' \
+ '\'model_suffix = "' + config['model_suffix'] + '"\' ' \
+ '\'model_start = ' + config['model_start'] + '\' ' \
+ '\'model_end = ' + config['model_end'] + '\' ' \
+ '\'plot_file_base = "' + output_file_base + '"\' ' \
+ os.path.join(livvkit.__path__[0], config['plot_script'])
# Be cautious about running subprocesses
p = subprocess.Popen(ncl_command,
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
ncl_out, ncl_err = p.communicate()
# TODO: Put some error checking here
output_plots = [
os.path.basename(p) for p in glob.glob(output_file_base + "*.png")
]
plot_list = []
for plot in output_plots:
plot_list.append(elements.image(plot, "", plot))
the_page = elements.page("lvargo13", config['description'],
elements.gallery("Plots", plot_list))
return the_page
def run(name, config):
"""
Runs the analysis.
Args:
name: The name of the test
config: A dictionary representation of the configuration file
Returns:
The result of elements.page with the list of elements to display
"""
# TODO: Put your analysis here
element_list = elements.error("Unimplemented test",
"This test contains no analysis code!")
return elements.page(name, config['description'], element_list)
def run(name, config):
"""
Runs the analysis.
Args:
name: The name of the test
config: A dictionary representation of the configuration file
Returns:
The result of elements.page with the list of elements to display
"""
config_arg_list = []
[
config_arg_list.extend(['--' + key, str(val)])
for key, val in config.items()
]
args = parse_args(config_arg_list)
args.img_dir = os.path.join(livvkit.output_dir, 'validation', 'imgs', name)
fn.mkdir_p(args.img_dir)
details, img_gal = main(args)
tbl_data = OrderedDict(sorted(details.items()))
tbl_el = {
'Type': 'V-H Table',
'Title': 'Validation',
'TableTitle': 'Analyzed variables',
'Headers': ['h0', 'K-S test (D, p)', 'T test (t, p)'],
'Data': {
'': tbl_data
}
}
tl = [
el.tab('Table', element_list=[tbl_el]),
el.tab('Gallery', element_list=[img_gal])
]
page = el.page(name, __doc__, tab_list=tl)
page['critical'] = args.critical
return page
def run(name, config):
"""
Runs the analysis of the coverage of the ice sheet over the land mass.
Produces both an overall coverage percentage metric and a coverage plot.
Args:
name: The name of the test
config: A dictionary representation of the configuration file
Returns:
An elements.page with the list of elements to display
"""
bench_data = os.path.join(livvkit.__path__[0], config['data_dir'],
config['bench_data'])
model_data = os.path.join(livvkit.__path__[0], config['data_dir'],
config['model_data'])
if not (os.path.exists(model_data) and os.path.exists(bench_data)):
# Add more handling here -- what do we want to return for failed tests
print(
"ERROR: Could not find necessary data to run the coverage validation!"
)
print(model_data)
print(bench_data)
print("")
return elements.error(
"coverage",
"Could not find necessary data to run the coverage validation!")
# Generate the script
plot_name = "coverage.png"
output_dir = os.path.join(livvkit.index_dir, 'validation', 'imgs')
output_path = os.path.join(output_dir, plot_name)
functions.mkdir_p(output_dir)
plot_coverage(config['plot_script'], model_data, bench_data, output_path)
plot_list = [elements.image(plot_name, " ", plot_name)]
the_page = elements.page('coverage', config['description'],
elements.gallery("Plots", plot_list))
return the_page
def run_suite(case, config, summary):
""" Run the full suite of numerics tests """
m = importlib.import_module(config['module'])
m.set_up()
config["name"] = case
analysis_data = {}
bundle = livvkit.numerics_model_module
model_dir = os.path.join(livvkit.model_dir, config['data_dir'], case)
bench_dir = os.path.join(livvkit.bench_dir, config['data_dir'], case)
plot_dir = os.path.join(livvkit.output_dir, "numerics", "imgs")
config["plot_dir"] = plot_dir
functions.mkdir_p(plot_dir)
model_cases = functions.collect_cases(model_dir)
bench_cases = functions.collect_cases(bench_dir)
for mscale in sorted(model_cases):
bscale = bench_cases[mscale] if mscale in bench_cases else []
for mproc in model_cases[mscale]:
full_name = '-'.join([mscale, mproc])
bpath = (os.path.join(bench_dir, mscale, mproc.replace("-", os.path.sep))
if mproc in bscale else "")
mpath = os.path.join(model_dir, mscale, mproc.replace("-", os.path.sep))
model_data = functions.find_file(mpath, "*" + config["output_ext"])
bench_data = functions.find_file(bpath, "*" + config["output_ext"])
analysis_data[full_name] = bundle.get_plot_data(model_data,
bench_data,
m.setup[case],
config)
try:
el = m.run(config, analysis_data)
except KeyError:
el = elements.error("Numerics Plots", "Missing data")
result = elements.page(case, config['description'], element_list=el)
summary[case] = _summarize_result(m, analysis_data, config)
_print_summary(m, case, summary[case])
functions.create_page_from_template("numerics.html",
os.path.join(livvkit.index_dir, "numerics", case + ".html"))
functions.write_json(result, os.path.join(livvkit.output_dir, "numerics"), case + ".json")
def main(cl_args=None):
""" Direct execution. """
if cl_args is None and len(sys.argv) > 1:
cl_args = sys.argv[1:]
args = parse_args(cl_args)
print("--------------------------------------------------------------------")
print(" ______ __ __ __ __ ")
print(" | ____| \ \ / / \ \ / / ")
print(" | |__ \ \ / / \ \ / / ")
print(" | __| \ \/ / \ \/ / ")
print(" | |____ \ / \ / ")
print(" |______| \/ \/ ")
print(" ")
print(" Extended Verification and Validation for Earth System Models ")
print("--------------------------------------------------------------------")
print("")
print(" Current run: " + livvkit.timestamp)
print(" User: "******" OS Type: " + livvkit.os_type)
print(" Machine: " + livvkit.machine)
print(" " + livvkit.comment)
from livvkit.components import validation
from livvkit import scheduler
from livvkit.util import functions
from livvkit.util import elements
if args.extensions:
functions.setup_output(jsd=os.path.join(args.livv_resource_dir, 'js'))
summary_elements = []
validation_config = {}
print(" -----------------------------------------------------------------")
print(" Beginning extensions test suite ")
print(" -----------------------------------------------------------------")
print("")
for conf in livvkit.validation_model_configs:
validation_config = functions.merge_dicts(validation_config,
functions.read_json(conf))
summary_elements.extend(scheduler.run_quiet(validation, validation_config,
group=False))
print(" -----------------------------------------------------------------")
print(" Extensions test suite complete ")
print(" -----------------------------------------------------------------")
print("")
result = elements.page("Summary", "", element_list=summary_elements)
functions.write_json(result, livvkit.output_dir, "index.json")
print("-------------------------------------------------------------------")
print(" Done! Results can be seen in a web browser at:")
print(" " + os.path.join(livvkit.output_dir, 'index.html'))
print("-------------------------------------------------------------------")
if args.serve:
try:
# Python 3
import http.server as server
import socketserver as socket
except ImportError:
# Python 2
# noinspection PyPep8Naming
import SimpleHTTPServer as server
# noinspection PyPep8Naming
import SocketServer as socket
httpd = socket.TCPServer(('', args.serve), server.SimpleHTTPRequestHandler)
sa = httpd.socket.getsockname()
print('\nServing HTTP on {host} port {port} (http://{host}:{port}/)'.format(host=sa[0], port=sa[1]))
print('\nView the generated website by navigating to:')
print('\n http://{host}:{port}/{path}/index.html'.format(host=sa[0], port=sa[1],
path=os.path.relpath(args.out_dir)
))
print('\nExit by pressing `ctrl+c` to send a keyboard interrupt.\n')
try:
httpd.serve_forever()
except KeyboardInterrupt:
print('\nKeyboard interrupt received, exiting.\n')
sys.exit(0)
def main(cl_args=None):
""" Direct execution. """
if cl_args is None and len(sys.argv) > 1:
cl_args = sys.argv[1:]
args = options.parse_args(cl_args)
print("-------------------------------------------------------------------")
print(" __ _____ ___ ____ _ __ ")
print(" / / / _/ | / / | / / /__ (_) /_ ")
print(" / /___/ / | |/ /| |/ / '_// / __/ ")
print(" /____/___/ |___/ |___/_/\_\/_/\__/ ")
print("")
print(" Land Ice Verification & Validation ")
print("-------------------------------------------------------------------")
print("")
print(" Current run: " + livvkit.timestamp)
print(" User: "******" OS Type: " + livvkit.os_type)
print(" Machine: " + livvkit.machine)
print(" " + livvkit.comment)
from livvkit.components import numerics
from livvkit.components import verification
from livvkit.components import performance
from livvkit.components import validation
from livvkit import scheduler
from livvkit.util import functions
from livvkit.util import elements
summary_elements = []
if livvkit.verify or livvkit.validate:
functions.setup_output()
if livvkit.verify:
summary_elements.append(scheduler.run("numerics", numerics,
functions.read_json(livvkit.numerics_model_config)))
summary_elements.append(scheduler.run("verification", verification,
functions.read_json(livvkit.verification_model_config)))
summary_elements.append(scheduler.run("performance", performance,
functions.read_json(livvkit.performance_model_config)))
if livvkit.validate:
print(" -----------------------------------------------------------------")
print(" Beginning the validation test suite ")
print(" -----------------------------------------------------------------")
print("")
validation_config = {}
for conf in livvkit.validation_model_configs:
validation_config = functions.merge_dicts(validation_config,
functions.read_json(conf))
summary_elements.extend(scheduler.run_quiet(validation, validation_config,
group=False))
print(" -----------------------------------------------------------------")
print(" Validation test suite complete ")
print(" -----------------------------------------------------------------")
print("")
if livvkit.verify or livvkit.validate:
result = elements.page("Summary", "", element_list=summary_elements)
functions.write_json(result, livvkit.output_dir, "index.json")
print("-------------------------------------------------------------------")
print(" Done! Results can be seen in a web browser at:")
print(" " + os.path.join(livvkit.output_dir, 'index.html'))
print("-------------------------------------------------------------------")
if args.serve:
try:
# Python 3
import http.server as server
import socketserver as socket
except ImportError:
# Python 2
# noinspection PyPep8Naming
import SimpleHTTPServer as server
# noinspection PyPep8Naming
import SocketServer as socket
httpd = socket.TCPServer(('', args.serve), server.SimpleHTTPRequestHandler)
sa = httpd.socket.getsockname()
print('\nServing HTTP on {host} port {port} (http://{host}:{port}/)'.format(host=sa[0], port=sa[1]))
print('\nView the generated website by navigating to:')
print('\n http://{host}:{port}/{path}/index.html'.format(host=sa[0], port=sa[1],
path=os.path.relpath(livvkit.output_dir)
))
print('\nExit by pressing `ctrl+c` to send a keyboard interrupt.\n')
try:
httpd.serve_forever()
except KeyboardInterrupt:
print('\nKeyboard interrupt received, exiting.\n')
sys.exit(0)
def run_suite(case, config, summary):
""" Run the full suite of performance tests """
config["name"] = case
timing_data = dict()
model_dir = os.path.join(livvkit.model_dir, config['data_dir'], case)
bench_dir = os.path.join(livvkit.bench_dir, config['data_dir'], case)
plot_dir = os.path.join(livvkit.output_dir, "performance", "imgs")
model_cases = functions.collect_cases(model_dir)
bench_cases = functions.collect_cases(bench_dir)
functions.mkdir_p(plot_dir)
# Generate all of the timing data
for subcase in sorted(model_cases):
bench_subcases = bench_cases[subcase] if subcase in bench_cases else []
timing_data[subcase] = dict()
for mcase in model_cases[subcase]:
config["case"] = "-".join([subcase, mcase])
bpath = (os.path.join(bench_dir, subcase, mcase.replace("-", os.path.sep))
if mcase in bench_subcases else None)
mpath = os.path.join(model_dir, subcase, mcase.replace("-", os.path.sep))
timing_data[subcase][mcase] = _analyze_case(mpath, bpath, config)
# Create scaling and timing breakdown plots
weak_data = weak_scaling(timing_data, config['scaling_var'],
config['weak_scaling_points'])
strong_data = strong_scaling(timing_data, config['scaling_var'],
config['strong_scaling_points'])
timing_plots = [
generate_scaling_plot(weak_data,
"Weak scaling for " + case.capitalize(),
"runtime (s)", "",
os.path.join(plot_dir, case + "_weak_scaling.png")
),
weak_scaling_efficiency_plot(weak_data,
"Weak scaling efficiency for " + case.capitalize(),
"Parallel efficiency (% of linear)", "",
os.path.join(plot_dir, case + "_weak_scaling_efficiency.png")
),
generate_scaling_plot(strong_data,
"Strong scaling for " + case.capitalize(),
"Runtime (s)", "",
os.path.join(plot_dir, case + "_strong_scaling.png")
),
strong_scaling_efficiency_plot(strong_data,
"Strong scaling efficiency for " + case.capitalize(),
"Parallel efficiency (% of linear)", "",
os.path.join(plot_dir,
case + "_strong_scaling_efficiency.png")
),
]
timing_plots = timing_plots + \
[generate_timing_breakdown_plot(timing_data[s],
config['scaling_var'],
"Timing breakdown for " + case.capitalize()+" "+s,
"",
os.path.join(plot_dir, case+"_"+s+"_timing_breakdown.png")
)
for s in sorted(six.iterkeys(timing_data), key=functions.sort_scale)]
# Build an image gallery and write the results
el = [
elements.gallery("Performance Plots", timing_plots)
]
result = elements.page(case, config["description"], element_list=el)
summary[case] = _summarize_result(timing_data, config)
_print_result(case, summary)
functions.create_page_from_template("performance.html",
os.path.join(livvkit.index_dir, "performance",
case + ".html"))
functions.write_json(result, os.path.join(livvkit.output_dir, "performance"),
case + ".json")
def main(cl_args=None):
""" Direct execution. """
if cl_args is None and len(sys.argv) > 1:
cl_args = sys.argv[1:]
args = options.parse_args(cl_args)
print(
"-------------------------------------------------------------------")
print(" __ _____ ___ ____ _ __ ")
print(" / / / _/ | / / | / / /__ (_) /_ ")
print(" / /___/ / | |/ /| |/ / '_// / __/ ")
print(" /____/___/ |___/ |___/_/\_\/_/\__/ ")
print("")
print(" Land Ice Verification & Validation ")
print(
"-------------------------------------------------------------------")
print("")
print(" Current run: " + livvkit.timestamp)
print(" User: "******" OS Type: " + livvkit.os_type)
print(" Machine: " + livvkit.machine)
print(" " + livvkit.comment)
from livvkit.components import numerics
from livvkit.components import verification
from livvkit.components import performance
from livvkit.components import validation
from livvkit import scheduler
from livvkit.util import functions
from livvkit.util import elements
summary_elements = []
if livvkit.verify or livvkit.validate:
functions.setup_output()
if livvkit.verify:
summary_elements.append(
scheduler.run("numerics", numerics,
functions.read_json(livvkit.numerics_model_config)))
summary_elements.append(
scheduler.run(
"verification", verification,
functions.read_json(livvkit.verification_model_config)))
summary_elements.append(
scheduler.run(
"performance", performance,
functions.read_json(livvkit.performance_model_config)))
if livvkit.validate:
print(
" -----------------------------------------------------------------"
)
print(" Beginning the validation test suite ")
print(
" -----------------------------------------------------------------"
)
print("")
validation_config = {}
for conf in livvkit.validation_model_configs:
validation_config = functions.merge_dicts(
validation_config, functions.read_json(conf))
summary_elements.extend(
scheduler.run_quiet(validation, validation_config, group=False))
print(
" -----------------------------------------------------------------"
)
print(" Validation test suite complete ")
print(
" -----------------------------------------------------------------"
)
print("")
if livvkit.verify or livvkit.validate:
result = elements.page("Summary", "", element_list=summary_elements)
functions.write_json(result, livvkit.output_dir, "index.json")
print(
"-------------------------------------------------------------------"
)
print(" Done! Results can be seen in a web browser at:")
print(" " + os.path.join(livvkit.output_dir, 'index.html'))
print(
"-------------------------------------------------------------------"
)
if args.serve:
try:
# Python 3
import http.server as server
import socketserver as socket
except ImportError:
# Python 2
# noinspection PyPep8Naming
import SimpleHTTPServer as server
# noinspection PyPep8Naming
import SocketServer as socket
httpd = socket.TCPServer(('', args.serve),
server.SimpleHTTPRequestHandler)
sa = httpd.socket.getsockname()
print('\nServing HTTP on {host} port {port} (http://{host}:{port}/)'.
format(host=sa[0], port=sa[1]))
print('\nView the generated website by navigating to:')
print('\n http://{host}:{port}/{path}/index.html'.format(
host=sa[0], port=sa[1], path=os.path.relpath(livvkit.output_dir)))
print('\nExit by pressing `ctrl+c` to send a keyboard interrupt.\n')
try:
httpd.serve_forever()
except KeyboardInterrupt:
print('\nKeyboard interrupt received, exiting.\n')
sys.exit(0)