def test_all_cutest_problem(self):
"""
Runs benchmark on all CUTEst problems on about half of the available Mantid minimizers
and with using weights.
"""
minimizers = [
'Levenberg-Marquardt', 'Levenberg-MarquardtMD', 'Simplex',
'SteepestDescent', 'Trust Region'
]
group_suffix_names = ['cutest']
use_errors = True
cutest_group_dir = os.path.join(self.base_problem_files_dir, 'CUTEst')
# test that fit executes on all NIST problems against a number of different minimizers
problems, results_per_group = fitbk.do_fitting_benchmark(
cutest_group_dir=cutest_group_dir,
minimizers=minimizers,
use_errors=use_errors)
# test that can print individual group table
for idx, group_results in enumerate(results_per_group):
fitout.print_group_results_tables(
minimizers,
group_results,
problems[idx],
group_name=group_suffix_names[idx],
use_errors=use_errors,
simple_text=True,
rst=True,
save_to_file=False,
color_scale=self.color_scale)
def test_all_neutron_data_problem(self):
"""
Runs benchmark on all neutron data problems on one minimizers
and using weights.
"""
minimizers = ['Trust Region']
group_suffix_names = ['neutron_data']
use_errors = True
neutron_data_group_dirs = [
os.path.join(self.base_problem_files_dir, 'Neutron_data')
]
# test that fit executes on all Neutron data problems
problems, results_per_group = fitbk.do_fitting_benchmark(
neutron_data_group_dirs=neutron_data_group_dirs,
minimizers=minimizers,
use_errors=use_errors)
# test that can print individual group table
for idx, group_results in enumerate(results_per_group):
fitout.print_group_results_tables(
minimizers,
group_results,
problems[idx],
group_name=group_suffix_names[idx],
use_errors=use_errors,
simple_text=True,
rst=True,
save_to_file=False,
color_scale=self.color_scale)
def run_all_with_or_without_errors(base_problem_files_dir, use_errors, minimizers,
group_names, group_suffix_names, color_scale, save_to_file=False):
"""
Run all benchmark problems available, with/without using weights in the cost
function. This is just a convenience function meant to be used by system/unit tests, or other scripts
ALL means: NIST + CUTEST + any fitting problems against observed Neutron data (+ any other
which may be added in the future)
At this point in time it is assumed that the problem files are in store relative to
a base_problem_files_dir as follows:
CUTEst/
NIST_nonlinear_regression/
Neutron_data/
Be warned this function can be verbose.
@param base_problem_files_dir :: base directory path of all the benchmark problems
@param use_errors :: whether to use observational errors as weights in the cost function
@param minimizers :: list of minimizers to test
@param group_names :: names for display purposes
@param group_suffix_names :: group names to use as suffixes, for example in file names
@param color_scale :: list with pairs of threshold value - color, to produce color
@param save_to_file :: whether to save the table outputs to files following specific naming conventions
"""
# Assume the benchmark problems are stores as follows
nist_group_dir = os.path.join(base_problem_files_dir, 'NIST_nonlinear_regression')
cutest_group_dir = os.path.join(base_problem_files_dir, 'CUTEst')
neutron_data_group_dirs = [os.path.join(base_problem_files_dir, 'Neutron_data')]
problems, results_per_group = do_fitting_benchmark(nist_group_dir=nist_group_dir,
cutest_group_dir=cutest_group_dir,
neutron_data_group_dirs=neutron_data_group_dirs,
minimizers=minimizers, use_errors=use_errors)
# Results for every test problem in each group
for idx, group_results in enumerate(results_per_group):
print("\n\n")
print("********************************************************")
print("**************** RESULTS FOR GROUP {0}, {1} ************".format(idx+1,
group_names[idx]))
print("********************************************************")
fitout.print_group_results_tables(minimizers, group_results, problems[idx],
group_name=group_suffix_names[idx],
use_errors=use_errors,
simple_text=True, rst=True, save_to_file=save_to_file,
color_scale=color_scale)
# Results aggregated (median) by group (NIST, Neutron data, CUTEst, etc.)
header = '\n\n**************** OVERALL SUMMARY - ALL GROUPS ******** \n\n'
print(header)
fitout.print_overall_results_table(minimizers, results_per_group, problems, group_names,
use_errors=use_errors, save_to_file=save_to_file)
# Flush to reduce mix-up with system tests/runner output
import sys
sys.stdout.flush()
def test_all_nist_problem(self):
"""
Runs benchmark on all NIST problems on about half of the available Mantid minimizers
and without using weights.
"""
minimizers = [
'BFGS', 'Conjugate gradient (Fletcher-Reeves imp.)',
'Conjugate gradient (Polak-Ribiere imp.)', 'Damped GaussNewton'
]
group_names = [
'NIST, "lower" difficulty', 'NIST, "average" difficulty',
'NIST, "higher" difficulty'
]
group_suffix_names = ['nist_lower', 'nist_average', 'nist_higher']
use_errors = False
nist_group_dir = os.path.join(self.base_problem_files_dir,
'NIST_nonlinear_regression')
# test that fit executes on all NIST problems against a number of different minimizers
problems, results_per_group = fitbk.do_fitting_benchmark(
nist_group_dir=nist_group_dir,
minimizers=minimizers,
use_errors=use_errors)
# test that can print individual group tables
for idx, group_results in enumerate(results_per_group):
print("\n\n")
print("********************************************************")
print("**************** RESULTS FOR GROUP {0}, {1} ************".
format(idx + 1, group_names[idx]))
print("********************************************************")
fitout.print_group_results_tables(
minimizers,
group_results,
problems[idx],
group_name=group_suffix_names[idx],
use_errors=use_errors,
simple_text=True,
rst=True,
save_to_file=False,
color_scale=self.color_scale)
# test that can print summary tables
header = '\n\n**************** OVERALL SUMMARY - ALL GROUPS ******** \n\n'
print(header)
fitout.print_overall_results_table(minimizers,
results_per_group,
problems,
group_names,
use_errors=use_errors,
save_to_file=False)
def test_all_neutron_data_problem(self):
"""
Runs benchmark on all neutron data problems on one minimizers
and using weights.
"""
minimizers = ['Trust Region']
group_suffix_names = ['neutron_data']
use_errors = True
neutron_data_group_dirs = [os.path.join(self.base_problem_files_dir, 'Neutron_data')]
# test that fit executes on all Neutron data problems
problems, results_per_group = fitbk.do_fitting_benchmark(neutron_data_group_dirs=neutron_data_group_dirs,
minimizers=minimizers, use_errors=use_errors)
# test that can print individual group table
for idx, group_results in enumerate(results_per_group):
fitout.print_group_results_tables(minimizers, group_results, problems[idx],
group_name=group_suffix_names[idx], use_errors=use_errors,
simple_text=True, rst=True, save_to_file=False,
color_scale=self.color_scale)
def test_all_cutest_problem(self):
"""
Runs benchmark on all CUTEst problems on about half of the available Mantid minimizers
and with using weights.
"""
minimizers = ['Levenberg-Marquardt', 'Levenberg-MarquardtMD',
'Simplex', 'SteepestDescent', 'Trust Region']
group_suffix_names = ['cutest']
use_errors = True
cutest_group_dir = os.path.join(self.base_problem_files_dir, 'CUTEst')
# test that fit executes on all NIST problems against a number of different minimizers
problems, results_per_group = fitbk.do_fitting_benchmark(cutest_group_dir=cutest_group_dir,
minimizers=minimizers, use_errors=use_errors)
# test that can print individual group table
for idx, group_results in enumerate(results_per_group):
fitout.print_group_results_tables(minimizers, group_results, problems[idx],
group_name=group_suffix_names[idx], use_errors=use_errors,
simple_text=True, rst=True, save_to_file=False,
color_scale=self.color_scale)
def test_all_nist_problem(self):
"""
Runs benchmark on all NIST problems on about half of the available Mantid minimizers
and without using weights.
"""
minimizers = ['BFGS', 'Conjugate gradient (Fletcher-Reeves imp.)',
'Conjugate gradient (Polak-Ribiere imp.)', 'Damped GaussNewton']
group_names = ['NIST, "lower" difficulty', 'NIST, "average" difficulty', 'NIST, "higher" difficulty']
group_suffix_names = ['nist_lower', 'nist_average', 'nist_higher']
use_errors = False
nist_group_dir = os.path.join(self.base_problem_files_dir, 'NIST_nonlinear_regression')
# test that fit executes on all NIST problems against a number of different minimizers
problems, results_per_group = fitbk.do_fitting_benchmark(nist_group_dir=nist_group_dir,
minimizers=minimizers, use_errors=use_errors)
# test that can print individual group tables
for idx, group_results in enumerate(results_per_group):
print("\n\n")
print("********************************************************")
print("**************** RESULTS FOR GROUP {0}, {1} ************".format(idx+1,
group_names[idx]))
print("********************************************************")
fitout.print_group_results_tables(minimizers, group_results, problems[idx],
group_name=group_suffix_names[idx], use_errors=use_errors,
simple_text=True, rst=True, save_to_file=False,
color_scale=self.color_scale)
# test that can print summary tables
header = '\n\n**************** OVERALL SUMMARY - ALL GROUPS ******** \n\n'
print(header)
fitout.print_overall_results_table(minimizers, results_per_group, problems, group_names,
use_errors=use_errors, save_to_file=False)
def run_all_with_or_without_errors(base_problem_files_dir,
use_errors,
minimizers,
group_names,
group_suffix_names,
color_scale,
save_to_file=False):
"""
Run all benchmark problems available, with/without using weights in the cost
function. This is just a convenience function meant to be used by system/unit tests, or other scripts
ALL means: NIST + CUTEST + any fitting problems against observed Neutron data (+ any other
which may be added in the future)
At this point in time it is assumed that the problem files are in store relative to
a base_problem_files_dir as follows:
CUTEst/
NIST_nonlinear_regression/
Neutron_data/
Be warned this function can be verbose.
@param base_problem_files_dir :: base directory path of all the benchmark problems
@param use_errors :: whether to use observational errors as weights in the cost function
@param minimizers :: list of minimizers to test
@param group_names :: names for display purposes
@param group_suffix_names :: group names to use as suffixes, for example in file names
@param color_scale :: list with pairs of threshold value - color, to produce color
@param save_to_file :: whether to save the table outputs to files following specific naming conventions
"""
# Assume the benchmark problems are stores as follows
nist_group_dir = os.path.join(base_problem_files_dir,
'NIST_nonlinear_regression')
cutest_group_dir = os.path.join(base_problem_files_dir, 'CUTEst')
neutron_data_group_dirs = [
os.path.join(base_problem_files_dir, 'Neutron_data')
]
problems, results_per_group = do_fitting_benchmark(
nist_group_dir=nist_group_dir,
cutest_group_dir=cutest_group_dir,
neutron_data_group_dirs=neutron_data_group_dirs,
minimizers=minimizers,
use_errors=use_errors)
# Results for every test problem in each group
for idx, group_results in enumerate(results_per_group):
print("\n\n")
print("********************************************************")
print(
"**************** RESULTS FOR GROUP {0}, {1} ************".format(
idx + 1, group_names[idx]))
print("********************************************************")
fitout.print_group_results_tables(minimizers,
group_results,
problems[idx],
group_name=group_suffix_names[idx],
use_errors=use_errors,
simple_text=True,
rst=True,
save_to_file=save_to_file,
color_scale=color_scale)
# Results aggregated (median) by group (NIST, Neutron data, CUTEst, etc.)
header = '\n\n**************** OVERALL SUMMARY - ALL GROUPS ******** \n\n'
print(header)
fitout.print_overall_results_table(minimizers,
results_per_group,
problems,
group_names,
use_errors=use_errors,
save_to_file=save_to_file)
# Flush to reduce mix-up with system tests/runner output
import sys
sys.stdout.flush()