def __init__(self):
# hook to be able to have 'run' as default command
argparse.ArgumentParser.set_default_subparser = set_default_subparser
self.experiments = {}
self.arguments = self.parse_arguments()
self.database = Database(self.arguments.testbed)
self.plotter = None
def __init__(self):
# hook to be able to have 'run' as default command
argparse.ArgumentParser.set_default_subparser = set_default_subparser
self.experiments = {}
self.arguments = self.parse_arguments()
self.database = Database(self.arguments.testbed)
self.plotter = None
class Engine:
def __init__(self):
# hook to be able to have 'run' as default command
argparse.ArgumentParser.set_default_subparser = set_default_subparser
self.experiments = {}
self.arguments = self.parse_arguments()
self.database = Database(self.arguments.testbed)
self.plotter = None
def organize_experiment(self, suite, name=None):
experiment = Experiment(suite, name)
self.experiments[experiment.name] = experiment
return experiment
def go(self):
self.filter_experiments()
self.arguments.func()
def run_command(self):
self.run()
if not self.arguments.quiet:
self.report()
self.plot()
def report_command(self):
self.report()
def plot_command(self):
self.plot()
def existing_results_of(self, experiments):
return self.database.stored_results_of(experiments)
def reject_already_measured_in(self, experiments):
return self.database.reject_already_measured_in(experiments.values())
def filter_experiments(self):
if self.arguments.experiment:
name = self.arguments.experiment
self.experiments = {name: self.experiments[name]}
for experiment in self.experiments.values():
experiment.restrict(self.arguments)
if self.arguments.force:
self.work = {
experiment: experiment.instances()
for experiment in self.experiments.values()
}
else:
self.work = self.reject_already_measured_in(self.experiments)
self.done = [
x for x in self.experiments.values() if x.name not in self.work
]
def run(self):
print("experiments to run: " + str(self.work))
print("experiments skipped: " + str(self.done))
old_results = self.existing_results_of(self.done)
sorted_work = sorted(self.work, key=lambda exp: exp.name)
new_results = {
experiment: experiment.run(self.arguments)
for experiment in sorted_work
}
all_results = dict(old_results)
all_results.update(new_results)
self.database.save(new_results)
def report(self):
for name, exp in sorted_items(self.experiments):
results = self.existing_results_of([exp])
for i, measure_name in enumerate(exp.measures):
try:
exp.report(name + ' ' + measure_name, results, i)
except Exception:
print("Failed to write a report")
def plot(self):
#try:
for name, exp in sorted_items(self.experiments):
results = self.existing_results_of([exp])
for i, measure_name in enumerate(exp.measures):
exp.plot(name + ' ' + measure_name, results, i)
pass
if self.plotter is not None:
results = self.existing_results_of(self.experiments.values())
self.plotter.run_with(results, 'all', 0)
#except Exception:
# print("Failed to output plot")
def set_plotter(self, configurator, name, description):
self.plotter = Plotter(configurator, name, description)
def parse_arguments(self):
parser = argparse.ArgumentParser(
prog='Metrika',
description='A scientifically rigorous measurement tool')
parser.add_argument('-v',
'--verbose',
action='store_true',
help='show more output')
parser.add_argument('-b',
'--bench',
help='restrict to only specified benchs')
parser.add_argument('--testbed',
default=self.machine_name,
help='name for the platform where we are running')
subparsers = parser.add_subparsers(help='possible commands')
parser_run = subparsers.add_parser('run', help='measure runs')
parser_run.add_argument('-i',
'--invocations',
type=int,
default=5,
help='number of invocations')
# parser_run.add_argument('-I', '--iterations', type=int, default=5, help='number of iterations')
parser_run.add_argument('-x', '--experiment', help='experiment name')
parser_run.add_argument(
'-f',
'--force',
action='store_true',
help='force measuring again even if results are already in database'
)
parser_run.add_argument('-q',
'--quiet',
action='store_true',
help='do not show summary of results')
parser_run.add_argument(
'-s',
'--show-output',
action='store_true',
help='show output of programs being run in stdout')
parser_run.add_argument('-e',
'--hide-errors',
action='store_true',
help='hide errors of programs being run')
parser_run.add_argument('-r',
'--restrict',
help='restrict a variable to a value')
parser_run.set_defaults(func=self.run_command)
parser_report = subparsers.add_parser(
'report',
help='report measures given results database, without running\n'
' benchs again')
parser_report.add_argument('-r',
'--restrict',
help='restrict a variable to a value')
parser_report.set_defaults(func=self.report_command)
parser_plot = subparsers.add_parser(
'plot', help='plot results from database contents')
parser_plot.add_argument('-r',
'--restrict',
help='restrict a variable to a value')
parser_plot.set_defaults(func=self.plot_command)
if len(sys.argv) < 2:
parser.print_usage()
sys.exit(1)
return parser.parse_args()
@property
def machine_name(self):
return socket.gethostname()
class Engine:
def __init__(self):
# hook to be able to have 'run' as default command
argparse.ArgumentParser.set_default_subparser = set_default_subparser
self.experiments = {}
self.arguments = self.parse_arguments()
self.database = Database(self.arguments.testbed)
self.plotter = None
def organize_experiment(self, suite, name=None):
experiment = Experiment(suite, name)
self.experiments[experiment.name] = experiment
return experiment
def go(self):
self.filter_experiments()
self.arguments.func()
def run_command(self):
self.run()
if not self.arguments.quiet:
self.report()
self.plot()
def report_command(self):
self.report()
def plot_command(self):
self.plot()
def existing_results_of(self, experiments):
return self.database.stored_results_of(experiments)
def reject_already_measured_in(self, experiments):
return self.database.reject_already_measured_in(experiments.values())
def filter_experiments(self):
if self.arguments.experiment:
name = self.arguments.experiment
self.experiments = {name: self.experiments[name]}
for experiment in self.experiments.values():
experiment.restrict(self.arguments)
if self.arguments.force:
self.work = {experiment: experiment.instances() for experiment in self.experiments.values()}
else:
self.work = self.reject_already_measured_in(self.experiments)
self.done = [x for x in self.experiments.values() if x.name not in self.work]
def run(self):
print("experiments to run: " + str(self.work))
print("experiments skipped: " + str(self.done))
old_results = self.existing_results_of(self.done)
sorted_work = sorted(self.work, key=lambda exp: exp.name)
new_results = {experiment: experiment.run(self.arguments) for experiment in sorted_work}
all_results = dict(old_results)
all_results.update(new_results)
self.database.save(new_results)
def report(self):
for name, exp in sorted_items(self.experiments):
results = self.existing_results_of([exp])
for i, measure_name in enumerate(exp.measures):
try:
exp.report(name + ' ' + measure_name, results, i)
except Exception:
print("Failed to write a report")
def plot(self):
#try:
for name, exp in sorted_items(self.experiments):
results = self.existing_results_of([exp])
for i, measure_name in enumerate(exp.measures):
exp.plot(name + ' ' + measure_name, results, i)
pass
if self.plotter is not None:
results = self.existing_results_of(self.experiments.values())
self.plotter.run_with(results, 'all', 0)
#except Exception:
# print("Failed to output plot")
def set_plotter(self, configurator, name, description):
self.plotter = Plotter(configurator, name, description)
def parse_arguments(self):
parser = argparse.ArgumentParser(prog='Metrika', description='A scientifically rigorous measurement tool')
parser.add_argument('-v', '--verbose', action='store_true', help='show more output')
parser.add_argument('-b', '--bench', help='restrict to only specified benchs')
parser.add_argument('--testbed', default=self.machine_name, help='name for the platform where we are running')
subparsers = parser.add_subparsers(help='possible commands')
parser_run = subparsers.add_parser('run', help='measure runs')
parser_run.add_argument('-i', '--invocations', type=int, default=5, help='number of invocations')
# parser_run.add_argument('-I', '--iterations', type=int, default=5, help='number of iterations')
parser_run.add_argument('-x', '--experiment', help='experiment name')
parser_run.add_argument('-f', '--force', action='store_true',
help='force measuring again even if results are already in database')
parser_run.add_argument('-q', '--quiet', action='store_true', help='do not show summary of results')
parser_run.add_argument('-s', '--show-output', action='store_true',
help='show output of programs being run in stdout')
parser_run.add_argument('-e', '--hide-errors', action='store_true',
help='hide errors of programs being run')
parser_run.add_argument('-r', '--restrict', help='restrict a variable to a value')
parser_run.set_defaults(func=self.run_command)
parser_report = subparsers.add_parser('report', help='report measures given results database, without running\n'
' benchs again')
parser_report.add_argument('-r', '--restrict', help='restrict a variable to a value')
parser_report.set_defaults(func=self.report_command)
parser_plot = subparsers.add_parser('plot', help='plot results from database contents')
parser_plot.add_argument('-r', '--restrict', help='restrict a variable to a value')
parser_plot.set_defaults(func=self.plot_command)
if len(sys.argv) < 2:
parser.print_usage()
sys.exit(1)
return parser.parse_args()
@property
def machine_name(self):
return socket.gethostname()