def test_parallel_run_errors():
with experiment_testing_context():
@experiment_function
def my_error_causing_test(a=1):
raise Exception('nononono')
my_error_causing_test.add_variant(a=2)
variants = my_error_causing_test.get_all_variants()
assert len(variants) == 2
run_multiple_experiments(variants,
parallel=True,
raise_exceptions=False)
with pytest.raises(Exception) as err:
run_multiple_experiments(variants,
parallel=True,
raise_exceptions=True)
print(
"^^^ Dont't worry, the above is not actually an error, we were just asserting that we caught the error."
)
assert str(err.value) == 'nononono'
def run(self, *args):
parser = argparse.ArgumentParser()
parser.add_argument(
'user_range',
action='store',
help=
'A selection of experiments to run. Examples: "3" or "3-5", or "3,4,5"'
)
parser.add_argument('-p',
'--parallel',
default=False,
action="store_true")
parser.add_argument('-np', '--n_processes', default=1, type=int)
parser.add_argument('-n', '--note')
parser.add_argument('-e',
'--raise_errors',
default=False,
action="store_true")
parser.add_argument('-d',
'--display_results',
default=False,
action="store_true")
parser.add_argument('-s',
'--slurm',
default=False,
action="store_true",
help='Run with slurm')
args = parser.parse_args(args)
ids = select_experiments(args.user_range, self.exp_record_dict)
if args.slurm:
run_multiple_experiments_with_slurm(
experiments=[load_experiment(eid) for eid in ids],
n_parallel=args.n_processes,
raise_exceptions=args.raise_errors,
run_args=self.run_args,
slurm_kwargs=self.slurm_kwargs)
else:
run_multiple_experiments(
experiments=[load_experiment(eid) for eid in ids],
parallel=args.parallel,
cpu_count=args.n_processes,
raise_exceptions=args.raise_errors,
run_args=self.run_args,
notes=(args.note, ) if args.note is not None else (),
display_results=args.display_results)
result = _warn_with_prompt(
'Finished running {} experiment{}.'.format(
len(ids), '' if len(ids) == 1 else 's'),
use_prompt=not self.close_after,
prompt='Press Enter to Continue, or "q" then Enter to Quit')
if result == 'q':
quit()
def test_run_multiple_experiments():
with experiment_testing_context():
experiments = my_api_test.get_all_variants()
assert len(experiments) == 3
records = run_multiple_experiments(experiments)
assert [record.get_result() for record in records] == [3, 4, 6]
records = run_multiple_experiments(experiments, parallel=True)
assert [record.get_result() for record in records] == [3, 4, 6]
def test_run_multiple_experiments():
with experiment_testing_context():
experiments = my_api_test.get_all_variants()
assert len(experiments)==3
records = run_multiple_experiments(experiments)
assert [record.get_result() for record in records] == [3, 4, 6]
records = run_multiple_experiments(experiments, parallel=True)
assert [record.get_result() for record in records] == [3, 4, 6]
def run(self, *args):
parser = argparse.ArgumentParser()
parser.add_argument('user_range', action='store', help='A selection of experiments to run. Examples: "3" or "3-5", or "3,4,5"')
parser.add_argument('-p', '--parallel', default=False, nargs='*')
parser.add_argument('-n', '--note')
parser.add_argument('-e', '--raise_errors', default='single', nargs='*', help='By default, error are only raised if a single experiment is run. Set "-e" to always rays errors. "-e 0" to never raise errors.')
parser.add_argument('-d', '--display_results', default=False, action = "store_true")
parser.add_argument('-s', '--slurm', default=False, action = "store_true", help='Run with slurm')
args = parser.parse_args(args)
n_processes = \
None if args.parallel is False else \
'all' if len(args.parallel)==0 else \
int(args.parallel[0]) if len(args.parallel)==1 else \
bad_value(args.parallel, '-p can have 0 or 1 arguments. Got: {}'.format(args.parallel))
ids = select_experiments(args.user_range, self.exp_record_dict)
# Raise errors if:
# -e
# -e 1
# No arg, and only 1 experiment running
raise_errors = (len(args.raise_errors)==0 or (len(args.raise_errors)==1 and args.raise_errors[0]=='1') or (args.raise_errors == 'single' and len(ids)==1))
if args.slurm:
run_multiple_experiments_with_slurm(
experiments=[load_experiment(eid) for eid in ids],
n_parallel = n_processes,
raise_exceptions = raise_errors,
run_args=self.run_args,
slurm_kwargs=self.slurm_kwargs
)
else:
exp_names = list(self.exp_record_dict.keys())
run_multiple_experiments(
experiments=[load_experiment(eid) for eid in ids],
prefixes=[exp_names.index(eid) for eid in ids],
parallel=n_processes,
raise_exceptions = raise_errors,
run_args=self.run_args,
notes=(args.note, ) if args.note is not None else (),
display_results=args.display_results
)
result = _warn_with_prompt('Finished running {} experiment{}.'.format(len(ids), '' if len(ids)==1 else 's'),
use_prompt=not self.close_after,
prompt='Press Enter to Continue, or "q" then Enter to Quit')
if result=='q':
quit()
def run(self, *args):
parser = argparse.ArgumentParser()
parser.add_argument('user_range', action='store', help='A selection of experiments to run. Examples: "3" or "3-5", or "3,4,5"')
parser.add_argument('-p', '--parallel', default=False, nargs='*')
parser.add_argument('-n', '--note')
parser.add_argument('-e', '--raise_errors', default='single', nargs='*', help='By default, error are only raised if a single experiment is run. Set "-e" to always rays errors. "-e 0" to never raise errors.')
parser.add_argument('-d', '--display_results', default=False, action = "store_true")
parser.add_argument('-s', '--slurm', default=False, action = "store_true", help='Run with slurm')
args = parser.parse_args(args)
n_processes = \
None if args.parallel is False else \
'all' if len(args.parallel)==0 else \
int(args.parallel[0]) if len(args.parallel)==1 else \
bad_value(args.parallel, '-p can have 0 or 1 arguments. Got: {}'.format(args.parallel))
ids = select_experiments(args.user_range, self.exp_record_dict)
# Raise errors if:
# -e
# -e 1
# No arg, and only 1 experiment running
raise_errors = (len(args.raise_errors)==0 or (len(args.raise_errors)==1 and args.raise_errors[0]=='1') or (args.raise_errors == 'single' and len(ids)==1))
if args.slurm:
run_multiple_experiments_with_slurm(
experiments=[load_experiment(eid) for eid in ids],
n_parallel = n_processes,
raise_exceptions = raise_errors,
run_args=self.run_args,
slurm_kwargs=self.slurm_kwargs
)
else:
exp_names = list(self.exp_record_dict.keys())
run_multiple_experiments(
experiments=[load_experiment(eid) for eid in ids],
prefixes=[exp_names.index(eid) for eid in ids],
parallel=n_processes,
raise_exceptions = raise_errors,
run_args=self.run_args,
notes=(args.note, ) if args.note is not None else (),
display_results=args.display_results
)
result = _warn_with_prompt('Finished running {} experiment{}.'.format(len(ids), '' if len(ids)==1 else 's'),
use_prompt=not self.close_after,
prompt='Press Enter to Continue, or "q" then Enter to Quit')
if result=='q':
quit()
def run(self, user_range, mode='-s', raise_exceptions=''):
assert mode in ('-s', '-e') or mode.startswith('-p')
ids = select_experiments(user_range, self.exp_record_dict)
run_multiple_experiments(
experiments=[load_experiment(eid) for eid in ids],
parallel=len(ids) > 1 and mode.startswith('-p'),
raise_exceptions=raise_exceptions == '-e',
run_args=self.run_args)
result = _warn_with_prompt(
'Finished running {} experiment{}.'.format(
len(ids), '' if len(ids) == 1 else 's'),
use_prompt=not self.close_after,
prompt='Press Enter to Continue, or "q" then Enter to Quit')
if result == 'q':
quit()
def test_parallel_run_errors():
with experiment_testing_context():
@experiment_function
def my_error_causing_test(a=1):
raise Exception('nononono')
my_error_causing_test.add_variant(a=2)
variants = my_error_causing_test.get_all_variants()
assert len(variants)==2
run_multiple_experiments(variants, parallel=True, raise_exceptions=False)
with pytest.raises(Exception) as err:
run_multiple_experiments(variants, parallel=True, raise_exceptions=True)
print("^^^ Dont't worry, the above is not actually an error, we were just asserting that we caught the error.")
assert str(err.value) == 'nononono'