def test_parallel_run_errors(): with experiment_testing_context(): @experiment_function def my_error_causing_test(a=1): raise Exception('nononono') my_error_causing_test.add_variant(a=2) variants = my_error_causing_test.get_all_variants() assert len(variants) == 2 run_multiple_experiments(variants, parallel=True, raise_exceptions=False) with pytest.raises(Exception) as err: run_multiple_experiments(variants, parallel=True, raise_exceptions=True) print( "^^^ Dont't worry, the above is not actually an error, we were just asserting that we caught the error." ) assert str(err.value) == 'nononono'
def run(self, *args): parser = argparse.ArgumentParser() parser.add_argument( 'user_range', action='store', help= 'A selection of experiments to run. Examples: "3" or "3-5", or "3,4,5"' ) parser.add_argument('-p', '--parallel', default=False, action="store_true") parser.add_argument('-np', '--n_processes', default=1, type=int) parser.add_argument('-n', '--note') parser.add_argument('-e', '--raise_errors', default=False, action="store_true") parser.add_argument('-d', '--display_results', default=False, action="store_true") parser.add_argument('-s', '--slurm', default=False, action="store_true", help='Run with slurm') args = parser.parse_args(args) ids = select_experiments(args.user_range, self.exp_record_dict) if args.slurm: run_multiple_experiments_with_slurm( experiments=[load_experiment(eid) for eid in ids], n_parallel=args.n_processes, raise_exceptions=args.raise_errors, run_args=self.run_args, slurm_kwargs=self.slurm_kwargs) else: run_multiple_experiments( experiments=[load_experiment(eid) for eid in ids], parallel=args.parallel, cpu_count=args.n_processes, raise_exceptions=args.raise_errors, run_args=self.run_args, notes=(args.note, ) if args.note is not None else (), display_results=args.display_results) result = _warn_with_prompt( 'Finished running {} experiment{}.'.format( len(ids), '' if len(ids) == 1 else 's'), use_prompt=not self.close_after, prompt='Press Enter to Continue, or "q" then Enter to Quit') if result == 'q': quit()
def test_run_multiple_experiments(): with experiment_testing_context(): experiments = my_api_test.get_all_variants() assert len(experiments) == 3 records = run_multiple_experiments(experiments) assert [record.get_result() for record in records] == [3, 4, 6] records = run_multiple_experiments(experiments, parallel=True) assert [record.get_result() for record in records] == [3, 4, 6]
def test_run_multiple_experiments(): with experiment_testing_context(): experiments = my_api_test.get_all_variants() assert len(experiments)==3 records = run_multiple_experiments(experiments) assert [record.get_result() for record in records] == [3, 4, 6] records = run_multiple_experiments(experiments, parallel=True) assert [record.get_result() for record in records] == [3, 4, 6]
def run(self, *args): parser = argparse.ArgumentParser() parser.add_argument('user_range', action='store', help='A selection of experiments to run. Examples: "3" or "3-5", or "3,4,5"') parser.add_argument('-p', '--parallel', default=False, nargs='*') parser.add_argument('-n', '--note') parser.add_argument('-e', '--raise_errors', default='single', nargs='*', help='By default, error are only raised if a single experiment is run. Set "-e" to always rays errors. "-e 0" to never raise errors.') parser.add_argument('-d', '--display_results', default=False, action = "store_true") parser.add_argument('-s', '--slurm', default=False, action = "store_true", help='Run with slurm') args = parser.parse_args(args) n_processes = \ None if args.parallel is False else \ 'all' if len(args.parallel)==0 else \ int(args.parallel[0]) if len(args.parallel)==1 else \ bad_value(args.parallel, '-p can have 0 or 1 arguments. Got: {}'.format(args.parallel)) ids = select_experiments(args.user_range, self.exp_record_dict) # Raise errors if: # -e # -e 1 # No arg, and only 1 experiment running raise_errors = (len(args.raise_errors)==0 or (len(args.raise_errors)==1 and args.raise_errors[0]=='1') or (args.raise_errors == 'single' and len(ids)==1)) if args.slurm: run_multiple_experiments_with_slurm( experiments=[load_experiment(eid) for eid in ids], n_parallel = n_processes, raise_exceptions = raise_errors, run_args=self.run_args, slurm_kwargs=self.slurm_kwargs ) else: exp_names = list(self.exp_record_dict.keys()) run_multiple_experiments( experiments=[load_experiment(eid) for eid in ids], prefixes=[exp_names.index(eid) for eid in ids], parallel=n_processes, raise_exceptions = raise_errors, run_args=self.run_args, notes=(args.note, ) if args.note is not None else (), display_results=args.display_results ) result = _warn_with_prompt('Finished running {} experiment{}.'.format(len(ids), '' if len(ids)==1 else 's'), use_prompt=not self.close_after, prompt='Press Enter to Continue, or "q" then Enter to Quit') if result=='q': quit()
def run(self, *args): parser = argparse.ArgumentParser() parser.add_argument('user_range', action='store', help='A selection of experiments to run. Examples: "3" or "3-5", or "3,4,5"') parser.add_argument('-p', '--parallel', default=False, nargs='*') parser.add_argument('-n', '--note') parser.add_argument('-e', '--raise_errors', default='single', nargs='*', help='By default, error are only raised if a single experiment is run. Set "-e" to always rays errors. "-e 0" to never raise errors.') parser.add_argument('-d', '--display_results', default=False, action = "store_true") parser.add_argument('-s', '--slurm', default=False, action = "store_true", help='Run with slurm') args = parser.parse_args(args) n_processes = \ None if args.parallel is False else \ 'all' if len(args.parallel)==0 else \ int(args.parallel[0]) if len(args.parallel)==1 else \ bad_value(args.parallel, '-p can have 0 or 1 arguments. Got: {}'.format(args.parallel)) ids = select_experiments(args.user_range, self.exp_record_dict) # Raise errors if: # -e # -e 1 # No arg, and only 1 experiment running raise_errors = (len(args.raise_errors)==0 or (len(args.raise_errors)==1 and args.raise_errors[0]=='1') or (args.raise_errors == 'single' and len(ids)==1)) if args.slurm: run_multiple_experiments_with_slurm( experiments=[load_experiment(eid) for eid in ids], n_parallel = n_processes, raise_exceptions = raise_errors, run_args=self.run_args, slurm_kwargs=self.slurm_kwargs ) else: exp_names = list(self.exp_record_dict.keys()) run_multiple_experiments( experiments=[load_experiment(eid) for eid in ids], prefixes=[exp_names.index(eid) for eid in ids], parallel=n_processes, raise_exceptions = raise_errors, run_args=self.run_args, notes=(args.note, ) if args.note is not None else (), display_results=args.display_results ) result = _warn_with_prompt('Finished running {} experiment{}.'.format(len(ids), '' if len(ids)==1 else 's'), use_prompt=not self.close_after, prompt='Press Enter to Continue, or "q" then Enter to Quit') if result=='q': quit()
def run(self, user_range, mode='-s', raise_exceptions=''): assert mode in ('-s', '-e') or mode.startswith('-p') ids = select_experiments(user_range, self.exp_record_dict) run_multiple_experiments( experiments=[load_experiment(eid) for eid in ids], parallel=len(ids) > 1 and mode.startswith('-p'), raise_exceptions=raise_exceptions == '-e', run_args=self.run_args) result = _warn_with_prompt( 'Finished running {} experiment{}.'.format( len(ids), '' if len(ids) == 1 else 's'), use_prompt=not self.close_after, prompt='Press Enter to Continue, or "q" then Enter to Quit') if result == 'q': quit()
def test_parallel_run_errors(): with experiment_testing_context(): @experiment_function def my_error_causing_test(a=1): raise Exception('nononono') my_error_causing_test.add_variant(a=2) variants = my_error_causing_test.get_all_variants() assert len(variants)==2 run_multiple_experiments(variants, parallel=True, raise_exceptions=False) with pytest.raises(Exception) as err: run_multiple_experiments(variants, parallel=True, raise_exceptions=True) print("^^^ Dont't worry, the above is not actually an error, we were just asserting that we caught the error.") assert str(err.value) == 'nononono'