def run_sql_experiment(main, mode, include_folders=None, log_dir=None, exp_prefix="experiment", exp_name=None, **kwargs): if exp_name is None: exp_name = timestamp() if log_dir is None: log_dir = os.path.join(DEFAULT_LOG_DIR, "local", exp_prefix.replace("_", "-"), exp_name) if include_folders is None: include_folders = list() if mode == 'ec2': include_folders.append('softqlearning') all_symlinks = list() for folder in include_folders: all_symlinks.append(_create_symlink(folder)) kwargs.update(added_project_directories=all_symlinks) run_experiment( method_call=main, mode=mode, exp_prefix=exp_prefix, exp_name=exp_name, log_dir=log_dir, **kwargs)
def test_default_log_dir(): default_path = osp.join(os.getcwd(), 'data/local/experiment') ensure_dir(default_path) folder_num = len(os.listdir(default_path)) run_experiment(dummy_func) assert len(os.listdir(default_path)) == folder_num + 1
def test_default_log_dir(): default_path = os.path.join(os.getcwd(), 'data/local/experiment') pathlib.Path(default_path).mkdir(parents=True, exist_ok=True) folder_num = len(os.listdir(default_path)) run_experiment(dummy_func) assert len(os.listdir(default_path)) == folder_num + 1
def test_default_log_dir(): # Because this test uses the default log directory, if any other tests are # run in parallel with it that use this directory, it will fail. # For this reason, all tests which use run_experiment, must pass a # non-default exp_prefix or log_dir. default_path = os.path.join(os.getcwd(), 'data/local/experiment') pathlib.Path(default_path).mkdir(parents=True, exist_ok=True) folder_num = len(os.listdir(default_path)) run_experiment(dummy_func) assert len(os.listdir(default_path)) == folder_num + 1
def test_experiment_with_variant(): # Note: exp_name in variant does nothing. exp_variant = {'exp_name': 'test_name'} exp_path = os.path.join(os.getcwd(), 'data/local/test-prefix') pathlib.Path(exp_path).mkdir(parents=True, exist_ok=True) old_folder_contents = set(os.listdir(exp_path)) # Pass a non-default exp_prefix, so test_default_log_dir is safe. run_experiment(dummy_func, exp_prefix='test_prefix', variant=exp_variant) new_folder_contents = set(os.listdir(exp_path)) folder_content_diff = new_folder_contents - old_folder_contents assert len(folder_content_diff) == 1 exp_folder_name = folder_content_diff.pop() assert exp_folder_name.startswith('test_prefix')
def test_experiment_with_not_callable_task(): with pytest.raises(ValueError): run_experiment(1)
def test_experiment_with_none_task(): with pytest.raises(Exception): run_experiment()
def test_experiment_with_variant(): variant = {'exp_name': 'test'} run_experiment(dummy_func, variant=variant)