def run_sql_experiment(main,
mode,
include_folders=None,
log_dir=None,
exp_prefix="experiment",
exp_name=None,
**kwargs):
if exp_name is None:
exp_name = timestamp()
if log_dir is None:
log_dir = os.path.join(DEFAULT_LOG_DIR, "local",
exp_prefix.replace("_", "-"), exp_name)
if include_folders is None:
include_folders = list()
if mode == 'ec2':
include_folders.append('softqlearning')
all_symlinks = list()
for folder in include_folders:
all_symlinks.append(_create_symlink(folder))
kwargs.update(added_project_directories=all_symlinks)
run_experiment(
method_call=main,
mode=mode,
exp_prefix=exp_prefix,
exp_name=exp_name,
log_dir=log_dir,
**kwargs)
def test_default_log_dir():
default_path = osp.join(os.getcwd(), 'data/local/experiment')
ensure_dir(default_path)
folder_num = len(os.listdir(default_path))
run_experiment(dummy_func)
assert len(os.listdir(default_path)) == folder_num + 1
def test_default_log_dir():
default_path = os.path.join(os.getcwd(), 'data/local/experiment')
pathlib.Path(default_path).mkdir(parents=True, exist_ok=True)
folder_num = len(os.listdir(default_path))
run_experiment(dummy_func)
assert len(os.listdir(default_path)) == folder_num + 1
def test_default_log_dir():
# Because this test uses the default log directory, if any other tests are
# run in parallel with it that use this directory, it will fail.
# For this reason, all tests which use run_experiment, must pass a
# non-default exp_prefix or log_dir.
default_path = os.path.join(os.getcwd(), 'data/local/experiment')
pathlib.Path(default_path).mkdir(parents=True, exist_ok=True)
folder_num = len(os.listdir(default_path))
run_experiment(dummy_func)
assert len(os.listdir(default_path)) == folder_num + 1
def test_experiment_with_variant():
# Note: exp_name in variant does nothing.
exp_variant = {'exp_name': 'test_name'}
exp_path = os.path.join(os.getcwd(), 'data/local/test-prefix')
pathlib.Path(exp_path).mkdir(parents=True, exist_ok=True)
old_folder_contents = set(os.listdir(exp_path))
# Pass a non-default exp_prefix, so test_default_log_dir is safe.
run_experiment(dummy_func, exp_prefix='test_prefix', variant=exp_variant)
new_folder_contents = set(os.listdir(exp_path))
folder_content_diff = new_folder_contents - old_folder_contents
assert len(folder_content_diff) == 1
exp_folder_name = folder_content_diff.pop()
assert exp_folder_name.startswith('test_prefix')
def test_experiment_with_not_callable_task():
with pytest.raises(ValueError):
run_experiment(1)
def test_experiment_with_none_task():
with pytest.raises(Exception):
run_experiment()
def test_experiment_with_variant():
variant = {'exp_name': 'test'}
run_experiment(dummy_func, variant=variant)
