def test_serialization_deserialization(self, monkeypatch, tmp_path):
"""Test serialization/de-serialization"""
with monkeypatch.context() as m:
m.setattr(sys, "argv",
["", "--config", "./tests/conf/yaml/test.yaml"])
# Serialize
config = ConfigArgBuilder(
*all_configs,
desc="Test Builder",
)
now = datetime.datetime.now()
curr_int_time = int(
f"{now.year}{now.month}{now.day}{now.hour}{now.second}")
config_values = config.save(
file_extension=".yaml",
file_name=f"pytest.{curr_int_time}",
user_specified_path=tmp_path).generate()
yaml_regex = re.compile(
fr"pytest.{curr_int_time}."
fr"[a-fA-F0-9]{{8}}-[a-fA-F0-9]{{4}}-[a-fA-F0-9]{{4}}-"
fr"[a-fA-F0-9]{{4}}-[a-fA-F0-9]{{12}}.spock.cfg.yaml")
matches = [
re.fullmatch(yaml_regex, val)
for val in os.listdir(str(tmp_path))
if re.fullmatch(yaml_regex, val) is not None
]
fname = f"{str(tmp_path)}/{matches[0].string}"
# Deserialize
m.setattr(sys, "argv", ["", "--config", f"{fname}"])
de_serial_config = ConfigArgBuilder(
*all_configs,
desc="Test Builder",
).generate()
assert config_values == de_serial_config
def test_yaml_s3_mock_file_writer_missing_s3(self, monkeypatch, tmp_path,
s3):
"""Test the YAML writer works correctly"""
with monkeypatch.context() as m:
aws_session, s3_client = s3
m.setattr(sys, "argv",
["", "--config", "./tests/conf/yaml/test.yaml"])
config = ConfigArgBuilder(
*all_configs,
desc="Test Builder",
)
# Mock a S3 bucket and object
mock_s3_bucket = "spock-test"
mock_s3_object = "fake/test/bucket/"
s3_client.create_bucket(Bucket=mock_s3_bucket)
# Test the chained version
now = datetime.datetime.now()
curr_int_time = int(
f"{now.year}{now.month}{now.day}{now.hour}{now.second}")
with pytest.raises(ValueError):
# Test the chained version
config.save(
user_specified_path=f"s3://foo/{mock_s3_object}",
file_extension=".yaml",
file_name=f"pytest.save.{curr_int_time}",
).generate()
def arg_builder(monkeypatch):
with monkeypatch.context() as m:
m.setattr(sys, 'argv', ['', '--config',
'./tests/conf/json/test.json'])
config = ConfigArgBuilder(TypeConfig, TypeOptConfig, TypeDefaultConfig, TypeDefaultOptConfig,
desc='Test Builder')
return config.generate()
def test_save_top_level(self, monkeypatch):
with monkeypatch.context() as m:
m.setattr(sys, "argv",
["", "--config", "./tests/conf/yaml/test_optuna.yaml"])
# Optuna config -- this will internally spawn the study object for the define-and-run style which will be returned
# as part of the call to sample()
optuna_config = OptunaTunerConfig(
study_name="Iris Logistic Regression Tests",
direction="maximize")
now = datetime.datetime.now()
curr_int_time = int(
f"{now.year}{now.month}{now.day}{now.hour}{now.second}")
config = (ConfigArgBuilder(LogisticRegressionHP).tuner(
optuna_config).save(
user_specified_path="/tmp",
file_name=f"pytest.{curr_int_time}",
).sample())
# Verify the sample was written out to file
yaml_regex = re.compile(
fr"pytest.{curr_int_time}."
fr"[a-fA-F0-9]{{8}}-[a-fA-F0-9]{{4}}-[a-fA-F0-9]{{4}}-"
fr"[a-fA-F0-9]{{4}}-[a-fA-F0-9]{{12}}.spock.cfg.yaml")
matches = [
re.fullmatch(yaml_regex, val) for val in os.listdir("/tmp")
if re.fullmatch(yaml_regex, val) is not None
]
fname = f"/tmp/{matches[0].string}"
assert os.path.exists(fname)
with open(fname, "r") as fin:
print(fin.read())
# Clean up if assert is good
if os.path.exists(fname):
os.remove(fname)
return config
def arg_builder(monkeypatch):
with monkeypatch.context() as m:
m.setattr(
sys,
"argv",
[
"",
"--config",
"./tests/conf/yaml/test_hp.yaml",
"--HPOne.hp_int.bounds",
"(1, 1000)",
"--HPOne.hp_int_log.bounds",
"(1, 1000)",
"--HPOne.hp_float.bounds",
"(1.0, 1000.0)",
"--HPOne.hp_float_log.bounds",
"(1.0, 1000.0)",
"--HPTwo.hp_choice_int.choices",
"[1, 2, 4, 8]",
"--HPTwo.hp_choice_float.choices",
"[1.0, 2.0, 4.0, 8.0]",
"--HPTwo.hp_choice_str.choices",
"['is', 'it ', 'me', 'youre', 'looking', 'for']",
],
)
optuna_config = OptunaTunerConfig(study_name="Tests",
direction="maximize")
config = ConfigArgBuilder(HPOne, HPTwo).tuner(optuna_config)
return config
def test_type_unknown(self, monkeypatch):
with monkeypatch.context() as m:
m.setattr(
sys, "argv",
["", "--config", "./tests/conf/yaml/test_incorrect.yaml"])
with pytest.raises(ValueError):
ConfigArgBuilder(*all_configs, desc="Test Builder")
def test_raises_missing_class(self, monkeypatch):
with monkeypatch.context() as m:
m.setattr(sys, "argv",
["", "--config", "./tests/conf/yaml/test.yaml"])
with pytest.raises(ValueError):
config = ConfigArgBuilder(*all_configs[:-1])
return config.generate()
def test_choice_raise(self, monkeypatch):
with monkeypatch.context() as m:
with pytest.raises(ValueError):
ConfigArgBuilder(
*all_configs,
no_cmd_line=True,
)
def test_yaml_file_writer_save_path(self, monkeypatch):
"""Test the YAML writer works correctly"""
with monkeypatch.context() as m:
m.setattr(
sys, "argv",
["", "--config", "./tests/conf/yaml/test_save_path.yaml"])
config = ConfigArgBuilder(
*all_configs,
desc="Test Builder",
)
# Test the chained version
now = datetime.datetime.now()
curr_int_time = int(
f"{now.year}{now.month}{now.day}{now.hour}{now.second}")
config_values = config.save(
file_extension=".yaml",
file_name=f"pytest.{curr_int_time}").generate()
yaml_regex = re.compile(
fr"pytest.{curr_int_time}."
fr"[a-fA-F0-9]{{8}}-[a-fA-F0-9]{{4}}-[a-fA-F0-9]{{4}}-"
fr"[a-fA-F0-9]{{4}}-[a-fA-F0-9]{{12}}.spock.cfg.yaml")
matches = [
re.fullmatch(yaml_regex, val)
for val in os.listdir(str(config_values.TypeConfig.save_path))
if re.fullmatch(yaml_regex, val) is not None
]
fname = f"{str(config_values.TypeConfig.save_path)}/{matches[0].string}"
with open(fname, "r") as fin:
print(fin.read())
assert os.path.exists(fname)
# Clean up if assert is good
if os.path.exists(fname):
os.remove(fname)
def arg_builder(monkeypatch):
with monkeypatch.context() as m:
m.setattr(sys, 'argv', [
'',
'--config',
'./tests/conf/yaml/test_class.yaml',
'--TypeConfig.bool_p',
'--TypeConfig.int_p',
'11',
'--TypeConfig.float_p',
'11.0',
'--TypeConfig.string_p',
'Hooray',
'--TypeConfig.list_p_float',
'[11.0,21.0]',
'--TypeConfig.list_p_int',
'[11, 21]',
'--TypeConfig.list_p_str',
"['Hooray', 'Working']",
'--TypeConfig.list_p_bool',
'[False, True]',
'--TypeConfig.tuple_p_float',
'(11.0, 21.0)',
'--TypeConfig.tuple_p_int',
'(11, 21)',
'--TypeConfig.tuple_p_str',
"('Hooray', 'Working')",
'--TypeConfig.tuple_p_bool',
'(False, True)',
'--TypeConfig.list_list_p_int',
"[[11, 21], [11, 21]]",
'--TypeConfig.choice_p_str',
'option_2',
'--TypeConfig.choice_p_int',
'20',
'--TypeConfig.choice_p_float',
'20.0',
'--TypeConfig.list_choice_p_str',
"['option_2']",
'--TypeConfig.list_list_choice_p_str',
"[['option_2'], ['option_2']]",
'--TypeConfig.list_choice_p_int',
'[20]',
'--TypeConfig.list_choice_p_float',
'[20.0]',
'--NestedStuff.one',
'12',
'--NestedStuff.two',
'ancora',
'--TypeConfig.nested_list.NestedListStuff.one',
'[11, 21]',
'--TypeConfig.nested_list.NestedListStuff.two',
"['Hooray', 'Working']",
])
config = ConfigArgBuilder(TypeConfig,
NestedStuff,
NestedListStuff,
desc='Test Builder')
return config.generate()
def test_choice_raise(self, monkeypatch):
with monkeypatch.context() as m:
with pytest.raises(ValueError):
ConfigArgBuilder(TypeConfig,
NestedStuff,
NestedListStuff,
TypeOptConfig,
no_cmd_line=True)
def test_sample_before_tuner(self, monkeypatch):
with monkeypatch.context() as m:
m.setattr(sys, "argv",
["", "--config", "./tests/conf/yaml/test_hp.yaml"])
optuna_config = OptunaTunerConfig(study_name="Basic Tests",
direction="maximize")
with pytest.raises(RuntimeError):
config = ConfigArgBuilder(HPOne, HPTwo).sample()
def arg_builder(monkeypatch):
with monkeypatch.context() as m:
m.setattr(sys, "argv",
["", "--config", "./tests/conf/yaml/test_hp.yaml"])
optuna_config = OptunaTunerConfig(study_name="Sample Tests",
direction="maximize")
config = ConfigArgBuilder(HPOne, HPTwo).tuner(optuna_config)
return config
def arg_builder(monkeypatch):
with monkeypatch.context() as m:
config = ConfigArgBuilder(
*all_configs,
no_cmd_line=True,
configs=["./tests/conf/yaml/test.yaml"],
)
return config.generate()
def test_coptional_raise(self, monkeypatch):
with monkeypatch.context() as m:
# m.setattr(sys, "argv", ["", "--config", "./tests/conf/yaml/empty.yaml"])
with pytest.raises(_SpockInstantiationError):
ConfigArgBuilder(OptionalFail,
desc="Test Builder",
configs=[],
no_cmd_line=True)
def test_invalid_cast_choice(self, monkeypatch):
with monkeypatch.context() as m:
m.setattr(sys, "argv",
["", "--config", "./tests/conf/yaml/test_hp_cast.yaml"])
optuna_config = OptunaTunerConfig(study_name="Basic Tests",
direction="maximize")
with pytest.raises(TypeError):
config = ConfigArgBuilder(HPOne, HPTwo).tuner(optuna_config)
def test_incorrect_tuner_config(self, monkeypatch):
with monkeypatch.context() as m:
m.setattr(sys, "argv",
["", "--config", "./tests/conf/yaml/test_hp.yaml"])
optuna_config = optuna.create_study(study_name="Tests",
direction="minimize")
with pytest.raises(TypeError):
config = ConfigArgBuilder(HPOne, HPTwo).tuner(optuna_config)
def test_callable_module(self, monkeypatch):
with monkeypatch.context() as m:
m.setattr(
sys, "argv",
["", "--config", "./tests/conf/yaml/test_fail_callable.yaml"])
with pytest.raises(_SpockValueError):
config = ConfigArgBuilder(*all_configs)
return config.generate()
def test_class_unknown(self, monkeypatch):
with monkeypatch.context() as m:
m.setattr(
sys,
"argv",
["", "--config", "./tests/conf/yaml/test_missing_class.yaml"],
)
with pytest.raises(TypeError):
ConfigArgBuilder(*all_configs, desc="Test Builder")
def arg_builder(monkeypatch):
with monkeypatch.context() as m:
m.setattr(sys, 'argv',
['', '--config', './tests/conf/yaml/inherited.yaml'])
config = ConfigArgBuilder(TypeInherited,
NestedStuff,
NestedListStuff,
desc='Test Builder')
return config.generate()
def test_cmd_line_kwarg_raise(self, monkeypatch):
with monkeypatch.context() as m:
with pytest.raises(TypeError):
config = ConfigArgBuilder(
*all_configs,
no_cmd_line=True,
configs="./tests/conf/yaml/test.yaml",
)
return config.generate()
def test_config_cycles(self, monkeypatch):
with monkeypatch.context() as m:
m.setattr(sys, 'argv',
['', '--config', './tests/conf/yaml/test_cycle.yaml'])
with pytest.raises(ValueError):
ConfigArgBuilder(TypeConfig,
NestedStuff,
NestedListStuff,
desc='Test Builder')
def arg_builder(monkeypatch):
with monkeypatch.context() as m:
config = ConfigArgBuilder(TypeConfig,
NestedStuff,
NestedListStuff,
TypeOptConfig,
no_cmd_line=True,
configs=['./tests/conf/yaml/test.yaml'])
return config.generate()
def test_default_file_writer(self, monkeypatch, tmp_path):
"""Test the default writer works correctly"""
with monkeypatch.context() as m:
m.setattr(sys, 'argv', ['', '--config',
'./tests/conf/yaml/test.yaml'])
config = ConfigArgBuilder(TypeConfig, TypeOptConfig, desc='Test Builder')
# Test the chained version
config.save(user_specified_path=tmp_path).generate()
assert len(list(tmp_path.iterdir())) == 1
def arg_builder(monkeypatch):
with monkeypatch.context() as m:
m.setattr(sys, "argv", [""])
config = ConfigArgBuilder(
*all_configs,
desc="Test Builder",
configs=["./tests/conf/yaml/test.yaml"],
)
return config.generate()
def arg_builder(monkeypatch):
with monkeypatch.context() as m:
m.setattr(
sys,
"argv",
[
"", "--config", "./tests/conf/yaml/test_class.yaml",
"--TypeConfig.bool_p", "--TypeConfig.int_p", "11",
"--TypeConfig.float_p", "11.0", "--TypeConfig.string_p",
"Hooray", "--TypeConfig.list_p_float", "[11.0,21.0]",
"--TypeConfig.list_p_int", "[11, 21]",
"--TypeConfig.list_p_str", "['Hooray', 'Working']",
"--TypeConfig.list_p_bool", "[False, True]",
"--TypeConfig.tuple_p_float", "(11.0, 21.0)",
"--TypeConfig.tuple_p_int", "(11, 21)",
"--TypeConfig.tuple_p_str", "('Hooray', 'Working')",
"--TypeConfig.tuple_p_bool", "(False, True)",
"--TypeConfig.tuple_p_mixed", "(5, 11.5)",
"--TypeConfig.list_list_p_int", "[[11, 21], [11, 21]]",
"--TypeConfig.choice_p_str", "option_2",
"--TypeConfig.choice_p_int", "20",
"--TypeConfig.choice_p_float", "20.0",
"--TypeConfig.list_choice_p_str", "['option_2']",
"--TypeConfig.list_list_choice_p_str",
"[['option_2'], ['option_2']]",
"--TypeConfig.list_choice_p_int", "[20]",
"--TypeConfig.list_choice_p_float", "[20.0]",
"--TypeConfig.class_enum", "NestedStuff",
"--NestedStuff.one", "12", "--NestedStuff.two", "ancora",
"--TypeConfig.nested_list.NestedListStuff.one", "[11, 21]",
"--TypeConfig.nested_list.NestedListStuff.two",
"['Hooray', 'Working']", "--TypeConfig.high_config",
"SingleNestedConfig",
"--SingleNestedConfig.double_nested_config",
"SecondDoubleNestedConfig",
"--SecondDoubleNestedConfig.morph_tolerance", "0.2",
"--TypeConfig.call_me", 'tests.base.attr_configs_test.bar',
"--TypeConfig.call_us",
"['tests.base.attr_configs_test.bar', 'tests.base.attr_configs_test.bar']",
"--TypeConfig.str_dict", "{'key_1': 2.5, 'key_2': 3.5}",
"--TypeConfig.int_list_str_dict",
"{'1': ['again', 'test'], '2': ['test', 'me']}",
"--TypeConfig.float_tuple_callable_dict",
'{"1.0": ("tests.base.attr_configs_test.bar", "tests.base.attr_configs_test.foo"), "2.0": ("tests.base.attr_configs_test.bar", "tests.base.attr_configs_test.foo")}',
"--TypeConfig.hardest",
'[{"key_1": ("tests.base.attr_configs_test.bar", "tests.base.attr_configs_test.foo"), "key_2": ("tests.base.attr_configs_test.bar", "tests.base.attr_configs_test.foo")}, {"key_3": ("tests.base.attr_configs_test.bar", "tests.base.attr_configs_test.foo"), "key_4": ("tests.base.attr_configs_test.bar", "tests.base.attr_configs_test.foo")}]'
],
)
config = ConfigArgBuilder(TypeConfig,
NestedStuff,
NestedListStuff,
SingleNestedConfig,
FirstDoubleNestedConfig,
SecondDoubleNestedConfig,
desc="Test Builder")
return config.generate()
def test_class_parameter_unknown(self, monkeypatch):
with monkeypatch.context() as m:
m.setattr(sys, 'argv', [
'', '--config', './tests/conf/yaml/test_class_incorrect.yaml'
])
with pytest.raises(ValueError):
ConfigArgBuilder(TypeConfig,
NestedStuff,
NestedListStuff,
desc='Test Builder')
def test_unknown_arg(self, monkeypatch):
with monkeypatch.context() as m:
m.setattr(
sys, "argv",
["", "--config", "./tests/conf/yaml/test_hp_unknown_arg.yaml"])
optuna_config = OptunaTunerConfig(study_name="Basic Tests",
direction="maximize")
with pytest.raises(ValueError):
config = ConfigArgBuilder(HPOne, HPTwo).tuner(optuna_config)
return config
def arg_builder(monkeypatch):
with monkeypatch.context() as m:
m.setattr(sys, 'argv',
['', '--config', './tests/conf/toml/test.toml'])
config = ConfigArgBuilder(TypeConfig,
NestedStuff,
NestedListStuff,
TypeOptConfig,
desc='Test Builder')
return config.generate()
def test_wrong_input_raise(self, monkeypatch):
with monkeypatch.context() as m:
m.setattr(sys, "argv",
["", "--config", "./tests/conf/yaml/test.foo"])
with pytest.raises(TypeError):
config = ConfigArgBuilder(
*all_configs,
desc="Test Builder",
)
return config.generate()
