def run_evaluation(config_filepath, backend="nccl", with_clearml=True):
"""Main entry to run model's evaluation:
- compute validation metrics
Args:
config_filepath (str): evaluation configuration .py file
backend (str): distributed backend: nccl, gloo, horovod or None to run without distributed config
with_clearml (bool): if True, uses ClearML as experiment tracking system
"""
assert torch.cuda.is_available(), torch.cuda.is_available()
assert torch.backends.cudnn.enabled
torch.backends.cudnn.benchmark = True
config_filepath = Path(config_filepath)
assert config_filepath.exists(), f"File '{config_filepath.as_posix()}' is not found"
with idist.Parallel(backend=backend) as parallel:
logger = setup_logger(name="Pascal-VOC12 Evaluation", distributed_rank=idist.get_rank())
config = ConfigObject(config_filepath)
InferenceConfigSchema.validate(config)
config.script_filepath = Path(__file__)
output_path = setup_experiment_tracking(config, with_clearml=with_clearml, task_type="testing")
config.output_path = output_path
utils.log_basic_info(logger, get_params(config, InferenceConfigSchema))
try:
parallel.run(evaluation, config, logger=logger, with_clearml=with_clearml)
except KeyboardInterrupt:
logger.info("Catched KeyboardInterrupt -> exit")
except Exception as e: # noqa
logger.exception("")
raise e
def test_config_object_mutations_assert(config_filepath):
with pytest.raises(TypeError, match=r"Argument mutations should be a mapping"):
ConfigObject(config_filepath, mutations="abc")
class A:
pass
with pytest.raises(ValueError, match=r"Failed to create value's AST"):
ConfigObject(config_filepath, mutations={"a": A()})
def setup_config(config_file):
config = ConfigObject(config_file)
config.seed = 12
config.debug = True
config.device = "cpu"
config.num_epochs = 12
if has_torch:
config.model = torch.nn.Linear(1, 1)
config.criterion = torch.nn.CrossEntropyLoss()
config.optimizer = 123
config.train_loader = [1, 2, 3]
return config
def test_config_object(config_filepath):
config = ConfigObject(config_filepath)
assert "a" in config
assert config["a"] == config.a == config.get("a") == 1
assert "b" in config
assert config["b"] == config.b == config.get("b") == 2
assert config["_data"] == config._data == config.get("_data") == 3
config.c = 3
config["d"] = 4
assert "c" in config
assert config["c"] == config.c == config.get("c") == 3
assert config["d"] == config.d == config.get("d") == 4
assert "config_filepath" in config
assert isinstance(config.config_filepath, Path)
assert config.config_filepath == config_filepath
assert config["config_filepath"] == config_filepath
assert config.get("config_filepath") == config_filepath
for k in config:
assert not k.startswith("__")
def foo(**kwargs):
for k in ["a", "b", "c", "d", "config_filepath"]:
assert k in kwargs
foo(**config)
for k, v in config.__dict__.items():
assert not inspect.ismodule(v)
def test_config_object_no_modules(mutations, config_filepath2):
import numpy as np
config = ConfigObject(config_filepath2, mutations=mutations)
for k, v in config.items():
assert not inspect.ismodule(v), f"{k}: {v}"
assert "a" in config
assert config.a == 1 if mutations is None else [1, 2, 3]
assert "arr" in config
np.testing.assert_allclose(config.arr, np.array([1, 2, 3]))
assert "out" in config
assert config.out == 12
def test_mp_config2(method, config_filepath2):
config = ConfigObject(config_filepath2)
ctx = mp.get_context(method)
p = ctx.Process(target=worker_config_checker, args=(config,))
p.start()
p.join()
def test_mp_config(method, config_filepath):
config = ConfigObject(config_filepath)
ctx = mp.get_context(method)
p = ctx.Process(target=worker_function, args=(config,))
p.start()
p.join()
def test_config_object_repr(config_filepath):
config = ConfigObject(config_filepath)
out = repr(config)
assert "a" in out
assert "b" in out
assert "data" in out
assert "_data" in out
def test_config_object_loading(config_filepath):
config = ConfigObject(config_filepath)
def foo(**kwargs):
for k in ["a", "b", "config_filepath"]:
assert k in kwargs
foo(**config)
def test_config_object_mutations(dirname):
filepath = dirname / "custom_module.py"
s = """
a = 123
b = 12.3
c = "abc"
d = True
# e = None
def func(x):
return x + a
out = func(10)
def func2(x):
return x + b
out2 = func2(1.0)
def func3(x):
if x == "abc":
return 1.0
elif x == "cba":
return -1.0
else:
return 0.0
out3 = func3(c)
out4 = 10 if d else -10
# out5 = 10 if e is None else -10
"""
with filepath.open("w") as h:
h.write(s)
config = ConfigObject(filepath, mutations={"a": 333, "b": 22.0, "c": "cba", "d": False})
assert config.a == 333
assert config.out == 10 + 333
assert config.b == 22.0
assert config.out2 == 1.0 + 22.0
assert config.c == "cba"
assert config.out3 == -1.0
assert not config.d
assert config.out4 == -10
def test_config_object_mutations_nonconst(old_value, new_value, dirname):
filepath = dirname / "custom_module.py"
s = f"""
a = {old_value}
"""
with filepath.open("w") as h:
h.write(s)
config = ConfigObject(filepath, mutations={"a": new_value})
assert config.a == new_value
def test_config_object_lazy_load(dirname):
filepath = dirname / "bad_config.py"
s = """
a = 123
raise RuntimeError("error")
"""
with filepath.open("w") as h:
h.write(s)
config = ConfigObject(filepath)
with pytest.raises(RuntimeError, match=r"error"):
assert config.a == 123
def test_config_object_mutations_validate(dirname):
filepath = dirname / "custom_module.py"
s = """
a = 123
def func(x):
return x + a
out = func(10)
"""
with filepath.open("w") as h:
h.write(s)
config = ConfigObject(filepath, mutations={"a": 333, "b": 22.0})
with pytest.raises(RuntimeError, match=r"Following mutations were not applied"):
assert config.a == 333
import argparse
from pathlib import Path
from py_config_runner import ConfigObject
from training import run
if __name__ == "__main__":
parser = argparse.ArgumentParser("Example application")
parser.add_argument("--config", type=Path, help="Input configuration file")
args = parser.parse_args()
assert args.config is not None
assert args.config.exists()
# Pass configuration file into py_config_runner.ConfigObject
# and fetch configuration parameters as attributes
# see inside run() function
config = ConfigObject(args.config)
run(config)
default=None,
help="Override train batch size")
parser.add_argument("--lr",
type=float,
default=None,
help="Override train learning rate")
parser.add_argument("--ep",
type=int,
default=None,
help="Override number of epochs")
args = parser.parse_args()
assert args.config is not None
assert args.config.exists()
# Define configuration mutations if certain cmd args are defined
mutations = {}
if args.bs is not None:
mutations["train_batch_size"] = args.bs
if args.lr is not None:
mutations["learning_rate"] = args.lr
if args.ep is not None:
mutations["num_epochs"] = args.ep
# Pass configuration file into py_config_runner.ConfigObject
# and fetch configuration parameters as attributes
# see inside run() function
config = ConfigObject(args.config, mutations=mutations)
run(config)
def test_config_object_init_kwargs(config_filepath):
# Pass a as kwargs
config = ConfigObject(config_filepath, a=10, another_data=123)
# assert that a is overriden by config_filepath
assert config.a == 1
assert config.another_data == 123
def test_config_object_items(config_filepath):
config = ConfigObject(config_filepath)
res = [(k, v) for k, v in config.items()]
assert len(res) == 4 + 1 # config + config_filepath
def test_config_object_length(config_filepath):
config = ConfigObject(config_filepath)
assert len(config) == 4 + 1 # config + config_filepath