def test_registered_create_array_arg(self):
CobaRegistry.register("test", TestArgObject)
klass = CobaRegistry.construct({"test": [1, 2, 3]})
self.assertEqual(klass.arg, [1, 2, 3])
def _construct(item: Any) -> Sequence[Any]:
result = None
if isinstance(item, str) and item in variables:
result = variables[item]
if isinstance(item, str) and item not in variables:
result = CobaRegistry.construct(item)
if isinstance(item, dict):
result = CobaRegistry.construct(item)
if isinstance(item, list):
pieces = list(map(_construct, item))
if hasattr(pieces[0][0], 'read'):
result = [
Pipes.join(s, *f) for s in pieces[0]
for f in product(*pieces[1:])
]
else:
result = sum(pieces, [])
if result is None:
raise CobaException(
f"We were unable to construct {item} in the given environment definition file."
)
return result if isinstance(
result, collections.abc.Sequence) else [result]
def test_not_registered(self):
CobaRegistry.register("test", TestObject)
with self.assertRaises(Exception) as cm:
CobaRegistry.construct("test2")
self.assertEqual("Unknown recipe test2", str(cm.exception))
def test_registered_create(self):
CobaRegistry.register("test", TestObject)
klass = CobaRegistry.construct("test")
self.assertEqual(klass.args, ())
self.assertEqual(klass.kwargs, {})
def test_registered_create_args3(self):
CobaRegistry.register("test", TestObject)
klass = CobaRegistry.construct({"test": "abc"})
self.assertEqual(klass.args, ("abc", ))
self.assertEqual(klass.kwargs, {})
def test_registered_create_args_kwargs(self):
CobaRegistry.register("test", TestObject)
klass = CobaRegistry.construct({"test": [1, 2, 3], "kwargs": {"a": 1}})
self.assertEqual(klass.args, (1, 2, 3))
self.assertEqual(klass.kwargs, {"a": 1})
def _config(cls) -> Dict[str, Any]:
if cls._config_backing is None:
try:
_raw_config: Dict[str, Any] = {
"api_keys": collections.defaultdict(lambda: None),
"cacher": {
"DiskCacher": None
},
"logger": {
"IndentLogger": "Console"
},
"experiment": {
"processes": 1,
"maxchunksperchild": 0,
"chunk_by": "source"
}
}
for key, value in cls._load_file_configs().items():
if key in _raw_config and isinstance(
_raw_config[key],
dict) and not CobaRegistry.is_known_recipe(value):
if key == "experiment" and "maxtasksperchild" in value:
value["maxchunksperchild"] = value[
"maxtasksperchild"]
del value["maxtasksperchild"]
_raw_config[key].update(value)
else:
_raw_config[key] = value
cls._config_backing = {
'api_keys': _raw_config['api_keys'],
'cacher': CobaRegistry.construct(_raw_config['cacher']),
'logger': CobaRegistry.construct(_raw_config['logger']),
'experiment': ExperimentConfig(**_raw_config['experiment'])
}
except CobaException as e:
messages = [
'',
"ERROR: An error occured while initializing CobaContext. Execution is unable to continue. Please see below for details:",
f" > {e}", ''
]
coba_exit('\n'.join(messages))
except Exception as e:
messages = [
'',
"ERROR: An error occured while initializing CobaContext. Execution is unable to continue. Please see below for details:",
''.join(traceback.format_tb(e.__traceback__)), ''.join(
traceback.TracebackException.from_exception(
e).format_exception_only())
]
coba_exit('\n'.join(messages))
return cls._config_backing
def test_invalid_recipe2(self):
CobaRegistry.register("test", TestObject)
recipe = {"test": [1, 2, 3], "name": "test", "args": [4, 5, 6]}
with self.assertRaises(Exception) as cm:
CobaRegistry.construct(recipe)
self.assertEqual(f"Invalid recipe {str(recipe)}", str(cm.exception))
def test_invalid_recipe3(self):
CobaRegistry.register("test", TestObject)
recipe = {"test": {"a": 1}, "name": "test", "kwargs": {"a": 1}}
with self.assertRaises(Exception) as cm:
CobaRegistry.construct(recipe)
self.assertEqual(f"Invalid recipe {str(recipe)}", str(cm.exception))
def test_registered_create_foreach1(self):
CobaRegistry.register("test", TestObject)
recipe = {"test": [[1, 2, 3]], "kwargs": {"a": 1}, "method": "foreach"}
klasses = CobaRegistry.construct(recipe)
self.assertEqual(len(klasses), 1)
self.assertEqual(klasses[0].args, (1, 2, 3))
self.assertEqual(klasses[0].kwargs, {"a": 1})
def test_registered_create_recursive3(self):
CobaRegistry.register("test", TestObject)
klass = CobaRegistry.construct({"test": {"a": "test"}})
self.assertEqual(klass.args, ())
self.assertEqual(1, len(klass.kwargs))
self.assertIsInstance(klass.kwargs["a"], TestObject)
self.assertEqual(klass.kwargs["a"].args, ())
self.assertEqual(klass.kwargs["a"].kwargs, {})
def test_registered_create_foreach4(self):
CobaRegistry.register("test", TestObject)
recipe = {"test": [[1, 2], 3], "method": "foreach"}
klasses = CobaRegistry.construct(recipe)
self.assertEqual(len(klasses), 2)
self.assertEqual(klasses[0].args, (1, 2))
self.assertEqual(klasses[0].kwargs, {})
self.assertEqual(klasses[1].args, (3, ))
self.assertEqual(klasses[1].kwargs, {})
def filter(self, config: Dict[str,Any]) -> 'Benchmark':
variables = { k: CobaRegistry.construct(v) for k,v in config.get("variables",{}).items() }
def _construct(item:Any) -> Sequence[Any]:
result = None
if isinstance(item, str) and item in variables:
result = variables[item]
if isinstance(item, str) and item not in variables:
result = CobaRegistry.construct(item)
if isinstance(item, dict):
result = CobaRegistry.construct(item)
if isinstance(item, list):
if any([ isinstance(i,list) for i in item ]):
raise Exception("Recursive structures are not supported in benchmark simulation configs.")
pieces = list(map(_construct, item))
result = [ Pipe.join(s, f) for s in pieces[0] for f in product(*pieces[1:])]
if result is None:
raise Exception(f"We were unable to construct {item} in the given benchmark file.")
return result if isinstance(result, collections.Sequence) else [result]
if not isinstance(config['simulations'], list): config['simulations'] = [config['simulations']]
simulations = [ simulation for recipe in config['simulations'] for simulation in _construct(recipe)]
return Benchmark(simulations)
def test_endpoint_loaded_after_decorator_register(self):
@coba_registry_class("MyTestObject")
class MyTestObject(TestObject):
pass
klass = CobaRegistry.retrieve("NoneSink")
self.assertEqual("NoneSink", klass.__name__)
def test_register_decorator(self):
@coba_registry_class("MyTestObject")
class MyTestObject(TestObject):
pass
klass = CobaRegistry.construct("MyTestObject")
self.assertIsInstance(klass, MyTestObject)
self.assertEqual(klass.args, ())
self.assertEqual(klass.kwargs, {})
def _construct(item:Any) -> Sequence[Any]:
result = None
if isinstance(item, str) and item in variables:
result = variables[item]
if isinstance(item, str) and item not in variables:
result = CobaRegistry.construct(item)
if isinstance(item, dict):
result = CobaRegistry.construct(item)
if isinstance(item, list):
if any([ isinstance(i,list) for i in item ]):
raise Exception("Recursive structures are not supported in benchmark simulation configs.")
pieces = list(map(_construct, item))
result = [ Pipe.join(s, f) for s in pieces[0] for f in product(*pieces[1:])]
if result is None:
raise Exception(f"We were unable to construct {item} in the given benchmark file.")
return result if isinstance(result, collections.Sequence) else [result]
def test_registered_create_foreach3(self):
CobaRegistry.register("test", TestObject)
recipe = {
"test": [1, 2],
"kwargs": [{
"a": 1
}, {
"a": 2
}],
"method": "foreach"
}
klasses = CobaRegistry.construct(recipe)
self.assertEqual(len(klasses), 2)
self.assertEqual(klasses[0].args, (1, ))
self.assertEqual(klasses[0].kwargs, {"a": 1})
self.assertEqual(klasses[1].args, (2, ))
self.assertEqual(klasses[1].kwargs, {"a": 2})
def from_file(arg) -> 'Benchmark': #type: ignore
"""Instantiate a Benchmark from a config file."""
if isinstance(arg, str) and arg.startswith('http'):
content = '\n'.join(ResponseToLines().filter(
HttpSource(arg).read()))
elif isinstance(arg, str) and not arg.startswith('http'):
content = '\n'.join(DiskSource(arg).read())
else:
content = arg.read() #type: ignore
return CobaRegistry.construct(CobaConfig.Benchmark['file_fmt']).filter(
JsonDecode().filter(content))
def read(self) -> Sequence[Environment]:
definitions: dict = JsonDecode().filter('\n'.join(self._source.read()))
variables = {
k: CobaRegistry.construct(v)
for k, v in definitions.get("variables", {}).items()
}
def _construct(item: Any) -> Sequence[Any]:
result = None
if isinstance(item, str) and item in variables:
result = variables[item]
if isinstance(item, str) and item not in variables:
result = CobaRegistry.construct(item)
if isinstance(item, dict):
result = CobaRegistry.construct(item)
if isinstance(item, list):
pieces = list(map(_construct, item))
if hasattr(pieces[0][0], 'read'):
result = [
Pipes.join(s, *f) for s in pieces[0]
for f in product(*pieces[1:])
]
else:
result = sum(pieces, [])
if result is None:
raise CobaException(
f"We were unable to construct {item} in the given environment definition file."
)
return result if isinstance(
result, collections.abc.Sequence) else [result]
if not isinstance(definitions['environments'], list):
definitions['environments'] = [definitions['environments']]
return [
environment for recipe in definitions['environments']
for environment in _construct(recipe)
]
def Logger(cls) -> Logger:
if cls._log is None:
cls._log = CobaRegistry.construct(cls._load_config()['logger'])
return cls._log
def Cacher(cls) -> Cacher[str, bytes]:
if cls._cache is None:
cls._cache = CobaRegistry.construct(cls._load_config()['cacher'])
return cls._cache
from coba.registry import CobaRegistry
from coba.pipes import NullSink, ConsoleSink, DiskSink, HttpSource
from coba.environments import OpenmlSimulation, SerializedSimulation, SupervisedSimulation
from coba.environments import Sort, Scale, Cycle, Shuffle, Take, Identity
from coba.contexts import DiskCacher, NullCacher, IndentLogger, NullLogger, BasicLogger
CobaRegistry.register("Null", NullSink)
CobaRegistry.register("Disk", DiskSink)
CobaRegistry.register("Http", HttpSource)
CobaRegistry.register("Console", ConsoleSink)
CobaRegistry.register("BasicLogger", BasicLogger)
CobaRegistry.register("IndentLogger", IndentLogger)
CobaRegistry.register("NullLogger", NullLogger)
CobaRegistry.register("DiskCacher", DiskCacher)
CobaRegistry.register("NullCacher", NullCacher)
CobaRegistry.register("OpenmlSimulation", OpenmlSimulation)
CobaRegistry.register("SerializedSimulation", SerializedSimulation)
CobaRegistry.register("SupervisedSimulation", SupervisedSimulation)
CobaRegistry.register("Identity", Identity)
CobaRegistry.register("Take", Take)
CobaRegistry.register("Shuffle", Shuffle)
CobaRegistry.register("Sort", Sort)
CobaRegistry.register("Scale", Scale)
CobaRegistry.register("Cycle", Cycle)
def test_registered_create_dict_arg(self):
CobaRegistry.register("test", TestArgObject)
with self.assertRaises(Exception):
klass = CobaRegistry.construct({"test": {"a": 1}})
def setUp(self) -> None:
CobaRegistry.register("OpenmlSimulation", OpenmlSimulation)
CobaRegistry.register("Take", Take)
def test_endpoint_loaded(self):
klass = CobaRegistry.retrieve("NoneSink")
self.assertEqual("NoneSink", klass.__name__)
def setUp(self) -> None:
CobaRegistry.clear() #make sure the registry is fresh each test
