def test_pre_post_decorators(self):
pre_decorators = [LogDecorator(1)]
post_decorators = [LogDecorator(2)]
sink = ListSink()
logger = DecoratedLogger(pre_decorators, BasicLogger(sink),
post_decorators)
logger.log('a')
self.assertEqual(['2 -- 1 -- a'], sink.items)
sink = ListSink()
logger = DecoratedLogger(pre_decorators, BasicLogger(sink),
post_decorators)
with logger.log('a'):
pass
self.assertEqual(['2 -- 1 -- a', '2 -- 1 -- a (completed)'],
sink.items)
sink = ListSink()
logger = DecoratedLogger(pre_decorators, BasicLogger(sink),
post_decorators)
with logger.time('a'):
pass
self.assertEqual(
['2 -- 1 -- a', '2 -- 1 -- a (0.0 seconds) (completed)'],
sink.items)
def test_run(self):
source = ListSource(list(range(10)))
sink = ListSink()
Pipes.join(source, ProcessNameFilter(), sink).run()
self.assertEqual(sink.items[0], ['MainProcess']*10)
def test_time_with_1(self):
#This test is somewhat time dependent.
#I don't think it should ever fail, but if it does
#try running it again and see if it works that time.
sink = ListSink()
logger = BasicLogger(sink)
logs = sink.items
with logger.time('a'):
logger.log('c')
time.sleep(0.15)
logger.log('d')
logger.log('e')
self.assertEqual(5, len(logs))
self.assertEqual(logs[0], 'a')
self.assertEqual(logs[1], 'c')
self.assertEqual(logs[2], 'd')
self.assertRegex(logs[3],
'^a \\(\\d+\\.\\d+ seconds\\) \\(completed\\)$')
self.assertEqual(logs[4], 'e')
self.assertAlmostEqual(float(logs[3][3:7]), 0.15, 1)
def test_sink(self):
pre_decorators = [LogDecorator(1)]
post_decorators = [LogDecorator(2)]
sink = ListSink()
logger = DecoratedLogger(pre_decorators, BasicLogger(sink),
post_decorators)
self.assertIs(logger.sink, sink)
sink = ListSink()
logger.sink = sink
self.assertIs(logger.sink, sink)
logger.log('a')
self.assertEqual(['2 -- 1 -- a'], sink.items)
def test_items_not_picklable(self):
stderr_sink = ListSink()
list(
Multiprocessor(ProcessNameFilter(), 2, 1,
stderr_sink).filter([NotPicklableFilter()]))
self.assertEqual(1, len(stderr_sink.items))
self.assertIn("pickle", stderr_sink.items[0])
def test_filter_exception(self):
stderr_sink = ListSink()
list(
Multiprocessor(ExceptionFilter(), 2, 1,
stderr_sink).filter(range(4)))
self.assertEqual(len(stderr_sink.items), 4)
for item in stderr_sink.items:
self.assertIn("Exception Filter", str(item))
def test_not_picklable_logging(self):
logger_sink = ListSink()
CobaContext.logger = BasicLogger(logger_sink)
CobaContext.cacher = NullCacher()
list(
CobaMultiprocessor(ProcessNameFilter(), 2,
1).filter([lambda a: 1]))
self.assertEqual(1, len(logger_sink.items))
self.assertIn("pickle", logger_sink.items[0])
def test_filter_exception_logging(self):
CobaContext.logger = DecoratedLogger([ExceptLog()],
BasicLogger(ListSink()), [])
CobaContext.cacher = NullCacher()
list(CobaMultiprocessor(ExceptionFilter(), 2, 1).filter(range(4)))
self.assertEqual(4, len(CobaContext.logger.sink.items))
self.assertIn("Exception Filter", CobaContext.logger.sink.items[0])
self.assertIn("Exception Filter", CobaContext.logger.sink.items[1])
self.assertIn("Exception Filter", CobaContext.logger.sink.items[2])
self.assertIn("Exception Filter", CobaContext.logger.sink.items[3])
def test_wrapped_not_picklable_learner_sans_reduce(self):
sim1 = LambdaSimulation(5, lambda i: i, lambda i, c: [0, 1, 2],
lambda i, c, a: cast(float, a))
learner = WrappedLearner(NotPicklableLearner())
experiment = Experiment([sim1], [learner])
CobaContext.logger = BasicLogger(ListSink())
experiment.evaluate()
self.assertEqual(1, len(CobaContext.logger.sink.items))
self.assertIn("pickle", CobaContext.logger.sink.items[0])
def test_log(self):
sink = ListSink()
logger = BasicLogger(sink)
logs = sink.items
logger.log('a')
logger.log('c')
logger.log('d')
self.assertEqual(logs[0], 'a')
self.assertEqual(logs[1], 'c')
self.assertEqual(logs[2], 'd')
def test_read_exception_logging(self):
CobaContext.logger = DecoratedLogger([ExceptLog()],
BasicLogger(ListSink()), [])
CobaContext.cacher = NullCacher()
def broken_generator():
yield [1]
raise Exception("Generator Exception")
list(CobaMultiprocessor(Identity(), 2, 1).filter(broken_generator()))
self.assertEqual(1, len(CobaContext.logger.sink.items))
self.assertIn("Generator Exception", CobaContext.logger.sink.items[0])
def test_exception_logged_but_not_thrown(self):
log_sink = ListSink()
CobaContext.logger = NullLogger()
CobaContext.cacher = NullCacher()
CobaContext.store = None
logger = DecoratedLogger([ExceptLog()], IndentLogger(), [])
CobaMultiprocessor.ProcessFilter(ExceptionFilter(), logger, None, None,
log_sink).filter(1)
self.assertIn('Exception Filter', log_sink.items[0])
def test_simple(self):
sink = ListSink()
sink.write("a")
sink.write("b")
sink.write("c")
self.assertEqual(["a", "b", "c"], sink.items)
def test_log_with_1(self):
sink = ListSink()
logger = IndentLogger(sink)
logs = sink.items
with logger.log('a'):
logger.log('c')
logger.log('d')
logger.log('e')
self.assertEqual(4, len(logs))
self.assertEqual(logs[0], 'a')
self.assertEqual(logs[1], ' * c')
self.assertEqual(logs[2], ' * d')
self.assertEqual(logs[3], 'e')
def test_logging(self):
logger_sink = ListSink()
logger = DecoratedLogger([], IndentLogger(logger_sink),
[NameLog(), StampLog()])
CobaContext.logger = logger
CobaContext.cacher = NullCacher()
items = list(
CobaMultiprocessor(ProcessNameFilter(), 2, 1).filter(range(4)))
self.assertEqual(len(logger_sink.items), 4)
self.assertCountEqual(items,
[l.split(' ')[3] for l in logger_sink.items])
self.assertCountEqual(items,
[l.split(' ')[-1] for l in logger_sink.items])
def test_sim_write_read_with_params_and_none_context(self):
sink = ListSink()
expected_env = MemorySimulation(
params={'a': 1},
interactions=[SimulatedInteraction(None, [1, 2], rewards=[2, 3])])
SerializedSimulation(expected_env).write(sink)
actual_env = SerializedSimulation(ListSource(sink.items))
self.assertEqual(expected_env.params, actual_env.params)
self.assertEqual(len(list(expected_env.read())),
len(list(actual_env.read())))
for e_interaction, a_interaction in zip(expected_env.read(),
actual_env.read()):
self.assertEqual(e_interaction.context, a_interaction.context)
self.assertEqual(e_interaction.actions, a_interaction.actions)
self.assertEqual(e_interaction.kwargs, a_interaction.kwargs)
def test_log_with_interrupt(self):
sink = ListSink()
logger = BasicLogger(sink)
logs = sink.items
with self.assertRaises(BaseException) as e:
with logger.log('a'):
logger.log('c')
logger.log('d')
raise KeyboardInterrupt()
self.assertEqual(4, len(logs))
self.assertEqual(logs[0], 'a')
self.assertEqual(logs[1], 'c')
self.assertEqual(logs[2], 'd')
self.assertEqual(logs[3], 'a (interrupt)')
def test_log_with_exception(self):
sink = ListSink()
logger = BasicLogger(sink)
logs = sink.items
with self.assertRaises(Exception):
with logger.log('a'):
logger.log('c')
logger.log('d')
raise Exception()
self.assertEqual(4, len(logs))
self.assertEqual(logs[0], 'a')
self.assertEqual(logs[1], 'c')
self.assertEqual(logs[2], 'd')
self.assertEqual(logs[3], 'a (exception)')
def test_log_with(self):
sink = ListSink()
logger = BasicLogger(sink)
logs = sink.items
with logger.log('a'):
logger.log('c')
logger.log('d')
logger.log('e')
self.assertEqual(5, len(logs))
self.assertEqual(logs[0], 'a')
self.assertEqual(logs[1], 'c')
self.assertEqual(logs[2], 'd')
self.assertEqual(logs[3], 'a (completed)')
self.assertEqual(logs[4], 'e')
def test_time_with_interrupt(self):
sink = ListSink()
logger = BasicLogger(sink)
logs = sink.items
with self.assertRaises(BaseException) as e:
with logger.time('a'):
logger.log('c')
logger.log('d')
raise KeyboardInterrupt()
self.assertEqual(4, len(logs))
self.assertEqual(logs[0], 'a')
self.assertEqual(logs[1], 'c')
self.assertEqual(logs[2], 'd')
self.assertRegex(logs[3],
'^a \\(\\d+\\.\\d+ seconds\\) \\(interrupt\\)$')
def test_time_two_separate(self):
#This test is somewhat time dependent.
#I don't think it should ever fail, but if it does
#try running it again and see if it works that time.
sink = ListSink()
logger = BasicLogger(sink)
logs = sink.items
with logger.time('a'):
time.sleep(0.05)
with logger.time('d'):
logger.log('e')
time.sleep(0.05)
logger.log('g')
with logger.time('a'):
time.sleep(0.05)
with logger.time('d'):
logger.log('e')
time.sleep(0.05)
self.assertEqual(11, len(logs))
self.assertEqual(logs[0], 'a')
self.assertEqual(logs[1], 'd')
self.assertEqual(logs[2], 'e')
self.assertRegex(logs[3], 'd \\(\\d+\\.\\d+ seconds\\)')
self.assertRegex(logs[4], 'a \\(\\d+\\.\\d+ seconds\\)')
self.assertEqual(logs[5], 'g')
self.assertEqual(logs[6], 'a')
self.assertEqual(logs[7], 'd')
self.assertEqual(logs[8], 'e')
self.assertRegex(logs[9], 'd \\(\\d+\\.\\d+ seconds\\)')
self.assertRegex(logs[10], 'a \\(\\d+\\.\\d+ seconds\\)')
self.assertAlmostEqual(float(logs[3][3:7]), 0.05, 1)
self.assertAlmostEqual(float(logs[4][3:7]), 0.10, 1)
self.assertAlmostEqual(float(logs[9][3:7]), 0.05, 1)
self.assertAlmostEqual(float(logs[10][3:7]), 0.10, 1)
def test_time_with_exception(self):
sink = ListSink()
logger = IndentLogger(sink)
logs = sink.items
try:
with self.assertRaises(Exception) as e:
with logger.time('a'):
logger.log('c')
logger.log('d')
raise Exception()
except:
pass
self.assertEqual(3, len(logs))
self.assertRegex(logs[0],
'^a \\(\\d+\\.\\d+ seconds\\) \\(exception\\)$')
self.assertEqual(logs[1], ' * c')
self.assertEqual(logs[2], ' * d')
def test_class_definitions_not_loaded_in_main(self):
stderr_sink = ListSink()
global Test
class Test:
pass
def test_function():
list(
Multiprocessor(ProcessNameFilter(), 2, 1,
stderr_sink).filter([Test()] * 2))
t = Thread(target=test_function)
t.start()
t.join(5)
self.assertFalse(t.is_alive())
self.assertIn("unable to find", stderr_sink.items[0])
def test_coba_config_set_correctly(self):
log_sink = ListSink()
CobaContext.logger = NullLogger()
CobaContext.cacher = NullCacher()
CobaContext.store = None
filter = CobaMultiprocessor.ProcessFilter(Identity(), IndentLogger(),
MemoryCacher(), {}, log_sink)
self.assertIsInstance(CobaContext.logger, NullLogger)
self.assertIsInstance(CobaContext.cacher, NullCacher)
self.assertIsNone(CobaContext.store)
filter.filter(1)
self.assertIsInstance(CobaContext.logger, IndentLogger)
self.assertIsInstance(CobaContext.cacher, MemoryCacher)
self.assertIsInstance(CobaContext.store, dict)
self.assertIsInstance(CobaContext.logger.sink, ListSink)
def test_time_with_3(self):
#This test is somewhat time dependent.
#I don't think it should ever fail, but if it does
#try running it again and see if it works that time.
sink = ListSink()
logger = IndentLogger(sink)
logs = sink.items
with logger.time('a'):
with logger.log('c'):
time.sleep(0.05)
with logger.time('d'):
logger.log('e')
time.sleep(0.05)
with logger.time('d'):
logger.log('e')
time.sleep(0.05)
logger.log('f')
logger.log('g')
self.assertEqual(8, len(logs))
self.assertRegex(logs[0],
'^a \\(\\d+\\.\\d+ seconds\\) \\(completed\\)')
self.assertEqual(logs[1], ' * c')
self.assertRegex(
logs[2], '^ > d \\(\\d+\\.\\d+ seconds\\) \\(completed\\)$')
self.assertEqual(logs[3], ' - e')
self.assertRegex(
logs[4], '^ > d \\(\\d+\\.\\d+ seconds\\) \\(completed\\)$')
self.assertEqual(logs[5], ' - e')
self.assertEqual(logs[6], ' * f')
self.assertEqual(logs[7], 'g')
self.assertAlmostEqual(float(logs[0][3:7]), 0.15, 1)
self.assertAlmostEqual(float(logs[2][9:13]), 0.05, 1)
self.assertAlmostEqual(float(logs[4][9:13]), 0.05, 1)
def test_ignore_raise(self):
CobaContext.logger = IndentLogger(ListSink())
sim1 = LambdaSimulation(2, lambda i: i, lambda i, c: [0, 1, 2],
lambda i, c, a: cast(float, a))
sim2 = LambdaSimulation(3, lambda i: i, lambda i, c: [3, 4, 5],
lambda i, c, a: cast(float, a))
experiment = Experiment(
[sim1, sim2], [ModuloLearner(), BrokenLearner()],
evaluation_task=OnlineOnPolicyEvalTask(False))
result = experiment.evaluate()
actual_learners = result.learners.to_tuples()
actual_environments = result.environments.to_tuples()
actual_interactions = result.interactions.to_tuples()
expected_learners = [(0, "Modulo", "Modulo(p=0)", '0'),
(1, "Broken", "Broken", float('nan'))]
expected_environments = [(0, 'LambdaSimulation'),
(1, 'LambdaSimulation')]
expected_interactions = [(0, 0, 1, 0), (0, 0, 2, 1), (1, 0, 1, 3),
(1, 0, 2, 4), (1, 0, 3, 5)]
self.assertIsInstance(CobaContext.logger, IndentLogger)
self.assertEqual(
2,
sum([
int("Unexpected exception:" in item)
for item in CobaContext.logger.sink.items
]))
self.assertCountEqual(actual_learners[0], expected_learners[0])
self.assertCountEqual(actual_learners[1][:3], expected_learners[1][:3])
self.assertTrue(math.isnan(expected_learners[1][3]))
self.assertCountEqual(actual_environments, expected_environments)
self.assertCountEqual(actual_interactions, expected_interactions)
def test_time_with_interrupt(self):
#This test is somewhat time dependent.
#I don't think it should ever fail, but if it does
#try running it again and see if it works that time.
sink = ListSink()
logger = IndentLogger(sink)
logs = sink.items
try:
with self.assertRaises(Exception) as e:
with logger.time('a'):
logger.log('c')
logger.log('d')
raise KeyboardInterrupt()
except:
pass
self.assertEqual(3, len(logs))
self.assertRegex(logs[0],
'^a \\(\\d+\\.\\d+ seconds\\) \\(interrupt\\)$')
self.assertEqual(logs[1], ' * c')
self.assertEqual(logs[2], ' * d')
def __init__(self, log_file: Optional[str] = None, minify:bool=True) -> None:
self._log_file = log_file
self._minify = minify
self._source = DiskSource(log_file) if log_file else ListSource()
self._sink = DiskSink(log_file) if log_file else ListSink(self._source.items)
def test_exception(self):
with self.assertRaises(Exception):
Pipes.join(ListSource(list(range(4))), ExceptionFilter(), ListSink()).run()
def test_sink_is_set(self):
logger = IndentLogger(ListSink())
self.assertIsInstance(logger.sink, ListSink)
logger.sink = NullSink()
self.assertIsInstance(logger.sink, NullSink)