def evaluate(self, learners: Sequence[Learner[_C,_A]], transaction_log:str = None, seed:int = None) -> Result: """Collect observations of a Learner playing the benchmark's simulations to calculate Results. Args: factories: See the base class for more information. Returns: See the base class for more information. """ benchmark_learners = [ BenchmarkLearner(learner, seed) for learner in learners ] #type: ignore restored = Result.from_transaction_log(transaction_log) task_source = TaskSource(self._simulation_pipes, benchmark_learners, restored) task_to_transactions = TaskToTransactions(self._ignore_raise) transaction_sink = TransactionSink(transaction_log, restored) n_given_learners = len(benchmark_learners) n_given_simulations = len(self._simulation_pipes) if len(restored.benchmark) != 0: assert n_given_learners == restored.benchmark['n_learners' ], "The currently evaluating benchmark doesn't match the given transaction log" assert n_given_simulations == restored.benchmark['n_simulations'], "The currently evaluating benchmark doesn't match the given transaction log" preamble_transactions = [] preamble_transactions.append(Transaction.version(TransactionPromote.CurrentVersion)) preamble_transactions.append(Transaction.benchmark(n_given_learners, n_given_simulations)) preamble_transactions.extend(Transaction.learners(benchmark_learners)) mp = self._processes if self._processes else ExecutionContext.Config.processes mt = self._maxtasksperchild if self._maxtasksperchild else ExecutionContext.Config.maxtasksperchild Pipe.join(MemorySource(preamble_transactions), [] , transaction_sink).run(1,None) Pipe.join(task_source , [task_to_transactions], transaction_sink).run(mp,mt) return transaction_sink.result
class LambdaSource(Source[Tuple[Sequence[Interaction[_C_out, _A_out]], Sequence[Sequence[Reward]]]]): def __init__(self, n_interactions: int, context: Callable[[int], _C_out], action_set: Callable[[int], Sequence[_A_out]], reward: Callable[[_C_out, _A_out], Reward], seed: int = None) -> None: coba.random.seed(seed) interactions: List[Interaction[_C_out, _A_out]] = [] reward_sets: List[Sequence[Reward]] = [] for i in range(n_interactions): _context = context(i) _action_set = action_set(i) _reward_set = [ reward(_context, _action) for _action in _action_set ] interactions.append(Interaction(_context, _action_set, i)) #type: ignore reward_sets.append(_reward_set) self._source = MemorySource((interactions, reward_sets)) def read( self ) -> Tuple[Sequence[Interaction[_C_out, _A_out]], Sequence[Sequence[Reward]]]: return self._source.read()
def test_multiprocess_singletask(self): source = MemorySource(list(range(4))) sink = MemorySink() Pipe.join(source, [Pipe_Tests.ProcessNameFilter()], sink).run(2, 1) self.assertEqual(len(set(sink.items)), 4)
def test_single_process_multitask(self): source = MemorySource(list(range(10))) sink = MemorySink() Pipe.join(source, [Pipe_Tests.ProcessNameFilter()], sink).run() self.assertEqual(sink.items, ['MainProcess'] * 10)
def test_logging(self): actual_logs = [] ExecutionContext.Logger = UniversalLogger( lambda msg, end: actual_logs.append((msg, end))) source = MemorySource(list(range(4))) sink = MemorySink() Pipe.join(source, [Pipe_Tests.ProcessNameFilter()], sink).run(2, 1) self.assertEqual(len(actual_logs), 4) self.assertEqual(sink.items, [l[0][20:] for l in actual_logs])
def test_exception_singleprocess(self): source = MemorySource(list(range(4))) sink = MemorySink() with self.assertRaises(Exception): Pipe.join(source, [Pipe_Tests.ExceptionFilter()], sink).run()