def evaluate(self, learners: Sequence[Learner[_C,_A]], transaction_log:str = None, seed:int = None) -> Result:
"""Collect observations of a Learner playing the benchmark's simulations to calculate Results.
Args:
factories: See the base class for more information.
Returns:
See the base class for more information.
"""
benchmark_learners = [ BenchmarkLearner(learner, seed) for learner in learners ] #type: ignore
restored = Result.from_transaction_log(transaction_log)
task_source = TaskSource(self._simulation_pipes, benchmark_learners, restored)
task_to_transactions = TaskToTransactions(self._ignore_raise)
transaction_sink = TransactionSink(transaction_log, restored)
n_given_learners = len(benchmark_learners)
n_given_simulations = len(self._simulation_pipes)
if len(restored.benchmark) != 0:
assert n_given_learners == restored.benchmark['n_learners' ], "The currently evaluating benchmark doesn't match the given transaction log"
assert n_given_simulations == restored.benchmark['n_simulations'], "The currently evaluating benchmark doesn't match the given transaction log"
preamble_transactions = []
preamble_transactions.append(Transaction.version(TransactionPromote.CurrentVersion))
preamble_transactions.append(Transaction.benchmark(n_given_learners, n_given_simulations))
preamble_transactions.extend(Transaction.learners(benchmark_learners))
mp = self._processes if self._processes else ExecutionContext.Config.processes
mt = self._maxtasksperchild if self._maxtasksperchild else ExecutionContext.Config.maxtasksperchild
Pipe.join(MemorySource(preamble_transactions), [] , transaction_sink).run(1,None)
Pipe.join(task_source , [task_to_transactions], transaction_sink).run(mp,mt)
return transaction_sink.result
class LambdaSource(Source[Tuple[Sequence[Interaction[_C_out, _A_out]],
Sequence[Sequence[Reward]]]]):
def __init__(self,
n_interactions: int,
context: Callable[[int], _C_out],
action_set: Callable[[int], Sequence[_A_out]],
reward: Callable[[_C_out, _A_out], Reward],
seed: int = None) -> None:
coba.random.seed(seed)
interactions: List[Interaction[_C_out, _A_out]] = []
reward_sets: List[Sequence[Reward]] = []
for i in range(n_interactions):
_context = context(i)
_action_set = action_set(i)
_reward_set = [
reward(_context, _action) for _action in _action_set
]
interactions.append(Interaction(_context, _action_set,
i)) #type: ignore
reward_sets.append(_reward_set)
self._source = MemorySource((interactions, reward_sets))
def read(
self
) -> Tuple[Sequence[Interaction[_C_out, _A_out]],
Sequence[Sequence[Reward]]]:
return self._source.read()
def test_multiprocess_singletask(self):
source = MemorySource(list(range(4)))
sink = MemorySink()
Pipe.join(source, [Pipe_Tests.ProcessNameFilter()], sink).run(2, 1)
self.assertEqual(len(set(sink.items)), 4)
def test_single_process_multitask(self):
source = MemorySource(list(range(10)))
sink = MemorySink()
Pipe.join(source, [Pipe_Tests.ProcessNameFilter()], sink).run()
self.assertEqual(sink.items, ['MainProcess'] * 10)
def test_logging(self):
actual_logs = []
ExecutionContext.Logger = UniversalLogger(
lambda msg, end: actual_logs.append((msg, end)))
source = MemorySource(list(range(4)))
sink = MemorySink()
Pipe.join(source, [Pipe_Tests.ProcessNameFilter()], sink).run(2, 1)
self.assertEqual(len(actual_logs), 4)
self.assertEqual(sink.items, [l[0][20:] for l in actual_logs])
def test_exception_singleprocess(self):
source = MemorySource(list(range(4)))
sink = MemorySink()
with self.assertRaises(Exception):
Pipe.join(source, [Pipe_Tests.ExceptionFilter()], sink).run()