def test_take4(self):
interactions = list(
map(Interaction, repeat(1), repeat([1, 2]),
[[3, 3], [4, 4], [5, 5]]))
take_interactions = list(Take(4).filter(interactions))
self.assertEqual(3, len(interactions))
self.assertEqual(0, len(take_interactions))
def __init__(self,*args, **kwargs) -> None:
"""Instantiate a UniversalBenchmark.
Args:
simulations: The sequence of simulations to benchmark against.
batcher: How each simulation is broken into evaluation batches.
ignore_raise: Should exceptions be raised or logged during evaluation.
shuffle_seeds: A sequence of seeds for interaction shuffling. None means no shuffle.
processes: The number of process to spawn during evalution (overrides coba config).
maxtasksperchild: The number of tasks each process will perform before a refresh.
See the overloads for more information.
"""
simulations = cast(Sequence[Source[Simulation[_C,_A]]], args[0])
shufflers = [ Shuffle(seed) for seed in kwargs.get('seeds', [None]) ]
taker = Take(kwargs.get('take', None))
if 'batch_count' in kwargs:
batcher = Batch(count=kwargs['batch_count'])
elif 'batch_sizes' in kwargs:
batcher = Batch(sizes=kwargs['batch_sizes'])
else:
batcher = Batch(size=kwargs.get('batch_size',1))
pipes: List[BenchmarkSimulation] = []
for source_id, source in enumerate(simulations):
for shuffler in shufflers:
source_description = str(source_id)
filters_description = []
if shuffler._seed is not None:
filters_description.append(f'{{"Shuffle":{shuffler._seed}}}')
if taker._count is not None:
filters_description.append(f'{{"Take":{taker._count}}}')
if batcher._size != 1:
filters_description.append(f'{{"Batch":{[batcher._size,batcher._count,batcher._sizes]}}}')
filters: List[Union[Filter[Simulation, Simulation],Filter[Simulation,BatchedSimulation]]] = [shuffler, taker, batcher]
pipes.append(BenchmarkSimulation(source, filters, source_description, filters_description))
self._simulation_pipes = cast(Sequence[Source[BatchedSimulation[Context,Action]]], pipes)
self._ignore_raise = cast(bool ,kwargs.get('ignore_raise', True))
self._processes = cast(Optional[int] ,kwargs.get('processes', None))
self._maxtasksperchild = cast(Optional[int] ,kwargs.get('maxtasksperchild', None))
def test_take2(self):
interactions = list(
map(Interaction, repeat(1), repeat([1, 2]),
[[3, 3], [4, 4], [5, 5]]))
take_interactions = list(Take(2).filter(interactions))
self.assertEqual(2, len(take_interactions))
self.assertEqual(interactions[0], take_interactions[0])
self.assertEqual(interactions[1], take_interactions[1])
self.assertEqual(3, len(interactions))
self.assertEqual(interactions[0], interactions[0])
self.assertEqual(interactions[1], interactions[1])
self.assertEqual(interactions[2], interactions[2])
def test_take4(self):
interactions = [
Interaction(1,[1,2],0),
Interaction(1,[1,2],1),
Interaction(1,[1,2],2)
]
rewards = [
[3,3],
[4,4],
[5,5]
]
simulation = MemorySimulation(interactions,rewards)
take_simulation = Take(4).filter(simulation)
self.assertEqual(3, len(simulation.interactions))
self.assertEqual(0, len(take_simulation.interactions))
def test_take1(self):
interactions = [
Interaction(1,[1,2],0),
Interaction(1,[1,2],1),
Interaction(1,[1,2],2)
]
rewards = [
[3,3],
[4,4],
[5,5]
]
simulation = MemorySimulation(interactions,rewards)
take_simulation = Take(1).filter(simulation)
self.assertEqual(1, len(take_simulation.interactions))
self.assertEqual(0, take_simulation.interactions[0].key)
self.assertEqual(3, len(simulation.interactions))
self.assertEqual(0, simulation.interactions[0].key)
self.assertEqual(1, simulation.interactions[1].key)
self.assertEqual(2, simulation.interactions[2].key)
def __init__(self,
simulations: Sequence[Simulation],
shuffle: Sequence[Optional[int]] = [None],
take: int = None) -> None:
"""Instantiate a Benchmark.
Args:
simulations: The collection of simulations to benchmark against.
shuffle: A collection of seeds to use for simulation shuffling. A seed of `None` means no shuffle will be applied.
take: The number of interactions to take from each simulation for evaluation.
"""
...
sources: List[Simulation] = simulations
filters: List[Sequence[Filter[Iterable[Interaction],
Iterable[Interaction]]]] = []
if shuffle != [None]:
filters.append([Shuffle(seed) for seed in shuffle])
if take is not None:
filters.append([Take(take)])
if len(filters) > 0:
simulation_sources = [
cast(Source[Simulation], Pipe.join(s, f))
for s, f in product(sources, product(*filters))
]
else:
simulation_sources = list(sources)
self._simulations: Sequence[Source[Simulation]] = simulation_sources
self._processes: Optional[int] = None
self._maxtasksperchild: Optional[int] = None
self._maxtasksperchild_set: bool = False
self._chunk_by: Optional[str] = None
def test_repr(self):
self.assertEqual('{"Take":2}', str(Take(2)))
self.assertEqual('{"Take":null}', str(Take(None)))