def test_take4(self): interactions = list( map(Interaction, repeat(1), repeat([1, 2]), [[3, 3], [4, 4], [5, 5]])) take_interactions = list(Take(4).filter(interactions)) self.assertEqual(3, len(interactions)) self.assertEqual(0, len(take_interactions))
def __init__(self,*args, **kwargs) -> None: """Instantiate a UniversalBenchmark. Args: simulations: The sequence of simulations to benchmark against. batcher: How each simulation is broken into evaluation batches. ignore_raise: Should exceptions be raised or logged during evaluation. shuffle_seeds: A sequence of seeds for interaction shuffling. None means no shuffle. processes: The number of process to spawn during evalution (overrides coba config). maxtasksperchild: The number of tasks each process will perform before a refresh. See the overloads for more information. """ simulations = cast(Sequence[Source[Simulation[_C,_A]]], args[0]) shufflers = [ Shuffle(seed) for seed in kwargs.get('seeds', [None]) ] taker = Take(kwargs.get('take', None)) if 'batch_count' in kwargs: batcher = Batch(count=kwargs['batch_count']) elif 'batch_sizes' in kwargs: batcher = Batch(sizes=kwargs['batch_sizes']) else: batcher = Batch(size=kwargs.get('batch_size',1)) pipes: List[BenchmarkSimulation] = [] for source_id, source in enumerate(simulations): for shuffler in shufflers: source_description = str(source_id) filters_description = [] if shuffler._seed is not None: filters_description.append(f'{{"Shuffle":{shuffler._seed}}}') if taker._count is not None: filters_description.append(f'{{"Take":{taker._count}}}') if batcher._size != 1: filters_description.append(f'{{"Batch":{[batcher._size,batcher._count,batcher._sizes]}}}') filters: List[Union[Filter[Simulation, Simulation],Filter[Simulation,BatchedSimulation]]] = [shuffler, taker, batcher] pipes.append(BenchmarkSimulation(source, filters, source_description, filters_description)) self._simulation_pipes = cast(Sequence[Source[BatchedSimulation[Context,Action]]], pipes) self._ignore_raise = cast(bool ,kwargs.get('ignore_raise', True)) self._processes = cast(Optional[int] ,kwargs.get('processes', None)) self._maxtasksperchild = cast(Optional[int] ,kwargs.get('maxtasksperchild', None))
def test_take2(self): interactions = list( map(Interaction, repeat(1), repeat([1, 2]), [[3, 3], [4, 4], [5, 5]])) take_interactions = list(Take(2).filter(interactions)) self.assertEqual(2, len(take_interactions)) self.assertEqual(interactions[0], take_interactions[0]) self.assertEqual(interactions[1], take_interactions[1]) self.assertEqual(3, len(interactions)) self.assertEqual(interactions[0], interactions[0]) self.assertEqual(interactions[1], interactions[1]) self.assertEqual(interactions[2], interactions[2])
def test_take4(self): interactions = [ Interaction(1,[1,2],0), Interaction(1,[1,2],1), Interaction(1,[1,2],2) ] rewards = [ [3,3], [4,4], [5,5] ] simulation = MemorySimulation(interactions,rewards) take_simulation = Take(4).filter(simulation) self.assertEqual(3, len(simulation.interactions)) self.assertEqual(0, len(take_simulation.interactions))
def test_take1(self): interactions = [ Interaction(1,[1,2],0), Interaction(1,[1,2],1), Interaction(1,[1,2],2) ] rewards = [ [3,3], [4,4], [5,5] ] simulation = MemorySimulation(interactions,rewards) take_simulation = Take(1).filter(simulation) self.assertEqual(1, len(take_simulation.interactions)) self.assertEqual(0, take_simulation.interactions[0].key) self.assertEqual(3, len(simulation.interactions)) self.assertEqual(0, simulation.interactions[0].key) self.assertEqual(1, simulation.interactions[1].key) self.assertEqual(2, simulation.interactions[2].key)
def __init__(self, simulations: Sequence[Simulation], shuffle: Sequence[Optional[int]] = [None], take: int = None) -> None: """Instantiate a Benchmark. Args: simulations: The collection of simulations to benchmark against. shuffle: A collection of seeds to use for simulation shuffling. A seed of `None` means no shuffle will be applied. take: The number of interactions to take from each simulation for evaluation. """ ... sources: List[Simulation] = simulations filters: List[Sequence[Filter[Iterable[Interaction], Iterable[Interaction]]]] = [] if shuffle != [None]: filters.append([Shuffle(seed) for seed in shuffle]) if take is not None: filters.append([Take(take)]) if len(filters) > 0: simulation_sources = [ cast(Source[Simulation], Pipe.join(s, f)) for s, f in product(sources, product(*filters)) ] else: simulation_sources = list(sources) self._simulations: Sequence[Source[Simulation]] = simulation_sources self._processes: Optional[int] = None self._maxtasksperchild: Optional[int] = None self._maxtasksperchild_set: bool = False self._chunk_by: Optional[str] = None
def test_repr(self): self.assertEqual('{"Take":2}', str(Take(2))) self.assertEqual('{"Take":null}', str(Take(None)))