def test_wrapped_not_picklable_learner_with_reduce(self):
sim1 = LambdaSimulation(5, lambda i: i, lambda i, c: [0, 1, 2],
lambda i, c, a: cast(float, a))
learner = WrappedLearner(NotPicklableLearnerWithReduce())
benchmark = Benchmark([sim1], shuffle=[1, 4])
benchmark.evaluate([learner])
def test_transaction_resume_1(self):
sim = LambdaSimulation(2, lambda i: i, lambda i, c: [0, 1, 2],
lambda i, c, a: cast(float, a))
working_learner = ModuloLearner()
broken_learner = BrokenLearner()
benchmark = Benchmark([sim])
#the second time the broken_factory() shouldn't ever be used for learning or choosing
#because it already worked the first time and we are "resuming" benchmark from transaction.log
try:
first_result = benchmark.evaluate(
[working_learner], "coba/tests/.temp/transactions.log")
second_result = benchmark.evaluate(
[broken_learner], "coba/tests/.temp/transactions.log")
actual_learners = second_result.learners.to_tuples()
actual_simulations = second_result.simulations.to_tuples()
actual_interactions = second_result.interactions.to_tuples()
expected_learners = [(0, "Modulo(p=0)", "Modulo", '0')]
expected_simulations = [(0, "LambdaSimulation", "None", "None",
'"LambdaSimulation"')]
expected_interactions = [(0, 0, 1, 0), (0, 0, 2, 1)]
except Exception as e:
raise
finally:
if Path('coba/tests/.temp/transactions.log').exists():
Path('coba/tests/.temp/transactions.log').unlink()
self.assertCountEqual(actual_learners, expected_learners)
self.assertCountEqual(actual_simulations, expected_simulations)
self.assertCountEqual(actual_interactions, expected_interactions)
def test_not_picklable_learner(self):
sim1 = LambdaSimulation(5, lambda t: t, lambda t: [0,1,2], lambda c,a: a)
learner = NotPicklableLearner()
benchmark = Benchmark([sim1], batch_sizes=[2], ignore_raise=False, seeds=[1,4])
with self.assertRaises(Exception) as cm:
benchmark.evaluate([learner])
self.assertTrue("Learners are required to be picklable" in str(cm.exception))
def test_wrapped_not_picklable_learner_sans_reduce(self):
sim1 = LambdaSimulation(5, lambda i: i, lambda i, c: [0, 1, 2],
lambda i, c, a: cast(float, a))
learner = WrappedLearner(NotPicklableLearner())
benchmark = Benchmark([sim1])
CobaConfig.Logger = BasicLogger(MemorySink())
benchmark.evaluate([learner])
self.assertEqual(1, len(CobaConfig.Logger.sink.items))
self.assertIn("pickle", CobaConfig.Logger.sink.items[0])
def test_seeds(self):
sim1 = LambdaSimulation(5, lambda t: t, lambda t: [0,1,2], lambda c,a: a)
learner = ModuloLearner()
benchmark = Benchmark([sim1], batch_sizes=[2], ignore_raise=False, seeds=[1,4])
actual_learners,actual_simulations,actual_batches = benchmark.evaluate([learner]).to_tuples()
expected_learners = [(0,"0","0")]
expected_simulations = [(0, '0', ['{"Shuffle":1}', '{"Batch":[None, None, [2]]}'], 2, 1, 1, 3), (1, '0', ['{"Shuffle":4}', '{"Batch":[None, None, [2]]}'], 2, 1, 1, 3)]
expected_batches = [(0, 0, [2], [mean([1,0])]), (1, 0, [2], [mean([2,0])])]
self.assertCountEqual(actual_learners, expected_learners)
self.assertCountEqual(actual_simulations, expected_simulations)
self.assertCountEqual(actual_batches, expected_batches)
def test_take(self):
sim1 = LambdaSimulation(5, lambda t: t, lambda t: [0,1,2], lambda c,a: a)
sim2 = LambdaSimulation(4, lambda t: t, lambda t: [3,4,5], lambda c,a: a)
learner = ModuloLearner()
benchmark = Benchmark([sim1,sim2], batch_count=1, take=5, ignore_raise=False)
actual_learners,actual_simulations,actual_batches = benchmark.evaluate([learner]).to_tuples()
expected_learners = [(0,"0","0")]
expected_simulations = [(0, '0', ['{"Take":5}', '{"Batch":[None, 1, None]}'], 5, 1, 1, 3), (1, '1', ['{"Take":5}', '{"Batch":[None, 1, None]}'], 0, 0, 0, 0)]
expected_batches = [(0, 0, [5], [mean([0,1,2,0,1])])]
self.assertCountEqual(actual_learners, expected_learners)
self.assertCountEqual(actual_simulations, expected_simulations)
self.assertCountEqual(actual_batches, expected_batches)
def test_learners(self):
sim = LambdaSimulation(5, lambda t: t, lambda t: [0,1,2], lambda c,a: a)
learner1 = ModuloLearner("0") #type: ignore
learner2 = ModuloLearner("1") #type: ignore
benchmark = Benchmark([sim], batch_count=1, ignore_raise=False)
actual_results = benchmark.evaluate([learner1, learner2])
actual_learners,actual_simulations,actual_batches = actual_results.to_tuples()
expected_learners = [(0,"0","0"), (1,"1","1")]
expected_simulations = [(0, '0', ['{"Batch":[None, 1, None]}'], 5, 1, 1, 3)]
expected_batches = [(0, 0, [5], [mean([0,1,2,0,1])]), (0, 1, [5], [mean([0,1,2,0,1])]) ]
self.assertCountEqual(actual_learners, expected_learners)
self.assertCountEqual(actual_simulations, expected_simulations)
self.assertCountEqual(actual_batches, expected_batches)
def test_sources(self):
sim1 = LambdaSimulation(2, lambda i: i, lambda i, c: [0, 1, 2],
lambda i, c, a: cast(float, a))
learner = ModuloLearner()
benchmark = Benchmark([sim1])
result = benchmark.evaluate([learner])
actual_learners = result.learners.to_tuples()
actual_simulations = result.simulations.to_tuples()
actual_interactions = result.interactions.to_tuples()
expected_learners = [(0, "Modulo(p=0)", "Modulo", '0')]
expected_simulations = [(0, "LambdaSimulation", "None", "None",
'"LambdaSimulation"')]
expected_interactions = [(0, 0, 1, 0), (0, 0, 2, 1)]
self.assertCountEqual(actual_learners, expected_learners)
self.assertCountEqual(actual_simulations, expected_simulations)
self.assertCountEqual(actual_interactions, expected_interactions)
def test_from_json(self):
json = """{
"batches" : {"count":1},
"ignore_first": false,
"shuffle" : [1283],
"simulations" : [
{"type":"classification","from":{"format":"openml","id":1116}}
]
}"""
benchmark = Benchmark.from_json(json)
self.assertEqual(1, len(benchmark._simulation_pipes))
def test_learners(self):
sim = LambdaSimulation(2, lambda i: i, lambda i, c: [0, 1, 2],
lambda i, c, a: cast(float, a))
learner1 = ModuloLearner("0") #type: ignore
learner2 = ModuloLearner("1") #type: ignore
benchmark = Benchmark([sim])
actual_result = benchmark.evaluate([learner1, learner2])
actual_learners = actual_result._learners.to_tuples()
actual_simulations = actual_result._simulations.to_tuples()
actual_interactions = actual_result._interactions.to_tuples()
expected_learners = [(0, "Modulo(p=0)", "Modulo", '0'),
(1, "Modulo(p=1)", "Modulo", '1')]
expected_simulations = [(0, "LambdaSimulation", "None", "None",
'"LambdaSimulation"')]
expected_interactions = [(0, 0, 1, 0), (0, 0, 2, 1), (0, 1, 1, 0),
(0, 1, 2, 1)]
self.assertCountEqual(actual_learners, expected_learners)
self.assertCountEqual(actual_simulations, expected_simulations)
self.assertCountEqual(actual_interactions, expected_interactions)
def test_eval_seeds(self):
sim1 = LambdaSimulation(3, lambda i: i, lambda i, c: [0, 1, 2],
lambda i, c, a: cast(float, a))
learner = RandomLearner()
benchmark = Benchmark([sim1], shuffle=[1, 4])
result = benchmark.evaluate([learner], seed=1)
actual_learners = result.learners.to_tuples()
actual_simulations = result.simulations.to_tuples()
actual_interactions = result.interactions.to_tuples()
expected_learners = [(0, "random", "random")]
expected_simulations = [(0, "LambdaSimulation", "1", "None",
'"LambdaSimulation",{"Shuffle":1}'),
(1, "LambdaSimulation", "4", "None",
'"LambdaSimulation",{"Shuffle":4}')]
expected_interactions = [(0, 0, 1, 0), (0, 0, 2, 2), (0, 0, 3, 1),
(1, 0, 1, 0), (1, 0, 2, 2), (1, 0, 3, 1)]
self.assertCountEqual(actual_learners, expected_learners)
self.assertCountEqual(actual_simulations, expected_simulations)
self.assertCountEqual(actual_interactions, expected_interactions)
def test_ignore_raise(self):
log_sink = MemorySink()
CobaConfig.Logger = IndentLogger(log_sink)
sim1 = LambdaSimulation(2, lambda i: i, lambda i, c: [0, 1, 2],
lambda i, c, a: cast(float, a))
sim2 = LambdaSimulation(3, lambda i: i, lambda i, c: [3, 4, 5],
lambda i, c, a: cast(float, a))
learners = [ModuloLearner(), BrokenLearner()]
benchmark = Benchmark([sim1, sim2])
result = benchmark.evaluate(learners)
actual_learners = result.learners.to_tuples()
actual_simulations = result.simulations.to_tuples()
actual_interactions = result.interactions.to_tuples()
expected_learners = [(0, "Modulo(p=0)", "Modulo", '0'),
(1, "Broken", "Broken", float('nan'))]
expected_simulations = [
(0, "LambdaSimulation", "None", "None", '"LambdaSimulation"'),
(1, "LambdaSimulation", "None", "None", '"LambdaSimulation"')
]
expected_interactions = [(0, 0, 1, 0), (0, 0, 2, 1), (1, 0, 1, 3),
(1, 0, 2, 4), (1, 0, 3, 5)]
self.assertEqual(
2,
sum([
int("Unexpected exception:" in item) for item in log_sink.items
]))
self.assertCountEqual(actual_learners[0], expected_learners[0])
self.assertCountEqual(actual_learners[1][:3], expected_learners[1][:3])
self.assertTrue(math.isnan(expected_learners[1][3]))
self.assertCountEqual(actual_simulations, expected_simulations)
self.assertCountEqual(actual_interactions, expected_interactions)
def test_transaction_resume_1(self):
sim = LambdaSimulation(5, lambda t: t, lambda t: [0,1,2], lambda c,a: a)
working_learner = ModuloLearner()
broken_learner = BrokenLearner()
benchmark = Benchmark([sim], batch_count=1)
#the second time the broken_factory() shouldn't ever be used for learning or choosing
#because it already worked the first time and we are "resuming" benchmark from transaction.log
try:
first_results = benchmark.evaluate([working_learner], "coba/tests/.temp/transactions.log")
second_results = benchmark.evaluate([broken_learner], "coba/tests/.temp/transactions.log")
actual_learners,actual_simulations,actual_batches = second_results.to_tuples()
expected_learners = [(0,"0","0")]
expected_simulations = [(0, '0', ['{"Batch":[None, 1, None]}'], 5, 1, 1, 3)]
expected_batches = [(0, 0, [5], [mean([0,1,2,0,1])])]
finally:
if Path('coba/tests/.temp/transactions.log').exists(): Path('coba/tests/.temp/transactions.log').unlink()
self.assertCountEqual(actual_learners, expected_learners)
self.assertCountEqual(actual_simulations, expected_simulations)
self.assertCountEqual(actual_batches, expected_batches)
def test_take(self):
sim1 = LambdaSimulation(5, lambda i: i, lambda i, c: [0, 1, 2],
lambda i, c, a: cast(float, a))
sim2 = LambdaSimulation(2, lambda i: i, lambda i, c: [3, 4, 5],
lambda i, c, a: cast(float, a))
learner = ModuloLearner()
benchmark = Benchmark([sim1, sim2], take=3)
result = benchmark.evaluate([learner])
actual_learners = result.learners.to_tuples()
actual_simulations = result.simulations.to_tuples()
actual_interactions = result.interactions.to_tuples()
expected_learners = [(0, "Modulo(p=0)", "Modulo", '0')]
expected_simulations = [(0, "LambdaSimulation", "None", "3",
'"LambdaSimulation",{"Take":3}'),
(1, "LambdaSimulation", "None", "3",
'"LambdaSimulation",{"Take":3}')]
expected_interactions = [(0, 0, 1, 0), (0, 0, 2, 1), (0, 0, 3, 2)]
self.assertCountEqual(actual_learners, expected_learners)
self.assertCountEqual(actual_simulations, expected_simulations)
self.assertCountEqual(actual_interactions, expected_interactions)
from batch import Batched
from coba.benchmarks import Benchmark
import re
def baseLearner():
from coba.learners import VowpalLearner
return VowpalLearner(seed=10, epsilon=0.1, flags='--coin')
learner_factories = [
Batched(delay=8, batchsize=1, learner=baseLearner),
Batched(delay=8, batchsize=2, learner=baseLearner),
Batched(delay=8, batchsize=4, learner=baseLearner),
Batched(delay=8, batchsize=8, learner=baseLearner),
]
processes = 4
maxtasksperchild = 1
json = "./exp.json"
log = re.sub('json$', 'log', json)
if __name__ == '__main__':
result = Benchmark.from_file(json).processes(processes).maxtasksperchild(
maxtasksperchild).evaluate(learner_factories, log)
result.standard_plot()
"""
This is an example script that creates a Benchmark that matches the bandit bakeoff paper.
This script requires that the matplotlib and vowpalwabbit packages be installed.
"""
from coba.learners import RandomLearner, EpsilonLearner, VowpalLearner, UcbTunedLearner, CorralLearner
from coba.benchmarks import Benchmark
if __name__ == '__main__':
benchmark = Benchmark.from_file("./examples/benchmark_short.json")
learners = [
RandomLearner(),
EpsilonLearner(epsilon=0.025),
UcbTunedLearner(),
VowpalLearner(bag=5, seed=10),
CorralLearner([VowpalLearner(bag=5, seed=10),
UcbTunedLearner()],
eta=.075,
T=40000,
seed=10),
]
benchmark.evaluate(learners, './examples/bakeoff.log',
seed=10).standard_plot()
#LambdaSimulation(2000, no_contexts, actions, random_rewards_1, seed=10), #not CB since reward is independent of context
#LambdaSimulation(2000, no_contexts, actions, random_rewards_2, seed=10), #not CB since reward is independent of context
#LambdaSimulation(2000, no_contexts, actions, random_rewards_3, seed=10), #not CB since reward is independent of context
LambdaSimulation(2000,
contexts,
actions,
linear_plus_random_rewards_1,
seed=10),
LambdaSimulation(2000,
contexts,
actions,
linear_plus_random_rewards_2,
seed=10),
LambdaSimulation(2000, contexts, actions, polynomial_reward_1,
seed=10),
]
#define a benchmark: this benchmark replays the simulation 15 times
benchmark = Benchmark(simulations, batch_size=1, seeds=list(range(5)))
#create the learner factories
learner_factories = [
RandomLearner(seed=10),
EpsilonLearner(epsilon=0.025, seed=10),
UcbTunedLearner(seed=10),
VowpalLearner(epsilon=0.025, seed=10),
VowpalLearner(epsilon=0.025, is_adf=False, seed=10),
VowpalLearner(bag=5, seed=10),
]
benchmark.evaluate(learner_factories).standard_plot()
#First, we define the learners that we want to test
learners = [
RandomLearner(),
EpsilonBanditLearner(epsilon=0.025),
VowpalLearner(
epsilon=.1), #This learner requires that VowpalWabbit be installed
]
#Then we define the simulations that we want to test our learners on
simulations = [
ValidationSimulation(300,
context_features=True,
action_features=False,
seed=1000)
]
#And also define a collection of seeds used to shuffle our simulations
seeds = [0, 1, 2, 3]
#We then create our benchmark using our simulations and seeds
benchmark = Benchmark(simulations, shuffle=seeds)
#Finally we evaluate our learners on our benchmark (the results will be saved in `result_file`).
result = benchmark.evaluate(learners)
#After evaluating can create a quick summary plot to get a sense of how the learners performed
result.plot_learners()
#We can also create a plot examining how one specific learner did across each shuffle of our simulation
result.plot_shuffles(learner_pattern="vw")
