def test_experiment_save_conflict(self, save_experiment, get_experiment):
"""
If an experiment is saved with a certain name and variants, and then
is created later with a different set of variants, e.g.,
split('show_promo', ('True', True), ('False', False))
split('show_promo', ('T', True), ('F', False))
...an exception should be thrown.
"""
backend = FakeBackend()
created = Experiment(
backend=backend, name="show_promo", started_on=datetime.utcnow(), variants=["True", "False"]
)
get_experiment.side_effect = [
None, # the first call fails
created, # but the second call succeeds after a successful save
created,
]
cleaver = Cleaver({}, FakeIdentityProvider(), backend)
assert cleaver.split("show_promo") in (True, False)
get_experiment.assert_called_with("show_promo", ("True", "False"))
save_experiment.assert_called_with("show_promo", ("True", "False"))
self.assertRaises(RuntimeError, cleaver.split, "show_promo", ("T", True), ("F", False))
def test_experiment_save_conflict(self, save_experiment, get_experiment):
"""
If an experiment is saved with a certain name and variants, and then
is created later with a different set of variants, e.g.,
split('show_promo', ('True', True), ('False', False))
split('show_promo', ('T', True), ('F', False))
...an exception should be thrown.
"""
backend = FakeBackend()
created = Experiment(backend=backend,
name='show_promo',
started_on=datetime.utcnow(),
variants=['True', 'False'])
get_experiment.side_effect = [
None, # the first call fails
created, # but the second call succeeds after a successful save
created
]
cleaver = Cleaver({}, FakeIdentityProvider(), backend)
assert cleaver.split('show_promo') in (True, False)
get_experiment.assert_called_with('show_promo', ('True', 'False'))
save_experiment.assert_called_with('show_promo', ('True', 'False'))
self.assertRaises(RuntimeError, cleaver.split, 'show_promo',
('T', True), ('F', False))
def test_variant_override(self, get_experiment):
cleaver = Cleaver({'cleaver.override': {
'show_promo': 'False'
}}, FakeIdentityProvider(), FakeBackend())
get_experiment.return_value.name = 'show_promo'
get_experiment.return_value.variants = ('True', 'False')
assert cleaver.split('show_promo') is False
def test_default_human_verification(self, mark_human, get_experiment):
backend = FakeBackend()
get_experiment.return_value = Experiment(
backend=backend, name="show_promo", started_on=datetime.utcnow(), variants=["True", "False"]
)
cleaver = Cleaver({}, FakeIdentityProvider(), backend)
assert cleaver.split("show_promo") in (True, False)
mark_human.assert_called_with("ABC123")
def test_experiment_get(self, get_experiment):
backend = FakeBackend()
get_experiment.return_value = Experiment(
backend=backend, name="show_promo", started_on=datetime.utcnow(), variants=["True", "False"]
)
cleaver = Cleaver({}, FakeIdentityProvider(), backend)
assert cleaver.split("show_promo") in (True, False)
get_experiment.assert_called_with("show_promo", ("True", "False"))
def test_variant_participation(self, get_identity, participate, get_experiment, random_variant):
cleaver = Cleaver({}, FakeIdentityProvider(), FakeBackend())
get_experiment.return_value.name = "show_promo"
get_experiment.return_value.variants = ("True", "False")
get_identity.return_value = "ABC123"
random_variant.return_value = iter(["True"])
assert cleaver.split("show_promo") in (True, False)
participate.assert_called_with("ABC123", "show_promo", "True")
def test_experiment_get(self, get_experiment):
backend = FakeBackend()
get_experiment.return_value = Experiment(backend=backend,
name='show_promo',
started_on=datetime.utcnow(),
variants=['True', 'False'])
cleaver = Cleaver({}, FakeIdentityProvider(), backend)
assert cleaver.split('show_promo') in (True, False)
get_experiment.assert_called_with('show_promo', ('True', 'False'))
def test_variant_participation(self, get_identity, participate,
get_experiment, random_variant):
cleaver = Cleaver({}, FakeIdentityProvider(), FakeBackend())
get_experiment.return_value.name = 'show_promo'
get_experiment.return_value.variants = ('True', 'False')
get_identity.return_value = 'ABC123'
random_variant.return_value = iter(['True'])
assert cleaver.split('show_promo') in (True, False)
participate.assert_called_with('ABC123', 'show_promo', 'True')
def test_default_human_verification(self, mark_human, get_experiment):
backend = FakeBackend()
get_experiment.return_value = Experiment(backend=backend,
name='show_promo',
started_on=datetime.utcnow(),
variants=['True', 'False'])
cleaver = Cleaver({}, FakeIdentityProvider(), backend)
assert cleaver.split('show_promo') in (True, False)
mark_human.assert_called_with('ABC123')
def test_experiment_save(self, save_experiment, get_experiment):
backend = FakeBackend()
get_experiment.side_effect = [
None, # the first call fails
Experiment(
backend=backend, name="show_promo", started_on=datetime.utcnow(), variants=["True", "False"]
), # but the second call succeeds after a successful save
]
cleaver = Cleaver({}, FakeIdentityProvider(), backend)
assert cleaver.split("show_promo") in (True, False)
get_experiment.assert_called_with("show_promo", ("True", "False"))
save_experiment.assert_called_with("show_promo", ("True", "False"))
def test_experiment_save(self, save_experiment, get_experiment):
backend = FakeBackend()
get_experiment.side_effect = [
None, # the first call fails
Experiment(
backend=backend,
name='show_promo',
started_on=datetime.utcnow(),
variants=[
'True', 'False'
]) # but the second call succeeds after a successful save
]
cleaver = Cleaver({}, FakeIdentityProvider(), backend)
assert cleaver.split('show_promo') in (True, False)
get_experiment.assert_called_with('show_promo', ('True', 'False'))
save_experiment.assert_called_with('show_promo', ('True', 'False'))
def test_variant_override(self, get_experiment):
cleaver = Cleaver({"cleaver.override": {"show_promo": "False"}}, FakeIdentityProvider(), FakeBackend())
get_experiment.return_value.name = "show_promo"
get_experiment.return_value.variants = ("True", "False")
assert cleaver.split("show_promo") is False
