def test_get_twice_returns_same_instance():
data = {"examples/1": {"name": "Paula"}}
session = Session(data)
ex1 = session.get(Example, 1)
ex2 = session.get(Example, 1)
assert ex1 is ex2
def get_settings(user_id, storage, directory=None):
"""
Look up the current settings dict for a given user ID.
This takes, in order of preference:
1. The global defaults,
2. Any experiment settings for experiments the user is in,
3. User-specific overrides.
"""
with storage.transaction(read_only=True) as store:
session = Session(store)
defaults = store.get("defaults", {})
bucket_id = user_bucket(user_id)
bucket = session.get(Bucket, bucket_id, default=EMPTY)
if bucket.needs_constraints():
user_entry = directory.lookup(user_id)
else:
user_entry = None
bucket_settings = bucket.get_settings(user_entry)
overrides = store.get("overrides/{user_id}".format(user_id=user_id),
{})
return {**defaults, **bucket_settings, **overrides}
def test_error_when_adding_instance_to_two_sessions():
session1, session2 = Session({}), Session({})
ex1 = Example(pk=1, name="Horse")
session1.add(ex1)
with pytest.raises(RuntimeError):
session2.add(ex1)
def test_get_then_edit_entity():
data = {"examples/1": {"name": "Paula"}}
session = Session(data)
ex1 = session.get(Example, 1)
ex1.name = "Paul"
session.flush()
assert data == {"examples/1": {"name": "Paul"}}
def release(store, name, constraints, branches):
"""
Release a given configuration.
This is the main utility for launching experiments. `store` is the storage
transaction mapping. `name` is a free-form name (generally an experiment
ID), `constraints` is the constraints covering this release, and `branches`
is an iterable of (branch ID, num buckets, settings) triples.
The utility will select buckets which are not already covering the given
settings, which allows for partial rollout before running a test.
"""
session = Session(store)
# Branches is a list of (name, n_buckets, settings) tuples
all_buckets = [
session.get(Bucket, x, default=CREATE) for x in range(NUM_BUCKETS)
]
edited_settings = set.union(*[set(x[2].keys()) for x in branches])
conflicting_experiments = set()
valid_bucket_indices = []
for idx, bucket in enumerate(all_buckets):
if is_valid_bucket(bucket, edited_settings, constraints):
valid_bucket_indices.append(idx)
else:
for entry in bucket.entries:
# Determine if this entry is a potential conflict
if set(entry.settings.keys()).isdisjoint(edited_settings):
continue
conflicting_experiment_id, _ = entry.key
conflicting_experiments.add(conflicting_experiment_id)
random.shuffle(valid_bucket_indices)
for branch_name, n_buckets, settings in branches:
key = [name, branch_name]
bucket_indices = valid_bucket_indices[:n_buckets]
if len(bucket_indices) < n_buckets:
raise NotEnoughBucketsException(conflicts=conflicting_experiments)
valid_bucket_indices = valid_bucket_indices[n_buckets:]
for bucket_idx in bucket_indices:
bucket = all_buckets[bucket_idx]
bucket.add(key, settings, constraints)
session.flush()
def test_sessions_are_not_global():
instance_session1 = Example(pk=1)
session = Session({})
session.add(instance_session1)
session.flush()
session2 = Session({})
with pytest.raises(KeyError):
instance_session2 = session2.get(Example, 1)
def handle(self, experiment):
"""Dispatch request."""
if self.request.method != "POST":
raise MethodNotAllowed()
user_ids = self.request.form.getlist("u")
with self.config.storage.transaction(read_only=True) as store:
session = Session(store)
try:
experiment_config = Experiment.from_store(store, experiment)
except LookupError:
raise NotFound(
"No experiment with ID {experiment_id!r}".format(
experiment_id=experiment
)
)
buckets = [
session.get(Bucket, idx, default=EMPTY) for idx in range(NUM_BUCKETS)
]
branch_ids = [branch["id"] for branch in experiment_config.branches]
branches = {x: [] for x in branch_ids}
relevant_settings = set()
for branch_config in experiment_config.branches:
relevant_settings.update(branch_config["settings"].keys())
for user_id in user_ids:
user_entry = self.config.directory.lookup(user_id)
if not experiment_config.includes_user(user_entry):
continue
user_overrides = store.get(
"overrides/{user_id}".format(user_id=user_id), {}
)
if any(x in relevant_settings for x in user_overrides.keys()):
continue
bucket = buckets[user_bucket(user_id)]
for branch_id, members in branches.items():
if bucket.covers([experiment_config.id, branch_id]):
members.append(user_id)
return {"branches": branches}
def test_add_then_remove_entity_has_no_effect():
data = {}
session = Session(data)
ex1 = Example(pk=1, name="Horse")
session.add(ex1)
session.remove(ex1)
session.flush()
assert data == {}
def test_error_when_removing_from_different_instance():
session1 = Session({"examples/1": {}})
session2 = Session({})
ex1 = session1.get(Example, 1)
with pytest.raises(RuntimeError):
session2.remove(ex1)
def test_modify_with_field():
data = {}
session = Session(data)
ex1 = Example(pk=1, name="Paul")
session.add(ex1)
session.flush()
ex1.name = "Paula"
session.flush()
assert data == {"examples/1": {"name": "Paula"}}
def test_remove_is_idempotent():
data = {"examples/1": {"name": "Paula"}}
session = Session(data)
ex1 = session.get(Example, 1)
session.remove(ex1)
session.remove(ex1)
session.flush()
def test_get_then_remove_entity():
data = {"examples/1": {"name": "Paula"}}
session = Session(data)
ex1 = session.get(Example, 1)
session.remove(ex1)
session.flush()
assert data == {}
def close(store, name, constraints, branches):
"""
Close a given configuration.
This is the main utility for ending experiments. `store` is the storage
transaction mapping. `name` is a free-form name (generally an experiment
ID), `constraints` is the constraints covering this release, and `branches`
is an iterable of (branch ID, num buckets, settings) triples.
Deliberately looks like `release` and works to counteract its effects.
"""
session = Session(store)
keys = [[name, x[0]] for x in branches]
for idx in range(NUM_BUCKETS):
bucket = session.get(Bucket, idx, default=None)
if bucket is not None:
for key in keys:
bucket.remove(key)
session.flush()
def test_add():
data = {}
session = Session(data)
ex1 = Example(pk=1)
session.add(ex1)
session.flush()
assert data == {"examples/1": {}}
def test_non_nullable_fields_cannot_be_saved_with_null_values():
instance = Example(pk=1)
instance.defaulted_field = None
session = Session({})
session.add(instance)
with pytest.raises(ValueError):
session.flush()
def test_can_get_empty_bucket_from_old_format():
session = Session({"buckets/1": []})
bucket = session.get(Bucket, 1)
# Force bucket to a string in order to reify the fields. This validates
# that the fields are accessible.
str(bucket)
def test_get_missing_entity_raises_keyerror():
data = {"examples/1": {"name": "Paula"}}
session = Session(data)
with pytest.raises(KeyError):
session.get(Example, 2)
def test_get():
data = {"examples/1": {"name": "Paula"}}
session = Session(data)
ex1 = session.get(Example, 1)
assert ex1.name == "Paula"
def test_error_when_instantiating_with_duplicate_id():
session = Session({})
session.add(Example(pk=1))
with pytest.raises(RuntimeError):
session.add(Example(pk=1))