def test_launch_experiment_with_overlapping_settings_errors():
config = Mock()
config.storage = DummyStore(
"",
data={
"experiments/foo": {
"branches": [{
"id": "foo",
"settings": BRANCH_SETTINGS
}]
},
"experiments/bar": {
"branches": [{
"id": "bar",
"settings": BRANCH_SETTINGS
}]
},
},
)
main(("launch", "foo"), config=config)
with pytest.raises(SystemExit) as e:
main(("launch", "bar"), config=config)
assert e.value.args == (1, )
assert "launched" not in config.storage["experiments/bar"]
assert "concluded" not in config.storage["experiments/bar"]
assert "bar" not in config.storage["active-experiments"]
def get_test_client():
config = Mock()
config.storage = DummyStore(
"",
data={
"defaults": {
"pony": "gravity"
},
"active-experiments": ["foo"],
"experiments/foo": {
"id": "foo",
"constraints": {
"excluded_tags": ["excluded"]
},
"branches": [{
"id": "bar",
"settings": {
"pony": "horse"
}
}],
},
},
)
now = datetime.datetime.now(dateutil.tz.tzutc())
config.directory = DummyDirectory(users=(
UserEntry(id=1, join_date=now, tags=("excluded", )),
UserEntry(id=2, join_date=now, tags=("excluded", )),
UserEntry(id=3, join_date=now, tags=()),
))
wsgi = get_wsgi_app(config)
return werkzeug.test.Client(wsgi)
def test_duplicate_accesses_continue_to_raise_keyerror():
store = DummyStore("")
transaction_map = TransactionMap(store)
with pytest.raises(KeyError):
transaction_map["test"]
with pytest.raises(KeyError):
transaction_map["test"]
def test_launch():
config = Mock()
config.storage = DummyStore("", data=DUMMY_DATA_PRE_LAUNCH)
main(("launch", "foo"), config=config)
assert "launched" in config.storage["experiments/foo"]
assert "concluded" not in config.storage["experiments/foo"]
assert "foo" in config.storage["active-experiments"]
def test_conclude_updates_defaults():
config = Mock()
config.storage = DummyStore("", data=DUMMY_DATA_POST_LAUNCH)
main(("conclude", "foo", "bar"), config=config)
assert "concluded" in config.storage["experiments/foo"]
assert "foo" not in config.storage["active-experiments"]
assert "foo" in config.storage["concluded-experiments"]
assert config.storage["defaults"] == BRANCH_SETTINGS
def test_conclude_no_branch():
config = Mock()
config.storage = DummyStore("", data=DUMMY_DATA_POST_LAUNCH)
main(("conclude", "foo", "--no-promote-branch"), config=config)
assert "concluded" in config.storage["experiments/foo"]
assert "foo" not in config.storage["active-experiments"]
assert "foo" in config.storage["concluded-experiments"]
assert not config.storage["defaults"]
def test_run_write_command():
config = unittest.mock.Mock()
config.storage = DummyStore("", data={})
output = io.StringIO()
with contextlib.redirect_stdout(output):
main(["set-default", "foo", '"bar"'], config=config)
assert output.getvalue() == ""
assert config.storage.data == {"defaults": '{"foo": "bar"}'}
def test_load_after_launch_with_skip_launched():
config = Mock()
config.storage = DummyStore("", data=DUMMY_DATA_POST_LAUNCH)
experiment_data = {"id": "foo"}
experiment_data.update(DUMMY_DATA_PRE_LAUNCH["experiments/foo"])
stderr = io.StringIO()
with contextlib.redirect_stderr(stderr), patch(
"jacquard.experiments.commands.yaml.safe_load",
return_value=experiment_data), patch(
"jacquard.experiments.commands.argparse.FileType",
return_value=str), _disable_argparse_cache():
main(("load-experiment", "--skip-launched", "foo.yaml"), config=config)
fresh_data = DummyStore("", data=DUMMY_DATA_POST_LAUNCH)
assert fresh_data.data == config.storage.data, "Data should be unchanged"
stderr_content = stderr.getvalue()
assert "" == stderr_content
def test_load_after_launch_errors():
config = Mock()
config.storage = DummyStore("", data=DUMMY_DATA_POST_LAUNCH)
experiment_data = {"id": "foo"}
experiment_data.update(DUMMY_DATA_PRE_LAUNCH["experiments/foo"])
stderr = io.StringIO()
with contextlib.redirect_stderr(stderr), pytest.raises(SystemExit):
with patch("jacquard.experiments.commands.yaml.safe_load",
return_value=experiment_data), patch(
"jacquard.experiments.commands.argparse.FileType",
return_value=str), _disable_argparse_cache():
main(("load-experiment", "foo.yaml"), config=config)
stderr_content = stderr.getvalue()
assert "Experiment 'foo' is live, refusing to edit" in stderr_content
fresh_data = DummyStore("", data=DUMMY_DATA_POST_LAUNCH)
assert fresh_data.data == config.storage.data, "Data should be unchanged"
def test_run_basic_command():
config = unittest.mock.Mock()
config.storage = DummyStore("", data={"foo": "bar"})
output = io.StringIO()
with contextlib.redirect_stdout(output):
main(["storage-dump"], config=config)
assert (output.getvalue().strip() == textwrap.dedent("""
foo
===
'bar'
""").strip())
def test_conclude_before_launch_not_allowed():
config = Mock()
config.storage = DummyStore("", data=DUMMY_DATA_PRE_LAUNCH)
stderr = io.StringIO()
with contextlib.redirect_stderr(stderr), pytest.raises(SystemExit):
main(("conclude", "foo", "bar"), config=config)
assert "concluded" not in config.storage["experiments/foo"]
assert config.storage["active-experiments"] is None
assert config.storage["concluded-experiments"] is None
assert config.storage["defaults"] is None
assert stderr.getvalue() == "Experiment 'foo' not launched!\n"
def test_conclude_twice_not_allowed():
config = Mock()
config.storage = DummyStore("", data=DUMMY_DATA_POST_LAUNCH)
# first conclude works fine
main(("conclude", "foo", "bar"), config=config)
# second conclude should error
stderr = io.StringIO()
with contextlib.redirect_stderr(stderr), pytest.raises(SystemExit):
main(("conclude", "foo", "other"), config=config)
assert "concluded" in config.storage["experiments/foo"]
assert "foo" not in config.storage["active-experiments"]
assert "foo" in config.storage["concluded-experiments"]
assert config.storage["defaults"] == BRANCH_SETTINGS
assert stderr.getvalue().startswith("Experiment 'foo' already concluded")
def test_overlapping_settings_allowed_if_disjoint_constraints():
config = Mock()
config.storage = DummyStore(
"",
data={
"experiments/foo": {
"branches": [{
"id": "foo",
"settings": BRANCH_SETTINGS
}],
"constraints": {
"required_tags": ["baz"]
},
},
"experiments/bar": {
"branches": [{
"id": "bar",
"settings": BRANCH_SETTINGS
}],
"constraints": {
"excluded_tags": ["baz"]
},
},
},
)
main(("launch", "foo"), config=config)
main(("launch", "bar"), config=config)
assert "launched" in config.storage["experiments/foo"]
assert "concluded" not in config.storage["experiments/foo"]
assert "foo" in config.storage["active-experiments"]
assert "launched" in config.storage["experiments/bar"]
assert "concluded" not in config.storage["experiments/bar"]
assert "bar" in config.storage["active-experiments"]
def open_storage(self):
return DummyStore("")