def test_launch_experiment_with_overlapping_settings_errors():
config = Mock()
config.storage = DummyStore(
"",
data={
"experiments/foo": {
"branches": [{
"id": "foo",
"settings": BRANCH_SETTINGS
}]
},
"experiments/bar": {
"branches": [{
"id": "bar",
"settings": BRANCH_SETTINGS
}]
},
},
)
main(("launch", "foo"), config=config)
with pytest.raises(SystemExit) as e:
main(("launch", "bar"), config=config)
assert e.value.args == (1, )
assert "launched" not in config.storage["experiments/bar"]
assert "concluded" not in config.storage["experiments/bar"]
assert "bar" not in config.storage["active-experiments"]
def test_launch():
config = Mock()
config.storage = DummyStore("", data=DUMMY_DATA_PRE_LAUNCH)
main(("launch", "foo"), config=config)
assert "launched" in config.storage["experiments/foo"]
assert "concluded" not in config.storage["experiments/foo"]
assert "foo" in config.storage["active-experiments"]
def test_help_message_when_given_no_subcommand():
try:
output = io.StringIO()
with contextlib.redirect_stdout(output):
main([])
except SystemExit:
pass
assert output.getvalue().startswith("usage: ")
def test_smoke_cli_help():
try:
output = io.StringIO()
with contextlib.redirect_stdout(output):
main(["--help"])
except SystemExit:
pass
assert output.getvalue().startswith("usage: ")
def test_conclude_updates_defaults():
config = Mock()
config.storage = DummyStore("", data=DUMMY_DATA_POST_LAUNCH)
main(("conclude", "foo", "bar"), config=config)
assert "concluded" in config.storage["experiments/foo"]
assert "foo" not in config.storage["active-experiments"]
assert "foo" in config.storage["concluded-experiments"]
assert config.storage["defaults"] == BRANCH_SETTINGS
def test_conclude_no_branch():
config = Mock()
config.storage = DummyStore("", data=DUMMY_DATA_POST_LAUNCH)
main(("conclude", "foo", "--no-promote-branch"), config=config)
assert "concluded" in config.storage["experiments/foo"]
assert "foo" not in config.storage["active-experiments"]
assert "foo" in config.storage["concluded-experiments"]
assert not config.storage["defaults"]
def test_constraints_are_specialised_on_launch():
config = Mock()
config.storage = DummyStore("", data=DUMMY_DATA_PRE_LAUNCH)
main(("launch", "foo"), config=config)
bucket_zero = config.storage["buckets/0"]
(name, settings, constraints) = bucket_zero["entries"][0]
assert "era" not in constraints
def test_run_write_command():
config = unittest.mock.Mock()
config.storage = DummyStore("", data={})
output = io.StringIO()
with contextlib.redirect_stdout(output):
main(["set-default", "foo", '"bar"'], config=config)
assert output.getvalue() == ""
assert config.storage.data == {"defaults": '{"foo": "bar"}'}
def test_jacquard_help_without_args_gives_dash_dash_help():
config = unittest.mock.Mock()
stdout_reference = io.StringIO()
stdout_actual = io.StringIO()
with contextlib.redirect_stdout(stdout_reference), pytest.raises(
SystemExit):
main(["--help"], config=config)
with contextlib.redirect_stdout(stdout_actual), pytest.raises(SystemExit):
main(["help"], config=config)
assert stdout_reference.getvalue() == stdout_actual.getvalue()
def test_run_basic_command():
config = unittest.mock.Mock()
config.storage = DummyStore("", data={"foo": "bar"})
output = io.StringIO()
with contextlib.redirect_stdout(output):
main(["storage-dump"], config=config)
assert (output.getvalue().strip() == textwrap.dedent("""
foo
===
'bar'
""").strip())
def test_conclude_before_launch_not_allowed():
config = Mock()
config.storage = DummyStore("", data=DUMMY_DATA_PRE_LAUNCH)
stderr = io.StringIO()
with contextlib.redirect_stderr(stderr), pytest.raises(SystemExit):
main(("conclude", "foo", "bar"), config=config)
assert "concluded" not in config.storage["experiments/foo"]
assert config.storage["active-experiments"] is None
assert config.storage["concluded-experiments"] is None
assert config.storage["defaults"] is None
assert stderr.getvalue() == "Experiment 'foo' not launched!\n"
def test_conclude_twice_not_allowed():
config = Mock()
config.storage = DummyStore("", data=DUMMY_DATA_POST_LAUNCH)
# first conclude works fine
main(("conclude", "foo", "bar"), config=config)
# second conclude should error
stderr = io.StringIO()
with contextlib.redirect_stderr(stderr), pytest.raises(SystemExit):
main(("conclude", "foo", "other"), config=config)
assert "concluded" in config.storage["experiments/foo"]
assert "foo" not in config.storage["active-experiments"]
assert "foo" in config.storage["concluded-experiments"]
assert config.storage["defaults"] == BRANCH_SETTINGS
assert stderr.getvalue().startswith("Experiment 'foo' already concluded")
def test_erroring_command():
config = unittest.mock.Mock()
ERROR_MESSAGE = "MOCK ERROR: Something went wrong in the"
mock_parser = unittest.mock.Mock()
mock_options = unittest.mock.Mock()
mock_options.func = unittest.mock.Mock(
side_effect=CommandError(ERROR_MESSAGE))
mock_parser.parse_args = unittest.mock.Mock(return_value=mock_options)
stderr = io.StringIO()
with contextlib.redirect_stderr(stderr), pytest.raises(SystemExit):
with unittest.mock.patch("jacquard.cli.argument_parser",
return_value=mock_parser):
main(["command"], config=config)
assert stderr.getvalue() == ERROR_MESSAGE + "\n"
def test_load_after_launch_with_skip_launched():
config = Mock()
config.storage = DummyStore("", data=DUMMY_DATA_POST_LAUNCH)
experiment_data = {"id": "foo"}
experiment_data.update(DUMMY_DATA_PRE_LAUNCH["experiments/foo"])
stderr = io.StringIO()
with contextlib.redirect_stderr(stderr), patch(
"jacquard.experiments.commands.yaml.safe_load",
return_value=experiment_data), patch(
"jacquard.experiments.commands.argparse.FileType",
return_value=str), _disable_argparse_cache():
main(("load-experiment", "--skip-launched", "foo.yaml"), config=config)
fresh_data = DummyStore("", data=DUMMY_DATA_POST_LAUNCH)
assert fresh_data.data == config.storage.data, "Data should be unchanged"
stderr_content = stderr.getvalue()
assert "" == stderr_content
def test_load_after_launch_errors():
config = Mock()
config.storage = DummyStore("", data=DUMMY_DATA_POST_LAUNCH)
experiment_data = {"id": "foo"}
experiment_data.update(DUMMY_DATA_PRE_LAUNCH["experiments/foo"])
stderr = io.StringIO()
with contextlib.redirect_stderr(stderr), pytest.raises(SystemExit):
with patch("jacquard.experiments.commands.yaml.safe_load",
return_value=experiment_data), patch(
"jacquard.experiments.commands.argparse.FileType",
return_value=str), _disable_argparse_cache():
main(("load-experiment", "foo.yaml"), config=config)
stderr_content = stderr.getvalue()
assert "Experiment 'foo' is live, refusing to edit" in stderr_content
fresh_data = DummyStore("", data=DUMMY_DATA_POST_LAUNCH)
assert fresh_data.data == config.storage.data, "Data should be unchanged"
def test_overlapping_settings_allowed_if_disjoint_constraints():
config = Mock()
config.storage = DummyStore(
"",
data={
"experiments/foo": {
"branches": [{
"id": "foo",
"settings": BRANCH_SETTINGS
}],
"constraints": {
"required_tags": ["baz"]
},
},
"experiments/bar": {
"branches": [{
"id": "bar",
"settings": BRANCH_SETTINGS
}],
"constraints": {
"excluded_tags": ["baz"]
},
},
},
)
main(("launch", "foo"), config=config)
main(("launch", "bar"), config=config)
assert "launched" in config.storage["experiments/foo"]
assert "concluded" not in config.storage["experiments/foo"]
assert "foo" in config.storage["active-experiments"]
assert "launched" in config.storage["experiments/bar"]
assert "concluded" not in config.storage["experiments/bar"]
assert "bar" in config.storage["active-experiments"]
def test_integration(test_file):
with (INTEGRATION_TESTS_ROOT / test_file).open("r") as f:
test_config = yaml.safe_load(f)
config = load_config(io.StringIO(TEST_CONFIG))
config = unittest.mock.Mock(wraps=config)
config.directory = DummyDirectory(users=(
UserEntry(
id="1",
join_date=datetime.datetime(2017, 1, 1,
tzinfo=dateutil.tz.tzutc()),
tags=(),
),
UserEntry(
id="2",
join_date=datetime.datetime(2017, 1, 2,
tzinfo=dateutil.tz.tzutc()),
tags=("tag1", "tag2"),
),
))
wsgi = get_wsgi_app(config)
test_client = werkzeug.test.Client(wsgi)
for step in test_config:
if "command" in step:
stdout = io.StringIO()
stderr = io.StringIO()
args = shlex.split(step["command"])
try:
with contextlib.redirect_stdout(stdout):
with contextlib.redirect_stderr(stderr):
with _temporary_working_directory(JACQUARD_ROOT):
main(args, config=config)
except SystemExit:
pass
output = stdout.getvalue()
if "expect_error" in step:
error_message = stderr.getvalue()
else:
assert not stderr.getvalue()
elif "get" in step:
path = step["get"]
data, status, headers = test_client.get(path)
assert status == "200 OK"
output = b"".join(data).decode("utf-8")
if "expect" in step:
expected_output = textwrap.dedent(step["expect"]).strip()
actual_output = textwrap.dedent(output).strip()
assert actual_output == expected_output
if "expect_yaml" in step:
expected_output = step["expect_yaml"]
actual_output = yaml.safe_load(output)
assert actual_output == expected_output
if "expect_yaml_keys" in step:
expected_keys = step["expect_yaml_keys"]
actual_output = yaml.safe_load(output)
assert set(actual_output.keys()) == set(expected_keys)
if "expect_error" in step:
expected_error = textwrap.dedent(step["expect_error"].strip())
actual_error = textwrap.dedent(error_message).strip()
assert actual_error == expected_error
""" Main entry point for `jacquard` command-line utility. This is to enable `python -m jacquard` if that is needed for any reason, normal use should be to use the `jacquard` command-line tool directly. """ from jacquard.cli import main main()