def test_launch_experiment_with_overlapping_settings_errors(): config = Mock() config.storage = DummyStore( "", data={ "experiments/foo": { "branches": [{ "id": "foo", "settings": BRANCH_SETTINGS }] }, "experiments/bar": { "branches": [{ "id": "bar", "settings": BRANCH_SETTINGS }] }, }, ) main(("launch", "foo"), config=config) with pytest.raises(SystemExit) as e: main(("launch", "bar"), config=config) assert e.value.args == (1, ) assert "launched" not in config.storage["experiments/bar"] assert "concluded" not in config.storage["experiments/bar"] assert "bar" not in config.storage["active-experiments"]
def test_launch(): config = Mock() config.storage = DummyStore("", data=DUMMY_DATA_PRE_LAUNCH) main(("launch", "foo"), config=config) assert "launched" in config.storage["experiments/foo"] assert "concluded" not in config.storage["experiments/foo"] assert "foo" in config.storage["active-experiments"]
def test_help_message_when_given_no_subcommand(): try: output = io.StringIO() with contextlib.redirect_stdout(output): main([]) except SystemExit: pass assert output.getvalue().startswith("usage: ")
def test_smoke_cli_help(): try: output = io.StringIO() with contextlib.redirect_stdout(output): main(["--help"]) except SystemExit: pass assert output.getvalue().startswith("usage: ")
def test_conclude_updates_defaults(): config = Mock() config.storage = DummyStore("", data=DUMMY_DATA_POST_LAUNCH) main(("conclude", "foo", "bar"), config=config) assert "concluded" in config.storage["experiments/foo"] assert "foo" not in config.storage["active-experiments"] assert "foo" in config.storage["concluded-experiments"] assert config.storage["defaults"] == BRANCH_SETTINGS
def test_conclude_no_branch(): config = Mock() config.storage = DummyStore("", data=DUMMY_DATA_POST_LAUNCH) main(("conclude", "foo", "--no-promote-branch"), config=config) assert "concluded" in config.storage["experiments/foo"] assert "foo" not in config.storage["active-experiments"] assert "foo" in config.storage["concluded-experiments"] assert not config.storage["defaults"]
def test_constraints_are_specialised_on_launch(): config = Mock() config.storage = DummyStore("", data=DUMMY_DATA_PRE_LAUNCH) main(("launch", "foo"), config=config) bucket_zero = config.storage["buckets/0"] (name, settings, constraints) = bucket_zero["entries"][0] assert "era" not in constraints
def test_run_write_command(): config = unittest.mock.Mock() config.storage = DummyStore("", data={}) output = io.StringIO() with contextlib.redirect_stdout(output): main(["set-default", "foo", '"bar"'], config=config) assert output.getvalue() == "" assert config.storage.data == {"defaults": '{"foo": "bar"}'}
def test_jacquard_help_without_args_gives_dash_dash_help(): config = unittest.mock.Mock() stdout_reference = io.StringIO() stdout_actual = io.StringIO() with contextlib.redirect_stdout(stdout_reference), pytest.raises( SystemExit): main(["--help"], config=config) with contextlib.redirect_stdout(stdout_actual), pytest.raises(SystemExit): main(["help"], config=config) assert stdout_reference.getvalue() == stdout_actual.getvalue()
def test_run_basic_command(): config = unittest.mock.Mock() config.storage = DummyStore("", data={"foo": "bar"}) output = io.StringIO() with contextlib.redirect_stdout(output): main(["storage-dump"], config=config) assert (output.getvalue().strip() == textwrap.dedent(""" foo === 'bar' """).strip())
def test_conclude_before_launch_not_allowed(): config = Mock() config.storage = DummyStore("", data=DUMMY_DATA_PRE_LAUNCH) stderr = io.StringIO() with contextlib.redirect_stderr(stderr), pytest.raises(SystemExit): main(("conclude", "foo", "bar"), config=config) assert "concluded" not in config.storage["experiments/foo"] assert config.storage["active-experiments"] is None assert config.storage["concluded-experiments"] is None assert config.storage["defaults"] is None assert stderr.getvalue() == "Experiment 'foo' not launched!\n"
def test_conclude_twice_not_allowed(): config = Mock() config.storage = DummyStore("", data=DUMMY_DATA_POST_LAUNCH) # first conclude works fine main(("conclude", "foo", "bar"), config=config) # second conclude should error stderr = io.StringIO() with contextlib.redirect_stderr(stderr), pytest.raises(SystemExit): main(("conclude", "foo", "other"), config=config) assert "concluded" in config.storage["experiments/foo"] assert "foo" not in config.storage["active-experiments"] assert "foo" in config.storage["concluded-experiments"] assert config.storage["defaults"] == BRANCH_SETTINGS assert stderr.getvalue().startswith("Experiment 'foo' already concluded")
def test_erroring_command(): config = unittest.mock.Mock() ERROR_MESSAGE = "MOCK ERROR: Something went wrong in the" mock_parser = unittest.mock.Mock() mock_options = unittest.mock.Mock() mock_options.func = unittest.mock.Mock( side_effect=CommandError(ERROR_MESSAGE)) mock_parser.parse_args = unittest.mock.Mock(return_value=mock_options) stderr = io.StringIO() with contextlib.redirect_stderr(stderr), pytest.raises(SystemExit): with unittest.mock.patch("jacquard.cli.argument_parser", return_value=mock_parser): main(["command"], config=config) assert stderr.getvalue() == ERROR_MESSAGE + "\n"
def test_load_after_launch_with_skip_launched(): config = Mock() config.storage = DummyStore("", data=DUMMY_DATA_POST_LAUNCH) experiment_data = {"id": "foo"} experiment_data.update(DUMMY_DATA_PRE_LAUNCH["experiments/foo"]) stderr = io.StringIO() with contextlib.redirect_stderr(stderr), patch( "jacquard.experiments.commands.yaml.safe_load", return_value=experiment_data), patch( "jacquard.experiments.commands.argparse.FileType", return_value=str), _disable_argparse_cache(): main(("load-experiment", "--skip-launched", "foo.yaml"), config=config) fresh_data = DummyStore("", data=DUMMY_DATA_POST_LAUNCH) assert fresh_data.data == config.storage.data, "Data should be unchanged" stderr_content = stderr.getvalue() assert "" == stderr_content
def test_load_after_launch_errors(): config = Mock() config.storage = DummyStore("", data=DUMMY_DATA_POST_LAUNCH) experiment_data = {"id": "foo"} experiment_data.update(DUMMY_DATA_PRE_LAUNCH["experiments/foo"]) stderr = io.StringIO() with contextlib.redirect_stderr(stderr), pytest.raises(SystemExit): with patch("jacquard.experiments.commands.yaml.safe_load", return_value=experiment_data), patch( "jacquard.experiments.commands.argparse.FileType", return_value=str), _disable_argparse_cache(): main(("load-experiment", "foo.yaml"), config=config) stderr_content = stderr.getvalue() assert "Experiment 'foo' is live, refusing to edit" in stderr_content fresh_data = DummyStore("", data=DUMMY_DATA_POST_LAUNCH) assert fresh_data.data == config.storage.data, "Data should be unchanged"
def test_overlapping_settings_allowed_if_disjoint_constraints(): config = Mock() config.storage = DummyStore( "", data={ "experiments/foo": { "branches": [{ "id": "foo", "settings": BRANCH_SETTINGS }], "constraints": { "required_tags": ["baz"] }, }, "experiments/bar": { "branches": [{ "id": "bar", "settings": BRANCH_SETTINGS }], "constraints": { "excluded_tags": ["baz"] }, }, }, ) main(("launch", "foo"), config=config) main(("launch", "bar"), config=config) assert "launched" in config.storage["experiments/foo"] assert "concluded" not in config.storage["experiments/foo"] assert "foo" in config.storage["active-experiments"] assert "launched" in config.storage["experiments/bar"] assert "concluded" not in config.storage["experiments/bar"] assert "bar" in config.storage["active-experiments"]
def test_integration(test_file): with (INTEGRATION_TESTS_ROOT / test_file).open("r") as f: test_config = yaml.safe_load(f) config = load_config(io.StringIO(TEST_CONFIG)) config = unittest.mock.Mock(wraps=config) config.directory = DummyDirectory(users=( UserEntry( id="1", join_date=datetime.datetime(2017, 1, 1, tzinfo=dateutil.tz.tzutc()), tags=(), ), UserEntry( id="2", join_date=datetime.datetime(2017, 1, 2, tzinfo=dateutil.tz.tzutc()), tags=("tag1", "tag2"), ), )) wsgi = get_wsgi_app(config) test_client = werkzeug.test.Client(wsgi) for step in test_config: if "command" in step: stdout = io.StringIO() stderr = io.StringIO() args = shlex.split(step["command"]) try: with contextlib.redirect_stdout(stdout): with contextlib.redirect_stderr(stderr): with _temporary_working_directory(JACQUARD_ROOT): main(args, config=config) except SystemExit: pass output = stdout.getvalue() if "expect_error" in step: error_message = stderr.getvalue() else: assert not stderr.getvalue() elif "get" in step: path = step["get"] data, status, headers = test_client.get(path) assert status == "200 OK" output = b"".join(data).decode("utf-8") if "expect" in step: expected_output = textwrap.dedent(step["expect"]).strip() actual_output = textwrap.dedent(output).strip() assert actual_output == expected_output if "expect_yaml" in step: expected_output = step["expect_yaml"] actual_output = yaml.safe_load(output) assert actual_output == expected_output if "expect_yaml_keys" in step: expected_keys = step["expect_yaml_keys"] actual_output = yaml.safe_load(output) assert set(actual_output.keys()) == set(expected_keys) if "expect_error" in step: expected_error = textwrap.dedent(step["expect_error"].strip()) actual_error = textwrap.dedent(error_message).strip() assert actual_error == expected_error
""" Main entry point for `jacquard` command-line utility. This is to enable `python -m jacquard` if that is needed for any reason, normal use should be to use the `jacquard` command-line tool directly. """ from jacquard.cli import main main()