def test_experiments_to_analyze_end_date_override(self):
executor = cli.AnalysisExecutor(
project_id="project",
dataset_id="dataset",
bucket="bucket",
date=dt.datetime(2021, 2, 15, tzinfo=UTC),
experiment_slugs=cli.All,
)
result = executor._experiment_configs_to_analyse(
cli_experiments, external_config.ExternalConfigCollection)
assert result == []
conf = dedent("""
[experiment]
end_date = 2021-03-01
""")
external_configs = external_config.ExternalConfigCollection([
external_config.ExternalConfig(
slug="my_cool_experiment",
spec=AnalysisSpec.from_dict(toml.loads(conf)),
last_modified=dt.datetime(2021, 2, 15, tzinfo=UTC),
)
])
def config_getter():
return external_configs
result = executor._experiment_configs_to_analyse(
cli_experiments, config_getter)
assert set(e.experiment.normandy_slug
for e in result) == {"my_cool_experiment"}
def test_recreate_enrollments(self, monkeypatch):
executor = cli.AnalysisExecutor(
project_id="project",
dataset_id="dataset",
bucket="bucket",
date=dt.datetime(2020, 10, 28, tzinfo=UTC),
experiment_slugs=["my_cool_experiment"],
recreate_enrollments=True,
)
bigquery_mock_client = Mock()
monkeypatch.setattr("jetstream.cli.BigQueryClient",
bigquery_mock_client)
strategy = DummyExecutorStrategy("project", "dataset")
success = executor.execute(
experiment_getter=cli_experiments,
config_getter=external_config.ExternalConfigCollection,
strategy=strategy,
)
assert success
assert len(strategy.worklist) == 1
assert strategy.worklist[0][
0].experiment.normandy_slug == "my_cool_experiment"
assert strategy.worklist[0][1] == dt.datetime(2020, 10, 28, tzinfo=UTC)
assert bigquery_mock_client.delete_table.called_once_with(
"project.dataset.enrollments_my_cool_experiment")
def test_experiments_to_analyze_all(self):
executor = cli.AnalysisExecutor(
project_id="project",
dataset_id="dataset",
bucket="bucket",
date=cli.All,
experiment_slugs=cli.All,
)
with pytest.raises(ValueError):
executor._experiment_configs_to_analyse(cli_experiments)
def test_experiments_to_analyze_specific_date(self):
executor = cli.AnalysisExecutor(
project_id="project",
dataset_id="dataset",
bucket="bucket",
date=dt.datetime(2020, 10, 31, tzinfo=UTC),
experiment_slugs=cli.All,
)
result = executor._experiment_configs_to_analyse(
cli_experiments, external_config.ExternalConfigCollection)
assert len(result) == 2
def test_experiments_to_analyze(self):
executor = cli.AnalysisExecutor(
project_id="project",
dataset_id="dataset",
bucket="bucket",
date=cli.All,
experiment_slugs=["bogus_experiment", "my_cool_experiment"],
)
result = executor._experiment_configs_to_analyse(
cli_experiments, external_config.ExternalConfigCollection)
assert set(e.experiment.normandy_slug
for e in result) == {"my_cool_experiment"}
def test_trivial_case(self):
executor = cli.AnalysisExecutor(
project_id="project",
dataset_id="dataset",
bucket="bucket",
date=dt.datetime(2020, 10, 28, tzinfo=UTC),
experiment_slugs=[],
)
strategy = DummyExecutorStrategy("project", "dataset")
success = executor.execute(
experiment_getter=experimenter.ExperimentCollection,
config_getter=external_config.ExternalConfigCollection,
strategy=strategy,
)
assert success
assert strategy.worklist == []
def test_bogus_experiment(self):
executor = cli.AnalysisExecutor(
project_id="project",
dataset_id="dataset",
bucket="bucket",
date=cli.All,
experiment_slugs=["bogus_experiment", "my_cool_experiment"],
)
strategy = DummyExecutorStrategy("project", "dataset")
executor.execute(
experiment_getter=cli_experiments,
config_getter=external_config.ExternalConfigCollection,
strategy=strategy,
today=dt.datetime(2020, 12, 31, tzinfo=UTC),
)
assert set(w[0].experiment.normandy_slug
for w in strategy.worklist) == {"my_cool_experiment"}
def test_bartleby(self):
"'I would prefer not to.' - Bartleby"
executor = cli.AnalysisExecutor(
project_id="project",
dataset_id="dataset",
bucket="bucket",
date=cli.All,
experiment_slugs=cli.All,
)
strategy = DummyExecutorStrategy("project", "dataset")
with pytest.raises(ValueError):
executor.execute(
experiment_getter=cli_experiments,
config_getter=external_config.ExternalConfigCollection,
strategy=strategy,
today=dt.datetime(2020, 12, 31, tzinfo=UTC),
)
def test_ensure_enrollments(self, monkeypatch):
executor = cli.AnalysisExecutor(
project_id="project",
dataset_id="dataset",
bucket="bucket",
date=cli.All,
experiment_slugs=["my_cool_experiment"],
)
Analysis = Mock()
monkeypatch.setattr("jetstream.cli.Analysis", Analysis)
executor.ensure_enrollments(
experiment_getter=cli_experiments,
config_getter=external_config.ExternalConfigCollection,
)
assert Analysis.ensure_enrollments.called_once()
def test_post_facto_rerun_includes_overall_date(self):
executor = cli.AnalysisExecutor(
project_id="project",
dataset_id="dataset",
bucket="bucket",
date=cli.All,
experiment_slugs=["my_cool_experiment"],
)
strategy = DummyExecutorStrategy("project", "dataset")
success = executor.execute(
experiment_getter=cli_experiments,
config_getter=external_config.ExternalConfigCollection,
strategy=strategy,
today=dt.datetime(2022, 12, 31, tzinfo=UTC),
)
assert success
assert max(w[1]
for w in strategy.worklist).date() == dt.date(2021, 2, 2)
def test_all_single_date(self, cli_experiments):
executor = cli.AnalysisExecutor(
project_id="project",
dataset_id="dataset",
bucket="bucket",
date=dt.datetime(2020, 10, 28, tzinfo=UTC),
experiment_slugs=cli.All,
)
strategy = DummyExecutorStrategy("project", "dataset")
success = executor.execute(
experiment_getter=lambda: cli_experiments,
config_getter=external_config.ExternalConfigCollection,
strategy=strategy,
)
assert success
assert len(strategy.worklist) == 2
assert {c.experiment.normandy_slug
for c, _ in strategy.worklist
} == {x.normandy_slug
for x in cli_experiments.experiments}
