def test_get_timelimits_if_ready(experiments):
config = AnalysisSpec().resolve(experiments[0])
config2 = AnalysisSpec().resolve(experiments[2])
analysis = Analysis("test", "test", config)
analysis2 = Analysis("test", "test", config2)
date = dt.datetime(2019, 12, 1, tzinfo=pytz.utc) + timedelta(0)
assert analysis._get_timelimits_if_ready(AnalysisPeriod.DAY, date) is None
assert analysis._get_timelimits_if_ready(AnalysisPeriod.WEEK, date) is None
date = dt.datetime(2019, 12, 1, tzinfo=pytz.utc) + timedelta(2)
assert analysis._get_timelimits_if_ready(AnalysisPeriod.DAY, date) is None
assert analysis._get_timelimits_if_ready(AnalysisPeriod.WEEK, date) is None
date = dt.datetime(2019, 12, 1, tzinfo=pytz.utc) + timedelta(7)
assert analysis._get_timelimits_if_ready(AnalysisPeriod.DAY, date)
assert analysis._get_timelimits_if_ready(AnalysisPeriod.WEEK, date) is None
date = dt.datetime(2019, 12, 1, tzinfo=pytz.utc) + timedelta(days=13)
assert analysis._get_timelimits_if_ready(AnalysisPeriod.DAY, date)
assert analysis._get_timelimits_if_ready(AnalysisPeriod.WEEK, date)
date = dt.datetime(2020, 2, 29, tzinfo=pytz.utc)
assert analysis._get_timelimits_if_ready(AnalysisPeriod.OVERALL,
date) is None
date = dt.datetime(2020, 3, 1, tzinfo=pytz.utc)
assert analysis._get_timelimits_if_ready(AnalysisPeriod.OVERALL, date)
assert analysis2._get_timelimits_if_ready(AnalysisPeriod.OVERALL,
date) is None
date = dt.datetime(2019, 12, 1, tzinfo=pytz.utc) + timedelta(days=34)
assert analysis._get_timelimits_if_ready(AnalysisPeriod.DAYS_28, date)
def test_regression_20200316():
experiment_json = r"""
{
"experiment_url": "https://blah/experiments/search-tips-aka-nudges/",
"type": "addon",
"name": "Search Tips aka Nudges",
"slug": "search-tips-aka-nudges",
"public_name": "Search Tips",
"public_description": "Search Tips are designed to increase engagement with the QuantumBar.",
"status": "Live",
"countries": [],
"platform": "All Platforms",
"start_date": 1578960000000,
"end_date": 1584921600000,
"population": "2% of Release Firefox 72.0 to 74.0",
"population_percent": "2.0000",
"firefox_channel": "Release",
"firefox_min_version": "72.0",
"firefox_max_version": "74.0",
"addon_experiment_id": null,
"addon_release_url": "https://bugzilla.mozilla.org/attachment.cgi?id=9120542",
"pref_branch": null,
"pref_name": null,
"pref_type": null,
"proposed_start_date": 1578960000000,
"proposed_enrollment": 21,
"proposed_duration": 69,
"normandy_slug": "addon-search-tips-aka-nudges-release-72-74-bug-1603564",
"normandy_id": 902,
"other_normandy_ids": [],
"variants": [
{
"description": "Standard address bar experience",
"is_control": false,
"name": "control",
"ratio": 50,
"slug": "control",
"value": null,
"addon_release_url": null,
"preferences": []
},
{
"description": "",
"is_control": true,
"name": "treatment",
"ratio": 50,
"slug": "treatment",
"value": null,
"addon_release_url": null,
"preferences": []
}
]
}
"""
experiment = ExperimentV1.from_dict(
json.loads(experiment_json)).to_experiment()
config = AnalysisSpec().resolve(experiment)
analysis = Analysis("test", "test", config)
analysis.run(current_date=dt.datetime(2020, 3, 16, tzinfo=pytz.utc),
dry_run=True)
def test_experiments_to_analyze_end_date_override(self):
executor = cli.AnalysisExecutor(
project_id="project",
dataset_id="dataset",
bucket="bucket",
date=dt.datetime(2021, 2, 15, tzinfo=UTC),
experiment_slugs=cli.All,
)
result = executor._experiment_configs_to_analyse(
cli_experiments, external_config.ExternalConfigCollection)
assert result == []
conf = dedent("""
[experiment]
end_date = 2021-03-01
""")
external_configs = external_config.ExternalConfigCollection([
external_config.ExternalConfig(
slug="my_cool_experiment",
spec=AnalysisSpec.from_dict(toml.loads(conf)),
last_modified=dt.datetime(2021, 2, 15, tzinfo=UTC),
)
])
def config_getter():
return external_configs
result = executor._experiment_configs_to_analyse(
cli_experiments, config_getter)
assert set(e.experiment.normandy_slug
for e in result) == {"my_cool_experiment"}
def from_github_repo(cls) -> "ExternalConfigCollection":
"""Pull in external config files."""
# download files to tmp directory
with TemporaryDirectory() as tmp_dir:
repo = Repo.clone_from(cls.JETSTREAM_CONFIG_URL, tmp_dir)
external_configs = []
for config_file in tmp_dir.glob("*.toml"):
last_modified = next(repo.iter_commits("main", paths=config_file)).committed_date
external_configs.append(
ExternalConfig(
config_file.stem,
AnalysisSpec.from_dict(toml.load(config_file)),
UTC.localize(dt.datetime.utcfromtimestamp(last_modified)),
)
)
outcomes = []
for outcome_file in tmp_dir.glob(f"**/{OUTCOMES_DIR}/*/*.toml"):
commit_hash = next(repo.iter_commits("main", paths=outcome_file)).hexsha
outcomes.append(
ExternalOutcome(
slug=outcome_file.stem,
spec=OutcomeSpec.from_dict(toml.load(outcome_file)),
platform=outcome_file.parent.name,
commit_hash=commit_hash,
)
)
return cls(external_configs, outcomes)
def test_metadata_from_config(mock_get, experiments):
config_str = dedent(
"""
[metrics]
weekly = ["view_about_logins", "my_cool_metric"]
daily = ["my_cool_metric"]
[metrics.my_cool_metric]
data_source = "main"
select_expression = "{{agg_histogram_mean('payload.content.my_cool_histogram')}}"
friendly_name = "Cool metric"
description = "Cool cool cool"
bigger_is_better = false
[metrics.my_cool_metric.statistics.bootstrap_mean]
[metrics.view_about_logins.statistics.bootstrap_mean]
"""
)
spec = AnalysisSpec.from_dict(toml.loads(config_str))
config = spec.resolve(experiments[4])
metadata = ExperimentMetadata.from_config(config)
assert StatisticResult.SCHEMA_VERSION == metadata.schema_version
assert "view_about_logins" in metadata.metrics
assert metadata.metrics["view_about_logins"].bigger_is_better
assert metadata.metrics["view_about_logins"].description != ""
assert "my_cool_metric" in metadata.metrics
assert metadata.metrics["my_cool_metric"].bigger_is_better is False
assert metadata.metrics["my_cool_metric"].friendly_name == "Cool metric"
assert metadata.metrics["my_cool_metric"].description == "Cool cool cool"
assert metadata.metrics["my_cool_metric"].analysis_bases == ["enrollments"]
assert metadata.external_config is None
def test_metadata_from_config_missing_metadata(mock_get, experiments):
config_str = dedent(
"""
[metrics]
weekly = ["view_about_logins", "my_cool_metric"]
daily = ["my_cool_metric"]
[metrics.my_cool_metric]
data_source = "main"
select_expression = "{{agg_histogram_mean('payload.content.my_cool_histogram')}}"
analysis_bases = ["exposures"]
[metrics.my_cool_metric.statistics.bootstrap_mean]
[metrics.view_about_logins.statistics.bootstrap_mean]
"""
)
spec = AnalysisSpec.from_dict(toml.loads(config_str))
config = spec.resolve(experiments[0])
metadata = ExperimentMetadata.from_config(config)
assert "my_cool_metric" in metadata.metrics
assert metadata.metrics["my_cool_metric"].bigger_is_better
assert metadata.metrics["my_cool_metric"].friendly_name == ""
assert metadata.metrics["my_cool_metric"].description == ""
assert metadata.metrics["my_cool_metric"].analysis_bases == ["exposures"]
def test_validate_doesnt_explode(experiments, monkeypatch):
m = Mock()
monkeypatch.setattr(jetstream.analysis, "dry_run_query", m)
x = experiments[0]
config = AnalysisSpec.default_for_experiment(x).resolve(x)
Analysis("spam", "eggs", config).validate()
assert m.call_count == 2
def test_validating_external_config(self, monkeypatch, experiments):
Analysis = Mock()
monkeypatch.setattr("jetstream.external_config.Analysis", Analysis)
spec = AnalysisSpec.from_dict({})
extern = ExternalConfig(
slug="cool_experiment",
spec=spec,
last_modified=dt.datetime.now(),
)
extern.validate(experiments[0])
assert Analysis.validate.called_once()
def entity_from_path(path: Path) -> Union[ExternalConfig, ExternalOutcome]:
is_outcome = path.parent.parent.name == OUTCOMES_DIR
slug = path.stem
config_dict = toml.loads(path.read_text())
if is_outcome:
platform = path.parent.name
spec = OutcomeSpec.from_dict(config_dict)
return ExternalOutcome(slug=slug, spec=spec, platform=platform, commit_hash=None)
return ExternalConfig(
slug=slug,
spec=AnalysisSpec.from_dict(config_dict),
last_modified=dt.datetime.fromtimestamp(path.stat().st_mtime, UTC),
)
def test_skip_works(experiments):
conf = dedent("""
[experiment]
skip = true
""")
spec = AnalysisSpec.from_dict(toml.loads(conf))
configured = spec.resolve(experiments[0])
with pytest.raises(ExplicitSkipException):
Analysis("test", "test",
configured).run(current_date=dt.datetime(2020,
1,
1,
tzinfo=pytz.utc),
dry_run=True)
def test_metadata_reference_branch(mock_get, experiments):
config_str = dedent(
"""
[experiment]
reference_branch = "a"
[metrics]
weekly = ["view_about_logins"]
[metrics.view_about_logins.statistics.bootstrap_mean]
"""
)
spec = AnalysisSpec.from_dict(toml.loads(config_str))
config = spec.resolve(experiments[4])
metadata = ExperimentMetadata.from_config(config)
assert metadata.external_config.reference_branch == "a"
assert (
metadata.external_config.url
== ExternalConfigCollection.JETSTREAM_CONFIG_URL + "/blob/main/normandy-test-slug.toml"
)
config_str = dedent(
"""
[metrics]
weekly = ["view_about_logins"]
[metrics.view_about_logins.statistics.bootstrap_mean]
"""
)
spec = AnalysisSpec.from_dict(toml.loads(config_str))
config = spec.resolve(experiments[2])
metadata = ExperimentMetadata.from_config(config)
assert metadata.external_config is None
def test_analysis_doesnt_choke_on_segments(experiments):
conf = dedent("""
[experiment]
segments = ["regular_users_v3"]
""")
spec = AnalysisSpec.from_dict(toml.loads(conf))
configured = spec.resolve(experiments[0])
assert isinstance(configured.experiment.segments[0],
mozanalysis.segments.Segment)
Analysis("test", "test",
configured).run(current_date=dt.datetime(2020,
1,
1,
tzinfo=pytz.utc),
dry_run=True)
def test_fenix_experiments_use_right_datasets(fenix_experiments, monkeypatch):
for experiment in fenix_experiments:
called = 0
def dry_run_query(query):
nonlocal called
called = called + 1
dataset = re.sub(r"[^A-Za-z0-9_]", "_", experiment.app_id)
assert dataset in query
assert query.count(dataset) == query.count("org_mozilla")
monkeypatch.setattr("jetstream.analysis.dry_run_query", dry_run_query)
config = AnalysisSpec.default_for_experiment(experiment).resolve(
experiment)
Analysis("spam", "eggs", config).validate()
assert called == 2
def test_simple_workflow(self, cli_experiments, monkeypatch):
monkeypatch.setattr("jetstream.cli.export_metadata", Mock())
fake_analysis = Mock()
experiment = cli_experiments.experiments[0]
spec = AnalysisSpec.default_for_experiment(experiment)
strategy = cli.SerialExecutorStrategy(
project_id="spam",
dataset_id="eggs",
bucket="bucket",
analysis_class=fake_analysis,
experiment_getter=lambda: cli_experiments,
config_getter=external_config.ExternalConfigCollection,
)
config = spec.resolve(experiment)
run_date = dt.datetime(2020, 10, 31, tzinfo=UTC)
strategy.execute([(config, run_date)])
fake_analysis.assert_called_once_with("spam", "eggs", config, None)
fake_analysis().run.assert_called_once_with(run_date)
def test_busted_config_fails(self, experiments):
config = dedent("""\
[metrics]
weekly = ["bogus_metric"]
[metrics.bogus_metric]
select_expression = "SUM(fake_column)"
data_source = "clients_daily"
statistics = { bootstrap_mean = {} }
""")
spec = AnalysisSpec.from_dict(toml.loads(config))
extern = ExternalConfig(
slug="bad_experiment",
spec=spec,
last_modified=datetime.datetime.now(),
)
with pytest.raises(DryRunFailedError):
extern.validate(experiments[0])
def test_no_enrollments(self, client, project_id, static_dataset,
temporary_dataset):
experiment = Experiment(
experimenter_slug="test-experiment-2",
type="rollout",
status="Live",
start_date=dt.datetime(2020, 3, 30, tzinfo=pytz.utc),
end_date=dt.datetime(2020, 6, 1, tzinfo=pytz.utc),
proposed_enrollment=7,
branches=[
Branch(slug="a", ratio=0.5),
Branch(slug="b", ratio=0.5)
],
reference_branch="a",
features=[],
normandy_slug="test-experiment-2",
)
config = AnalysisSpec().resolve(experiment)
self.analysis_mock_run(config, static_dataset, temporary_dataset,
project_id)
query_job = client.client.query(f"""
SELECT
*
FROM `{project_id}.{temporary_dataset}.test_experiment_2_week_1`
ORDER BY enrollment_date DESC
""")
assert query_job.result().total_rows == 0
stats = client.client.list_rows(
f"{project_id}.{temporary_dataset}.statistics_test_experiment_2_week_1"
).to_dataframe()
count_by_branch = stats.query("statistic == 'count'").set_index(
"branch")
assert count_by_branch.loc["a", "point"] == 0.0
assert count_by_branch.loc["b", "point"] == 0.0
assert (client.client.get_table(
f"{project_id}.{temporary_dataset}.statistics_test_experiment_2_weekly"
) is not None)
def test_simple_workflow(self, cli_experiments):
experiment = cli_experiments.experiments[0]
spec = AnalysisSpec.default_for_experiment(experiment)
config = spec.resolve(experiment)
with mock.patch(
"jetstream.cli.submit_workflow") as submit_workflow_mock:
strategy = cli.ArgoExecutorStrategy(
"spam",
"eggs",
"bucket",
"zone",
"cluster_id",
False,
None,
None,
lambda: cli_experiments,
)
run_date = dt.datetime(2020, 10, 31, tzinfo=UTC)
strategy.execute([(config, run_date)])
submit_workflow_mock.assert_called_once_with(
project_id="spam",
zone="zone",
cluster_id="cluster_id",
workflow_file=strategy.RUN_WORKFLOW,
parameters={
"experiments": [{
"slug": "my_cool_experiment",
"dates": ["2020-10-31"]
}],
"project_id":
"spam",
"dataset_id":
"eggs",
"bucket":
"bucket",
},
monitor_status=False,
cluster_ip=None,
cluster_cert=None,
)
def from_github_repo(cls) -> "ExternalConfigCollection":
"""Pull in external config files."""
g = Github()
repo = g.get_repo(cls.JETSTREAM_CONFIG_REPO)
files = repo.get_contents("")
if isinstance(files, ContentFile):
files = [files]
configs = []
for file in files:
if file.name.endswith(".toml"):
slug = os.path.splitext(file.name)[0]
spec = AnalysisSpec.from_dict(toml.loads(file.decoded_content.decode("utf-8")))
last_modified = parser.parse(str(file.last_modified))
configs.append(ExternalConfig(slug, spec, last_modified))
return cls(configs)
def validate(self) -> None:
if self.platform not in PLATFORM_CONFIGS:
raise ValueError(f"Platform '{self.platform}' is unsupported.")
app_id = PLATFORM_CONFIGS[self.platform].app_id
dummy_experiment = jetstream.experimenter.Experiment(
experimenter_slug="dummy-experiment",
normandy_slug="dummy_experiment",
type="v6",
status="Live",
branches=[],
end_date=None,
reference_branch="control",
is_high_population=False,
start_date=dt.datetime.now(UTC),
proposed_enrollment=14,
app_id=app_id,
app_name=self.platform, # seems to be unused
)
spec = AnalysisSpec.default_for_experiment(dummy_experiment)
spec.merge_outcome(self.spec)
conf = spec.resolve(dummy_experiment)
Analysis("no project", "no dataset", conf).validate()
def test_regression_20200320():
experiment_json = r"""
{
"experiment_url": "https://experimenter.services.mozilla.com/experiments/impact-of-level-2-etp-on-a-custom-distribution/",
"type": "pref",
"name": "Impact of Level 2 ETP on a Custom Distribution",
"slug": "impact-of-level-2-etp-on-a-custom-distribution",
"public_name": "Impact of Level 2 ETP",
"status": "Live",
"start_date": 1580169600000,
"end_date": 1595721600000,
"proposed_start_date": 1580169600000,
"proposed_enrollment": null,
"proposed_duration": 180,
"normandy_slug": "pref-impact-of-level-2-etp-on-a-custom-distribution-release-72-80-bug-1607493",
"normandy_id": 906,
"other_normandy_ids": [],
"variants": [
{
"description": "",
"is_control": true,
"name": "treatment",
"ratio": 100,
"slug": "treatment",
"value": "true",
"addon_release_url": null,
"preferences": []
}
]
}
""" # noqa
experiment = ExperimentV1.from_dict(
json.loads(experiment_json)).to_experiment()
config = AnalysisSpec().resolve(experiment)
analysis = Analysis("test", "test", config)
with pytest.raises(NoEnrollmentPeriodException):
analysis.run(current_date=dt.datetime(2020, 3, 19, tzinfo=pytz.utc),
dry_run=True)
def test_metadata_with_outcomes(experiments, fake_outcome_resolver):
config_str = dedent(
"""
[metrics]
weekly = ["view_about_logins"]
[metrics.view_about_logins.statistics.bootstrap_mean]
"""
)
spec = AnalysisSpec.from_dict(toml.loads(config_str))
config = spec.resolve(experiments[5])
metadata = ExperimentMetadata.from_config(config)
assert "view_about_logins" in metadata.metrics
assert metadata.metrics["view_about_logins"].bigger_is_better
assert metadata.metrics["view_about_logins"].description != ""
assert "tastiness" in metadata.outcomes
assert "performance" in metadata.outcomes
assert "speed" in metadata.outcomes["performance"].default_metrics
assert metadata.outcomes["tastiness"].friendly_name == "Tastiness outcomes"
assert "meals_eaten" in metadata.outcomes["tastiness"].metrics
assert metadata.outcomes["tastiness"].default_metrics == []
def test_metrics(self, client, project_id, static_dataset,
temporary_dataset):
experiment = Experiment(
experimenter_slug="test-experiment",
type="rollout",
status="Live",
start_date=dt.datetime(2020, 3, 30, tzinfo=pytz.utc),
end_date=dt.datetime(2020, 6, 1, tzinfo=pytz.utc),
proposed_enrollment=7,
branches=[
Branch(slug="branch1", ratio=0.5),
Branch(slug="branch2", ratio=0.5)
],
reference_branch="branch2",
features=[],
normandy_slug="test-experiment",
)
config = AnalysisSpec().resolve(experiment)
test_clients_daily = DataSource(
name="clients_daily",
from_expr=f"`{project_id}.test_data.clients_daily`",
)
test_active_hours = Metric(
name="active_hours",
data_source=test_clients_daily,
select_expr=agg_sum("active_hours_sum"),
)
config.metrics = {
AnalysisPeriod.WEEK: [Summary(test_active_hours, BootstrapMean())]
}
self.analysis_mock_run(config, static_dataset, temporary_dataset,
project_id)
query_job = client.client.query(f"""
SELECT
*
FROM `{project_id}.{temporary_dataset}.test_experiment_week_1`
ORDER BY enrollment_date DESC
""")
expected_metrics_results = [
{
"client_id": "bbbb",
"branch": "branch2",
"enrollment_date": datetime.date(2020, 4, 3),
"num_enrollment_events": 1,
"analysis_window_start": 0,
"analysis_window_end": 6,
},
{
"client_id": "aaaa",
"branch": "branch1",
"enrollment_date": datetime.date(2020, 4, 2),
"num_enrollment_events": 1,
"analysis_window_start": 0,
"analysis_window_end": 6,
},
]
for i, row in enumerate(query_job.result()):
for k, v in expected_metrics_results[i].items():
assert row[k] == v
assert (client.client.get_table(
f"{project_id}.{temporary_dataset}.test_experiment_weekly")
is not None)
assert (client.client.get_table(
f"{project_id}.{temporary_dataset}.statistics_test_experiment_week_1"
) is not None)
stats = client.client.list_rows(
f"{project_id}.{temporary_dataset}.statistics_test_experiment_week_1"
).to_dataframe()
count_by_branch = stats.query("statistic == 'count'").set_index(
"branch")
assert count_by_branch.loc["branch1", "point"] == 1.0
assert count_by_branch.loc["branch2", "point"] == 1.0
assert (client.client.get_table(
f"{project_id}.{temporary_dataset}.statistics_test_experiment_weekly"
) is not None)
def test_logging(self, monkeypatch, client, project_id, static_dataset,
temporary_dataset):
experiment = Experiment(
experimenter_slug="test-experiment",
type="rollout",
status="Live",
start_date=dt.datetime(2020, 3, 30, tzinfo=pytz.utc),
end_date=dt.datetime(2020, 6, 1, tzinfo=pytz.utc),
proposed_enrollment=7,
branches=[
Branch(slug="branch1", ratio=0.5),
Branch(slug="branch2", ratio=0.5)
],
reference_branch="branch2",
normandy_slug="test-experiment",
is_high_population=False,
app_name="firefox_desktop",
app_id="firefox-desktop",
)
config = AnalysisSpec().resolve(experiment)
test_clients_daily = DataSource(
name="clients_daily",
from_expr=f"`{project_id}.test_data.clients_daily`",
)
test_active_hours = Metric(
name="active_hours",
data_source=test_clients_daily,
select_expression=agg_sum("active_hours_sum"),
)
config.metrics = {
AnalysisPeriod.WEEK: [
Summary(test_active_hours,
BootstrapMean(confidence_interval=10))
]
}
log_config = LogConfiguration(
log_project_id=project_id,
log_dataset_id=temporary_dataset,
log_table_id="logs",
log_to_bigquery=True,
task_profiling_log_table_id="task_profiling_logs",
task_monitoring_log_table_id="task_monitoring_logs",
capacity=1,
)
self.analysis_mock_run(monkeypatch, config, static_dataset,
temporary_dataset, project_id, log_config)
assert client.client.get_table(
f"{project_id}.{temporary_dataset}.logs") is not None
logs = list(
client.client.list_rows(f"{project_id}.{temporary_dataset}.logs"))
assert len(logs) >= 1
error_logs = [log for log in logs if log.get("log_level") == "ERROR"]
assert (
"Error while computing statistic bootstrap_mean for metric active_hours"
in error_logs[0].get("message"))
assert error_logs[0].get("log_level") == "ERROR"
def test_with_segments(self, monkeypatch, client, project_id,
static_dataset, temporary_dataset):
experiment = Experiment(
experimenter_slug="test-experiment",
type="rollout",
status="Live",
start_date=dt.datetime(2020, 3, 30, tzinfo=pytz.utc),
end_date=dt.datetime(2020, 6, 1, tzinfo=pytz.utc),
proposed_enrollment=7,
branches=[
Branch(slug="branch1", ratio=0.5),
Branch(slug="branch2", ratio=0.5)
],
reference_branch="branch2",
normandy_slug="test-experiment",
is_high_population=False,
app_name="firefox_desktop",
app_id="firefox-desktop",
)
config = AnalysisSpec().resolve(experiment)
test_clients_daily = DataSource(
name="clients_daily",
from_expr=f"`{project_id}.test_data.clients_daily`",
)
test_active_hours = Metric(
name="active_hours",
data_source=test_clients_daily,
select_expression=agg_sum("active_hours_sum"),
)
test_clients_last_seen = SegmentDataSource(
"clients_last_seen", f"`{project_id}.test_data.clients_last_seen`")
regular_user_v3 = Segment(
"regular_user_v3",
test_clients_last_seen,
"COALESCE(LOGICAL_OR(is_regular_user_v3), FALSE)",
)
config.experiment.segments = [regular_user_v3]
config.metrics = {
AnalysisPeriod.WEEK: [Summary(test_active_hours, BootstrapMean())]
}
self.analysis_mock_run(monkeypatch, config, static_dataset,
temporary_dataset, project_id)
query_job = client.client.query(f"""
SELECT
*
FROM `{project_id}.{temporary_dataset}.test_experiment_enrollments_week_1`
ORDER BY enrollment_date DESC
""")
expected_metrics_results = [
{
"client_id": "bbbb",
"branch": "branch2",
"enrollment_date": datetime.date(2020, 4, 3),
"num_enrollment_events": 1,
"analysis_window_start": 0,
"analysis_window_end": 6,
"regular_user_v3": True,
},
{
"client_id": "aaaa",
"branch": "branch1",
"enrollment_date": datetime.date(2020, 4, 2),
"num_enrollment_events": 1,
"analysis_window_start": 0,
"analysis_window_end": 6,
"regular_user_v3": False,
},
]
for i, row in enumerate(query_job.result()):
for k, v in expected_metrics_results[i].items():
assert row[k] == v
assert (client.client.get_table(
f"{project_id}.{temporary_dataset}.test_experiment_enrollments_weekly"
) is not None)
assert (client.client.get_table(
f"{project_id}.{temporary_dataset}.statistics_test_experiment_week_1"
) is not None)
stats = client.client.list_rows(
f"{project_id}.{temporary_dataset}.statistics_test_experiment_week_1"
).to_dataframe()
# Only one count per segment and branch, please
assert (stats.query(
"metric == 'identity' and statistic == 'count'").groupby(
["segment", "analysis_basis", "window_index",
"branch"]).size() == 1).all()
count_by_branch = stats.query(
"segment == 'all' and statistic == 'count'").set_index("branch")
assert count_by_branch.loc["branch1", "point"] == 1.0
assert count_by_branch.loc["branch2", "point"] == 1.0
assert count_by_branch.loc["branch2",
"analysis_basis"] == "enrollments"
def test_no_enrollments(self, monkeypatch, client, project_id,
static_dataset, temporary_dataset):
experiment = Experiment(
experimenter_slug="test-experiment-2",
type="rollout",
status="Live",
start_date=dt.datetime(2020, 3, 30, tzinfo=pytz.utc),
end_date=dt.datetime(2020, 6, 1, tzinfo=pytz.utc),
proposed_enrollment=7,
branches=[
Branch(slug="a", ratio=0.5),
Branch(slug="b", ratio=0.5)
],
reference_branch="a",
normandy_slug="test-experiment-2",
is_high_population=False,
app_name="firefox_desktop",
app_id="firefox-desktop",
)
config = AnalysisSpec().resolve(experiment)
test_clients_daily = DataSource(
name="clients_daily",
from_expr=f"`{project_id}.test_data.clients_daily`",
)
test_active_hours = Metric(
name="active_hours",
data_source=test_clients_daily,
select_expression=agg_sum("active_hours_sum"),
)
config.metrics = {
AnalysisPeriod.WEEK: [Summary(test_active_hours, BootstrapMean())]
}
self.analysis_mock_run(monkeypatch, config, static_dataset,
temporary_dataset, project_id)
query_job = client.client.query(f"""
SELECT
*
FROM `{project_id}.{temporary_dataset}.test_experiment_2_enrollments_week_1`
ORDER BY enrollment_date DESC
""")
assert query_job.result().total_rows == 0
stats = client.client.list_rows(
f"{project_id}.{temporary_dataset}.statistics_test_experiment_2_week_1"
).to_dataframe()
count_by_branch = stats.query("statistic == 'count'").set_index(
"branch")
assert count_by_branch.loc["a", "point"] == 0.0
assert count_by_branch.loc["b", "point"] == 0.0
assert count_by_branch.loc["b", "analysis_basis"] == "enrollments"
assert (client.client.get_table(
f"{project_id}.{temporary_dataset}.statistics_test_experiment_2_weekly"
) is not None)
def test_metrics_with_exposure(self, monkeypatch, client, project_id,
static_dataset, temporary_dataset):
experiment = Experiment(
experimenter_slug="test-experiment",
type="rollout",
status="Live",
start_date=dt.datetime(2020, 3, 30, tzinfo=pytz.utc),
end_date=dt.datetime(2020, 6, 1, tzinfo=pytz.utc),
proposed_enrollment=7,
branches=[
Branch(slug="branch1", ratio=0.5),
Branch(slug="branch2", ratio=0.5)
],
reference_branch="branch2",
normandy_slug="test-experiment",
is_high_population=False,
app_name="firefox_desktop",
app_id="firefox-desktop",
)
config = AnalysisSpec().resolve(experiment)
test_clients_daily = DataSource(
name="clients_daily",
from_expr=f"`{project_id}.test_data.clients_daily`",
)
test_active_hours = Metric(
name="active_hours",
data_source=test_clients_daily,
select_expression=agg_sum("active_hours_sum"),
analysis_bases=[AnalysisBasis.EXPOSURES],
)
config.metrics = {
AnalysisPeriod.WEEK: [Summary(test_active_hours, BootstrapMean())]
}
config.experiment.exposure_signal = ExposureSignal(
name="ad_exposure",
data_source=test_clients_daily,
select_expression="active_hours_sum > 0",
friendly_name="Ad exposure",
description="Clients have clicked on ad",
window_start="enrollment_start",
window_end="analysis_window_end",
)
self.analysis_mock_run(monkeypatch, config, static_dataset,
temporary_dataset, project_id)
query_job = client.client.query(f"""
SELECT
*
FROM `{project_id}.{temporary_dataset}.test_experiment_exposures_week_1`
ORDER BY enrollment_date DESC
""")
expected_metrics_results = [
{
"client_id": "bbbb",
"branch": "branch2",
"enrollment_date": datetime.date(2020, 4, 3),
"num_enrollment_events": 1,
"analysis_window_start": 0,
"analysis_window_end": 6,
},
{
"client_id": "aaaa",
"branch": "branch1",
"enrollment_date": datetime.date(2020, 4, 2),
"num_enrollment_events": 1,
"analysis_window_start": 0,
"analysis_window_end": 6,
},
]
r = query_job.result()
for i, row in enumerate(r):
for k, v in expected_metrics_results[i].items():
assert row[k] == v
assert (client.client.get_table(
f"{project_id}.{temporary_dataset}.test_experiment_exposures_weekly"
) is not None)
assert (client.client.get_table(
f"{project_id}.{temporary_dataset}.statistics_test_experiment_week_1"
) is not None)
assert (client.client.get_table(
f"{project_id}.{temporary_dataset}.statistics_test_experiment_weekly"
) is not None)
def test_export_metadata(mock_storage_client, experiments):
config_str = dedent(
"""
[experiment]
end_date = "2021-07-01"
[metrics]
weekly = ["view_about_logins", "my_cool_metric"]
[metrics.my_cool_metric]
data_source = "main"
select_expression = "{{agg_histogram_mean('payload.content.my_cool_histogram')}}"
[metrics.my_cool_metric.statistics.bootstrap_mean]
[metrics.view_about_logins.statistics.bootstrap_mean]
"""
)
spec = AnalysisSpec.from_dict(toml.loads(config_str))
config = spec.resolve(experiments[0])
mock_client = MagicMock()
mock_storage_client.return_value = mock_client
mock_bucket = MagicMock()
mock_client.get_bucket.return_value = mock_bucket
mock_blob = MagicMock()
mock_bucket.blob.return_value = mock_blob
mock_blob.upload_from_string.return_value = ""
export_metadata(config, "test_bucket", "project")
mock_client.get_bucket.assert_called_once()
mock_bucket.blob.assert_called_once()
expected = json.loads(
r"""
{
"metrics": {
"view_about_logins": {
"friendly_name": "about:logins viewers",
"description": "Counts the number of clients that viewed about:logins.\n",
"bigger_is_better": true,
"analysis_bases": ["enrollments"]
},
"my_cool_metric": {
"friendly_name": "",
"description": "",
"bigger_is_better": true,
"analysis_bases": ["enrollments"]
}
},
"outcomes": {},
"external_config": {
"end_date": "2021-07-01",
"enrollment_period": null,
"reference_branch": null,
"skip": false,
"start_date": null,
"url": """
+ '"https://github.com/mozilla/jetstream-config/blob/main/normandy-test-slug.toml"'
+ r"""},
"schema_version":"""
+ str(StatisticResult.SCHEMA_VERSION)
+ """
}
"""
)
mock_blob.upload_from_string.assert_called_once_with(
data=json.dumps(expected, sort_keys=True, indent=4), content_type="application/json"
)
def validate(self, experiment: jetstream.experimenter.Experiment) -> None:
spec = AnalysisSpec.default_for_experiment(experiment)
spec.merge(self.spec)
conf = spec.resolve(experiment)
Analysis("no project", "no dataset", conf).validate()
class TestExternalConfigIntegration:
config_str = dedent("""
[metrics]
weekly = ["view_about_logins"]
[metrics.view_about_logins.statistics.bootstrap_mean]
""")
spec = AnalysisSpec.from_dict(toml.loads(config_str))
def test_old_config(self, client, project_id, temporary_dataset):
config = ExternalConfig(
slug="new_table",
spec=self.spec,
last_modified=pytz.UTC.localize(datetime.datetime.utcnow() -
datetime.timedelta(days=1)),
)
# table created after config loaded
client.client.create_table(f"{temporary_dataset}.new_table_day1")
client.add_labels_to_table(
"new_table_day1",
{"last_updated": client._current_timestamp_label()},
)
config_collection = ExternalConfigCollection([config])
updated_configs = config_collection.updated_configs(
project_id, temporary_dataset)
assert len(updated_configs) == 0
def test_updated_config(self, client, temporary_dataset, project_id):
config = ExternalConfig(
slug="old_table",
spec=self.spec,
last_modified=pytz.UTC.localize(datetime.datetime.utcnow() +
datetime.timedelta(days=1)),
)
client.client.create_table(f"{temporary_dataset}.old_table_day1")
client.add_labels_to_table(
"old_table_day1",
{"last_updated": client._current_timestamp_label()},
)
client.client.create_table(f"{temporary_dataset}.old_table_day2")
client.add_labels_to_table(
"old_table_day2",
{"last_updated": client._current_timestamp_label()},
)
config_collection = ExternalConfigCollection([config])
updated_configs = config_collection.updated_configs(
project_id, temporary_dataset)
assert len(updated_configs) == 1
assert updated_configs[0].slug == config.slug
def test_updated_config_while_analysis_active(self, client,
temporary_dataset,
project_id):
client.client.create_table(f"{temporary_dataset}.active_table_day0")
client.add_labels_to_table(
"active_table_day0",
{"last_updated": client._current_timestamp_label()},
)
client.client.create_table(f"{temporary_dataset}.active_table_day1")
client.add_labels_to_table(
"active_table_day1",
{"last_updated": client._current_timestamp_label()},
)
config = ExternalConfig(
slug="active_table",
spec=self.spec,
last_modified=pytz.UTC.localize(datetime.datetime.utcnow()),
)
client.client.create_table(f"{temporary_dataset}.active_table_day2")
client.add_labels_to_table(
"active_table_day2",
{"last_updated": client._current_timestamp_label()},
)
client.client.create_table(f"{temporary_dataset}.active_table_weekly")
client.add_labels_to_table(
"active_table_weekly",
{"last_updated": client._current_timestamp_label()},
)
config_collection = ExternalConfigCollection([config])
updated_configs = config_collection.updated_configs(
project_id, temporary_dataset)
assert len(updated_configs) == 1
assert updated_configs[0].slug == config.slug
def test_new_config_without_a_table_is_marked_changed(
self, client, temporary_dataset, project_id):
config = ExternalConfig(
slug="my_cool_experiment",
spec=self.spec,
last_modified=pytz.UTC.localize(datetime.datetime.utcnow()),
)
config_collection = ExternalConfigCollection([config])
updated_configs = config_collection.updated_configs(
project_id, temporary_dataset)
assert [updated.slug
for updated in updated_configs] == ["my_cool_experiment"]
def test_valid_config_validates(self, experiments):
extern = ExternalConfig(
slug="cool_experiment",
spec=self.spec,
last_modified=datetime.datetime.now(),
)
extern.validate(experiments[0])
def test_busted_config_fails(self, experiments):
config = dedent("""\
[metrics]
weekly = ["bogus_metric"]
[metrics.bogus_metric]
select_expression = "SUM(fake_column)"
data_source = "clients_daily"
statistics = { bootstrap_mean = {} }
""")
spec = AnalysisSpec.from_dict(toml.loads(config))
extern = ExternalConfig(
slug="bad_experiment",
spec=spec,
last_modified=datetime.datetime.now(),
)
with pytest.raises(DryRunFailedError):
extern.validate(experiments[0])
def test_valid_outcome_validates(self):
config = dedent("""\
friendly_name = "Fred"
description = "Just your average paleolithic dad."
[metrics.rocks_mined]
select_expression = "COALESCE(SUM(pings_aggregated_by_this_row), 0)"
data_source = "clients_daily"
statistics = { bootstrap_mean = {} }
friendly_name = "Rocks mined"
description = "Number of rocks mined at the quarry"
""")
spec = OutcomeSpec.from_dict(toml.loads(config))
extern = ExternalOutcome(
slug="good_outcome",
spec=spec,
platform="firefox_desktop",
commit_hash="0000000",
)
extern.validate()
def test_busted_outcome_fails(self):
config = dedent("""\
friendly_name = "Fred"
description = "Just your average paleolithic dad."
[metrics.rocks_mined]
select_expression = "COALESCE(SUM(fake_column_whoop_whoop), 0)"
data_source = "clients_daily"
statistics = { bootstrap_mean = {} }
friendly_name = "Rocks mined"
description = "Number of rocks mined at the quarry"
""")
spec = OutcomeSpec.from_dict(toml.loads(config))
extern = ExternalOutcome(
slug="bogus_outcome",
spec=spec,
platform="firefox_desktop",
commit_hash="0000000",
)
with pytest.raises(DryRunFailedError):
extern.validate()
def test_is_high_population_check(experiments):
x = experiments[3]
config = AnalysisSpec.default_for_experiment(x).resolve(x)
with pytest.raises(HighPopulationException):
Analysis("spam", "eggs", config).check_runnable()