def test_window_limit_to_int(self):
time_limits = mozanalysis.experiment.TimeLimits(
first_enrollment_date="2019-01-05",
last_enrollment_date="2019-01-06",
analysis_windows=(mozanalysis.experiment.AnalysisWindow(2, 4),),
first_date_data_required="2019-01-07",
last_date_data_required="2019-01-10",
)
exposure_signal = ExposureSignal(
name="ad_exposure",
data_source=mozanalysis.metrics.desktop.search_clients_daily,
select_expression="ad_click > 0",
friendly_name="Ad exposure",
description="Clients have clicked on ad",
)
assert exposure_signal._window_limit_to_int(None, time_limits) is None
assert exposure_signal._window_limit_to_int(11, time_limits) == 11
assert exposure_signal._window_limit_to_int("enrollment_start", time_limits) == 0
assert exposure_signal._window_limit_to_int("enrollment_end", time_limits) == 1
assert exposure_signal._window_limit_to_int("analysis_window_end", time_limits) == 5
assert exposure_signal._window_limit_to_int("analysis_window_start", time_limits) == 3
with pytest.raises(Exception):
exposure_signal._window_limit_to_int("invalid", time_limits)
def test_exposure_signal(self, experiments):
config_str = dedent("""
[experiment.exposure_signal]
name = "ad_exposure"
data_source = "search_clients_daily"
select_expression = "ad_click > 0"
friendly_name = "Ad exposure"
description = "Clients have clicked on ad"
[metrics]
weekly = ["ad_clicks"]
[metrics.ad_clicks.statistics.bootstrap_mean]
num_samples = 10
""")
spec = config.AnalysisSpec.from_dict(toml.loads(config_str))
assert spec.experiment.exposure_signal.window_start is None
assert spec.experiment.exposure_signal.window_end is None
cfg = spec.resolve(experiments[0])
assert cfg.experiment.exposure_signal == ExposureSignal(
name="ad_exposure",
data_source=mozanalysis.metrics.desktop.search_clients_daily,
select_expression="ad_click > 0",
friendly_name="Ad exposure",
description="Clients have clicked on ad",
)
def test_invalid_window(self):
with pytest.raises(Exception):
ExposureSignal(
name="ad_exposure",
data_source=mozanalysis.metrics.desktop.search_clients_daily,
select_expression="ad_click > 0",
friendly_name="Ad exposure",
description="Clients have clicked on ad",
window_start="invalid",
)
def resolve(self, spec: "AnalysisSpec",
experiment: ExperimentConfiguration) -> ExposureSignal:
return ExposureSignal(
name=self.name,
data_source=self.data_source.resolve(spec, experiment=experiment),
select_expression=self.select_expression,
friendly_name=self.friendly_name,
description=self.description,
window_start=self.window_start,
window_end=self.window_end,
)
def test_create_exposure_signal(self):
exposure_signal = ExposureSignal(
name="ad_exposure",
data_source=mozanalysis.metrics.desktop.search_clients_daily,
select_expression="ad_click > 0",
friendly_name="Ad exposure",
description="Clients have clicked on ad",
window_start=12,
)
assert exposure_signal.window_end is None
assert exposure_signal.window_start == 12
def test_exposure_signal_to_mozanalysis(self):
time_limits = mozanalysis.experiment.TimeLimits(
first_enrollment_date="2019-01-05",
last_enrollment_date="2019-01-06",
analysis_windows=(mozanalysis.experiment.AnalysisWindow(2, 4),),
first_date_data_required="2019-01-07",
last_date_data_required="2019-01-10",
)
exposure_signal = ExposureSignal(
name="ad_exposure",
data_source=mozanalysis.metrics.desktop.search_clients_daily,
select_expression="ad_click > 0",
friendly_name="Ad exposure",
description="Clients have clicked on ad",
)
assert isinstance(
exposure_signal.to_mozanalysis_exposure_signal(time_limits),
mozanalysis.exposure.ExposureSignal,
)
assert exposure_signal.to_mozanalysis_exposure_signal(time_limits).build_query(time_limits)
def test_metrics_with_exposure(self, monkeypatch, client, project_id,
static_dataset, temporary_dataset):
experiment = Experiment(
experimenter_slug="test-experiment",
type="rollout",
status="Live",
start_date=dt.datetime(2020, 3, 30, tzinfo=pytz.utc),
end_date=dt.datetime(2020, 6, 1, tzinfo=pytz.utc),
proposed_enrollment=7,
branches=[
Branch(slug="branch1", ratio=0.5),
Branch(slug="branch2", ratio=0.5)
],
reference_branch="branch2",
normandy_slug="test-experiment",
is_high_population=False,
app_name="firefox_desktop",
app_id="firefox-desktop",
)
config = AnalysisSpec().resolve(experiment)
test_clients_daily = DataSource(
name="clients_daily",
from_expr=f"`{project_id}.test_data.clients_daily`",
)
test_active_hours = Metric(
name="active_hours",
data_source=test_clients_daily,
select_expression=agg_sum("active_hours_sum"),
analysis_bases=[AnalysisBasis.EXPOSURES],
)
config.metrics = {
AnalysisPeriod.WEEK: [Summary(test_active_hours, BootstrapMean())]
}
config.experiment.exposure_signal = ExposureSignal(
name="ad_exposure",
data_source=test_clients_daily,
select_expression="active_hours_sum > 0",
friendly_name="Ad exposure",
description="Clients have clicked on ad",
window_start="enrollment_start",
window_end="analysis_window_end",
)
self.analysis_mock_run(monkeypatch, config, static_dataset,
temporary_dataset, project_id)
query_job = client.client.query(f"""
SELECT
*
FROM `{project_id}.{temporary_dataset}.test_experiment_exposures_week_1`
ORDER BY enrollment_date DESC
""")
expected_metrics_results = [
{
"client_id": "bbbb",
"branch": "branch2",
"enrollment_date": datetime.date(2020, 4, 3),
"num_enrollment_events": 1,
"analysis_window_start": 0,
"analysis_window_end": 6,
},
{
"client_id": "aaaa",
"branch": "branch1",
"enrollment_date": datetime.date(2020, 4, 2),
"num_enrollment_events": 1,
"analysis_window_start": 0,
"analysis_window_end": 6,
},
]
r = query_job.result()
for i, row in enumerate(r):
for k, v in expected_metrics_results[i].items():
assert row[k] == v
assert (client.client.get_table(
f"{project_id}.{temporary_dataset}.test_experiment_exposures_weekly"
) is not None)
assert (client.client.get_table(
f"{project_id}.{temporary_dataset}.statistics_test_experiment_week_1"
) is not None)
assert (client.client.get_table(
f"{project_id}.{temporary_dataset}.statistics_test_experiment_weekly"
) is not None)