def test_ts_time_limits_create_not_enough_data():
with pytest.raises(ValueError):
TimeLimits.for_ts(
first_enrollment_date="2019-01-01",
last_date_full_data="2019-01-13",
time_series_period="weekly",
num_dates_enrollment=8,
)
def test_metrics_query_based_on_exposure():
exp = Experiment("slug", "2019-01-01", 8)
tl = TimeLimits.for_ts(
first_enrollment_date="2019-01-01",
last_date_full_data="2019-03-01",
time_series_period="weekly",
num_dates_enrollment=8,
)
enrollments_sql = exp.build_enrollments_query(
time_limits=tl, enrollments_query_type="fenix-fallback")
sql_lint(enrollments_sql)
metrics_sql = exp.build_metrics_query(
metric_list=[
m for m in mozanalysis.metrics.fenix.__dict__.values()
if isinstance(m, Metric)
],
time_limits=tl,
enrollments_table="enrollments",
analysis_basis=AnalysisBasis.EXPOSURES,
)
sql_lint(metrics_sql)
assert "e.exposure_date" in metrics_sql
def test_exposure_signal_query_custom_windows():
exp = Experiment("slug", "2019-01-01", 8, app_id="my_cool_app")
tl = TimeLimits.for_ts(
first_enrollment_date="2019-01-01",
last_date_full_data="2019-03-01",
time_series_period="weekly",
num_dates_enrollment=8,
)
enrollment_sql = exp.build_enrollments_query(
time_limits=tl,
enrollments_query_type="glean-event",
exposure_signal=ExposureSignal(
name="exposures",
data_source=mozanalysis.metrics.fenix.baseline,
select_expr="metrics.counter.events_total_uri_count > 0",
friendly_name="URI visited exposure",
description="Exposed when URI visited",
window_start=1,
window_end=3,
),
)
sql_lint(enrollment_sql)
assert "exposures" in enrollment_sql
assert "metrics.counter.events_total_uri_count > 0" in enrollment_sql
assert "DATE_ADD('2019-01-01', INTERVAL 1 DAY)" in enrollment_sql
assert "DATE_ADD('2019-01-01', INTERVAL 3 DAY)" in enrollment_sql
def test_add_analysis_windows_to_enrollments(spark):
exp = Experiment('a-stub', '20190101', num_dates_enrollment=8)
enrollments = exp.get_enrollments(spark,
_get_enrollment_view(slug="a-stub"))
assert enrollments.count() == 3
tl = TimeLimits.for_ts(
first_enrollment_date=exp.start_date,
last_date_full_data='20190114',
time_series_period='daily',
num_dates_enrollment=exp.num_dates_enrollment,
)
assert len(tl.analysis_windows) == 7
new_enrollments = exp._add_analysis_windows_to_enrollments(enrollments, tl)
nep = new_enrollments.toPandas()
assert len(nep) == enrollments.count() * len(tl.analysis_windows)
a = nep[nep['client_id'] == 'aaaa']
assert len(a) == len(tl.analysis_windows)
assert (a.mozanalysis_analysis_window_start.sort_values() == np.arange(
len(tl.analysis_windows))).all()
assert (a.mozanalysis_analysis_window_end.sort_values() == np.arange(
len(tl.analysis_windows))).all()
def test_segments_megaquery_not_detectably_malformed():
exp = Experiment("slug", "2019-01-01", 8)
tl = TimeLimits.for_ts(
first_enrollment_date="2019-01-01",
last_date_full_data="2019-03-01",
time_series_period="weekly",
num_dates_enrollment=8,
)
enrollments_sql = exp.build_enrollments_query(
time_limits=tl,
segment_list=[s for s in msd.__dict__.values() if isinstance(s, msd.Segment)],
enrollments_query_type="normandy",
)
sql_lint(enrollments_sql)
metrics_sql = exp.build_metrics_query(
metric_list=[m for m in mad.__dict__.values() if isinstance(m, mad.Metric)],
time_limits=tl,
enrollments_table="enrollments",
)
sql_lint(metrics_sql)
def test_ts_time_limits_create3():
tl = TimeLimits.for_ts(first_enrollment_date='2019-01-01',
last_date_full_data='2019-01-15',
time_series_period='weekly',
num_dates_enrollment=8)
assert tl.first_enrollment_date == '2019-01-01'
assert tl.last_enrollment_date == '2019-01-08'
assert len(tl.analysis_windows) == 1
assert tl.analysis_windows[0].start == 0
assert tl.analysis_windows[0].end == 6
assert tl.first_date_data_required == '2019-01-01'
assert tl.last_date_data_required == '2019-01-14'
def test_ts_time_limits_create3():
tl = TimeLimits.for_ts(
first_enrollment_date="2019-01-01",
last_date_full_data="2019-01-15",
time_series_period="weekly",
num_dates_enrollment=8,
)
assert tl.first_enrollment_date == "2019-01-01"
assert tl.last_enrollment_date == "2019-01-08"
assert len(tl.analysis_windows) == 1
assert tl.analysis_windows[0].start == 0
assert tl.analysis_windows[0].end == 6
assert tl.first_date_data_required == "2019-01-01"
assert tl.last_date_data_required == "2019-01-14"
def test_megaquery_not_detectably_malformed():
exp = Experiment('slug', '2019-01-01', 8)
tl = TimeLimits.for_ts(first_enrollment_date='2019-01-01',
last_date_full_data='2019-03-01',
time_series_period='weekly',
num_dates_enrollment=8)
sql = exp.build_query(
metric_list=[
m for m in mad.__dict__.values() if isinstance(m, mad.Metric)
],
time_limits=tl,
enrollments_query_type='normandy',
)
sql_lint(sql)
def test_query_not_detectably_malformed():
exp = Experiment('slug', '2019-01-01', 8)
tl = TimeLimits.for_ts(
first_enrollment_date='2019-01-01',
last_date_full_data='2019-03-01',
time_series_period='weekly',
num_dates_enrollment=8
)
sql = exp.build_query(
metric_list=[],
time_limits=tl,
enrollments_query_type='normandy',
)
sql_lint(sql)
def test_firefox_ios_klar_app_id_propagation():
exp = Experiment("slug", "2019-01-01", 8, app_id="my_cool_app")
tl = TimeLimits.for_ts(
first_enrollment_date="2019-01-01",
last_date_full_data="2019-03-01",
time_series_period="weekly",
num_dates_enrollment=8,
)
sds = SegmentDataSource(
name="cool_data_source",
from_expr="`moz-fx-data-shared-prod`.{dataset}.cool_table",
default_dataset="org_mozilla_ios_klar",
)
segment = Segment(
name="cool_segment",
select_expr="COUNT(*)",
data_source=sds,
)
enrollments_sql = exp.build_enrollments_query(
time_limits=tl,
segment_list=[segment],
enrollments_query_type="glean-event",
)
sql_lint(enrollments_sql)
metrics_sql = exp.build_metrics_query(
metric_list=[
m for m in mozanalysis.metrics.klar_ios.__dict__.values()
if isinstance(m, Metric)
],
time_limits=tl,
enrollments_table="enrollments",
)
sql_lint(metrics_sql)
assert "org_mozilla_ios_klar" not in enrollments_sql
assert "my_cool_app" in enrollments_sql
sql_lint(metrics_sql)
def test_query_not_detectably_malformed():
exp = Experiment('slug', '2019-01-01', 8)
tl = TimeLimits.for_ts(first_enrollment_date='2019-01-01',
last_date_full_data='2019-03-01',
time_series_period='weekly',
num_dates_enrollment=8)
sql = exp.build_query(
metric_list=[],
time_limits=tl,
enrollments_query_type='normandy',
)
# This query is actually slightly malformed, due to a trailing comma.
# We should add a metric here if the linter ever improves.
sql_lint(sql)
def test_exposure_query():
exp = Experiment("slug", "2019-01-01", 8, app_id="my_cool_app")
tl = TimeLimits.for_ts(
first_enrollment_date="2019-01-01",
last_date_full_data="2019-03-01",
time_series_period="weekly",
num_dates_enrollment=8,
)
enrollment_sql = exp.build_enrollments_query(
time_limits=tl,
enrollments_query_type="glean-event",
)
sql_lint(enrollment_sql)
assert "exposures" in enrollment_sql
def test_query_not_detectably_malformed_fenix_fallback():
exp = Experiment("slug", "2019-01-01", 8)
tl = TimeLimits.for_ts(
first_enrollment_date="2019-01-01",
last_date_full_data="2019-03-01",
time_series_period="weekly",
num_dates_enrollment=8,
)
enrollments_sql = exp.build_enrollments_query(
time_limits=tl, enrollments_query_type="fenix-fallback")
sql_lint(enrollments_sql)
metrics_sql = exp.build_metrics_query(
metric_list=[],
time_limits=tl,
enrollments_table="enrollments",
)
sql_lint(metrics_sql)
def test_metrics_query_with_exposure_signal_custom_windows():
exp = Experiment("slug", "2019-01-01", 8)
tl = TimeLimits.for_ts(
first_enrollment_date="2019-01-01",
last_date_full_data="2019-03-01",
time_series_period="weekly",
num_dates_enrollment=8,
)
enrollments_sql = exp.build_enrollments_query(
time_limits=tl, enrollments_query_type="fenix-fallback")
sql_lint(enrollments_sql)
metrics_sql = exp.build_metrics_query(
metric_list=[
m for m in mozanalysis.metrics.fenix.__dict__.values()
if isinstance(m, Metric)
],
time_limits=tl,
enrollments_table="enrollments",
analysis_basis=AnalysisBasis.EXPOSURES,
exposure_signal=ExposureSignal(
name="exposures",
data_source=mozanalysis.metrics.fenix.baseline,
select_expr="metrics.counter.events_total_uri_count > 0",
friendly_name="URI visited exposure",
description="Exposed when URI visited",
window_start=1,
window_end=3,
),
)
sql_lint(metrics_sql)
assert "DATE_ADD('2019-01-01', INTERVAL 1 DAY)" in metrics_sql
assert "DATE_ADD('2019-01-01', INTERVAL 3 DAY)" in metrics_sql
def test_ts_time_limits_create_not_enough_data():
with pytest.raises(ValueError):
TimeLimits.for_ts(first_enrollment_date='20190101',
last_date_full_data='20190113',
time_series_period='weekly',
num_dates_enrollment=8)
def _get_timelimits_if_ready(
self, period: AnalysisPeriod,
current_date: datetime) -> Optional[TimeLimits]:
"""
Returns a TimeLimits instance if experiment is due for analysis.
Otherwise returns None.
"""
prior_date = current_date - timedelta(days=1)
prior_date_str = prior_date.strftime("%Y-%m-%d")
current_date_str = current_date.strftime("%Y-%m-%d")
if not self.config.experiment.proposed_enrollment:
self.logger.info("Skipping %s; no enrollment period",
self.config.experiment.slug)
return None
dates_enrollment = self.config.experiment.proposed_enrollment + 1
if self.config.experiment.start_date is None:
return None
time_limits_args = {
"first_enrollment_date":
self.config.experiment.start_date.strftime("%Y-%m-%d"),
"num_dates_enrollment":
dates_enrollment,
}
if period != AnalysisPeriod.OVERALL:
try:
current_time_limits = TimeLimits.for_ts(
last_date_full_data=current_date_str,
time_series_period=period.adjective,
**time_limits_args,
)
except ValueError:
# There are no analysis windows yet.
# TODO: Add a more specific check.
return None
try:
prior_time_limits = TimeLimits.for_ts(
last_date_full_data=prior_date_str,
time_series_period=period.adjective,
**time_limits_args,
)
except ValueError:
# We have an analysis window today, and we didn't yesterday,
# so we must have just closed our first window.
return current_time_limits
if len(current_time_limits.analysis_windows) == len(
prior_time_limits.analysis_windows):
# No new data today
return None
return current_time_limits
# Period is OVERALL
if self.config.experiment.end_date != prior_date:
return None
return TimeLimits.for_single_analysis_window(
last_date_full_data=prior_date_str,
analysis_start_days=0,
analysis_length_dates=(self.config.experiment.end_date -
self.config.experiment.start_date).days -
dates_enrollment + 1,
**time_limits_args,
)
def _get_timelimits_if_ready(
self, period: AnalysisPeriod, current_date: datetime
) -> Optional[TimeLimits]:
"""
Returns a TimeLimits instance if experiment is due for analysis.
Otherwise returns None.
"""
prior_date = current_date - timedelta(days=1)
prior_date_str = prior_date.strftime("%Y-%m-%d")
current_date_str = current_date.strftime("%Y-%m-%d")
dates_enrollment = self.config.experiment.proposed_enrollment + 1
if self.config.experiment.start_date is None:
return None
time_limits_args = {
"first_enrollment_date": self.config.experiment.start_date.strftime("%Y-%m-%d"),
"num_dates_enrollment": dates_enrollment,
}
if period != AnalysisPeriod.OVERALL:
try:
current_time_limits = TimeLimits.for_ts(
last_date_full_data=current_date_str,
time_series_period=period.mozanalysis_label,
**time_limits_args,
)
except ValueError:
# There are no analysis windows yet.
# TODO: Add a more specific check.
return None
try:
prior_time_limits = TimeLimits.for_ts(
last_date_full_data=prior_date_str,
time_series_period=period.mozanalysis_label,
**time_limits_args,
)
except ValueError:
# We have an analysis window today, and we didn't yesterday,
# so we must have just closed our first window.
return current_time_limits
if len(current_time_limits.analysis_windows) == len(prior_time_limits.analysis_windows):
# No new data today
return None
return current_time_limits
assert period == AnalysisPeriod.OVERALL
if (
self.config.experiment.end_date is None
or self.config.experiment.end_date.date() != current_date.date()
or self.config.experiment.status != "Complete"
):
return None
if self.config.experiment.end_date is None:
return None
analysis_length_dates = (
(self.config.experiment.end_date - self.config.experiment.start_date).days
- dates_enrollment
+ 1
)
if analysis_length_dates < 0:
raise errors.EnrollmentLongerThanAnalysisException(self.config.experiment.normandy_slug)
return TimeLimits.for_single_analysis_window(
last_date_full_data=prior_date_str,
analysis_start_days=0,
analysis_length_dates=analysis_length_dates,
**time_limits_args,
)