def test_time_limits_create5():
# But not an 8 day window
with pytest.raises(ValueError):
TimeLimits.for_single_analysis_window(
first_enrollment_date="2019-01-01",
last_date_full_data="2019-01-14",
analysis_start_days=0,
analysis_length_dates=8,
num_dates_enrollment=8,
)
def test_time_limits_has_right_date_in_error_message():
msg_re = r"until we have data for 2020-03-30."
with pytest.raises(ValueError, match=msg_re):
TimeLimits.for_single_analysis_window(
first_enrollment_date="2020-03-03",
last_date_full_data="2020-03-23",
analysis_start_days=0,
analysis_length_dates=21,
num_dates_enrollment=8,
)
def test_no_analysis_exception_when_shared_parent_dataframe(spark):
# Check that we don't fall victim to
# https://issues.apache.org/jira/browse/SPARK-10925
df = spark.createDataFrame(
[ # Just need the schema, really
['someone', '20190102', 'fake', 1],
],
[
"client_id",
"submission_date_s3",
"branch",
"some_value",
])
enrollments = df.groupby('client_id', 'branch').agg(
F.min('submission_date_s3').alias('enrollment_date'))
exp = Experiment('a-stub', '20180101')
time_limits = TimeLimits.for_single_analysis_window(
exp.start_date,
last_date_full_data='20190522',
analysis_start_days=28,
analysis_length_dates=7)
enrollments = exp._add_analysis_windows_to_enrollments(
enrollments, time_limits)
exp._get_results_for_one_data_source(
enrollments,
df,
[F.max(F.col('some_value'))],
)
def test_process_enrollments(spark):
exp = Experiment('a-stub', '20190101')
enrollments = exp.get_enrollments(spark,
_get_enrollment_view(slug="a-stub"))
assert enrollments.count() == 4
# With final data collected on '20190114', we have 7 dates of data
# for 'cccc' enrolled on '20190108' but not for 'dddd' enrolled on
# '20190109'.
tl = TimeLimits.for_single_analysis_window(
first_enrollment_date=exp.start_date,
last_date_full_data='20190114',
analysis_start_days=0,
analysis_length_dates=7,
num_dates_enrollment=exp.num_dates_enrollment)
assert tl.last_enrollment_date == '20190108'
assert len(tl.analysis_windows) == 1
assert tl.analysis_windows[0].end == 6
pe = exp._process_enrollments(enrollments, tl)
assert pe.count() == 3
pe = exp._process_enrollments(enrollments.alias('main_summary'), tl)
assert pe.select(F.col('enrollments.enrollment_date'))
with pytest.raises(AnalysisException):
assert pe.select(F.col('main_summary.enrollment_date'))
def test_process_data_source_df(spark):
start_date = '20190101'
exp_8d = Experiment('experiment-with-8-day-cohort', start_date, 8)
data_source_df = _get_data_source_df(spark)
end_date = '20190114'
# Are the fixtures sufficiently complicated that we're actually testing
# things?
assert _simple_return_agg_date(F.min, data_source_df) < start_date
assert _simple_return_agg_date(F.max, data_source_df) > end_date
tl_03 = TimeLimits.for_single_analysis_window(
first_enrollment_date=exp_8d.start_date,
last_date_full_data=end_date,
analysis_start_days=0,
analysis_length_dates=3,
num_dates_enrollment=exp_8d.num_dates_enrollment)
assert tl_03.first_date_data_required == start_date
assert tl_03.last_date_data_required == '20190110'
proc_ds = exp_8d._process_data_source_df(data_source_df, tl_03)
assert _simple_return_agg_date(F.min,
proc_ds) == tl_03.first_date_data_required
assert _simple_return_agg_date(F.max,
proc_ds) == tl_03.last_date_data_required
tl_23 = TimeLimits.for_single_analysis_window(
first_enrollment_date=exp_8d.start_date,
last_date_full_data=end_date,
analysis_start_days=2,
analysis_length_dates=3,
num_dates_enrollment=exp_8d.num_dates_enrollment)
assert tl_23.first_date_data_required == add_days(start_date, 2)
assert tl_23.last_date_data_required == '20190112'
p_ds_2 = exp_8d._process_data_source_df(data_source_df, tl_23)
assert _simple_return_agg_date(F.min,
p_ds_2) == tl_23.first_date_data_required
assert _simple_return_agg_date(F.max,
p_ds_2) == tl_23.last_date_data_required
assert proc_ds.select(F.col('data_source.client_id'))
with pytest.raises(AnalysisException):
assert data_source_df.select(F.col('data_source.client_id'))
def test_time_limits_create2():
# We don't have 14 dates of data for an 8-day cohort:
with pytest.raises(ValueError):
TimeLimits.for_single_analysis_window(
first_enrollment_date="2019-01-01",
last_date_full_data="2019-01-14",
analysis_start_days=0,
analysis_length_dates=14,
num_dates_enrollment=8,
)
# We don't have 15 full dates of data for any users
with pytest.raises(AssertionError):
TimeLimits.for_single_analysis_window(
first_enrollment_date="2019-01-01",
last_date_full_data="2019-01-14",
analysis_start_days=0,
analysis_length_dates=15,
)
def test_time_limits_create6():
# Of course the flexi-experiment has data for a 1 day window
tl = TimeLimits.for_single_analysis_window(
first_enrollment_date="2019-01-01",
last_date_full_data="2019-01-14",
analysis_start_days=0,
analysis_length_dates=1,
)
assert tl.first_enrollment_date == "2019-01-01"
assert tl.last_enrollment_date == "2019-01-14"
assert len(tl.analysis_windows) == 1
assert tl.analysis_windows[0].start == 0
assert tl.analysis_windows[0].end == 0
assert tl.first_date_data_required == "2019-01-01"
assert tl.last_date_data_required == "2019-01-14"
def test_time_limits_create7():
# If the analysis starts later, so does the data source
tl = TimeLimits.for_single_analysis_window(
first_enrollment_date="2019-01-01",
last_date_full_data="2019-01-14",
analysis_start_days=7,
analysis_length_dates=1,
)
assert tl.first_enrollment_date == "2019-01-01"
assert tl.last_enrollment_date == "2019-01-07"
assert len(tl.analysis_windows) == 1
assert tl.analysis_windows[0].start == 7
assert tl.analysis_windows[0].end == 7
assert tl.first_date_data_required == "2019-01-08"
assert tl.last_date_data_required == "2019-01-14"
def test_time_limits_create3():
# For the 8-day cohort We have enough data for a 7 day window
tl = TimeLimits.for_single_analysis_window(
first_enrollment_date="2019-01-01",
last_date_full_data="2019-01-14",
analysis_start_days=0,
analysis_length_dates=7,
num_dates_enrollment=8,
)
assert tl.first_enrollment_date == "2019-01-01"
assert tl.last_enrollment_date == "2019-01-08"
assert len(tl.analysis_windows) == 1
assert tl.analysis_windows[0].start == 0
assert tl.analysis_windows[0].end == 6
assert tl.first_date_data_required == "2019-01-01"
assert tl.last_date_data_required == "2019-01-14"
def test_time_limits_create4():
# Or a 2 day window
tl = TimeLimits.for_single_analysis_window(
first_enrollment_date="2019-01-01",
last_date_full_data="2019-01-14",
analysis_start_days=0,
analysis_length_dates=2,
num_dates_enrollment=8,
)
assert tl.first_enrollment_date == "2019-01-01"
assert tl.last_enrollment_date == "2019-01-08"
assert len(tl.analysis_windows) == 1
assert tl.analysis_windows[0].start == 0
assert tl.analysis_windows[0].end == 1
assert tl.first_date_data_required == "2019-01-01"
assert tl.last_date_data_required == "2019-01-09"
def validate(self) -> None:
self.check_runnable()
dates_enrollment = self.config.experiment.proposed_enrollment + 1
if self.config.experiment.end_date is not None:
end_date = self.config.experiment.end_date
analysis_length_dates = (
(end_date - self.config.experiment.start_date).days -
dates_enrollment + 1)
else:
analysis_length_dates = 21 # arbitrary
end_date = self.config.experiment.start_date + timedelta(
days=analysis_length_dates + dates_enrollment - 1)
if analysis_length_dates < 0:
logging.error(
"Proposed enrollment longer than analysis dates length:" +
f"{self.config.experiment.normandy_slug}")
raise Exception("Cannot validate experiment")
limits = TimeLimits.for_single_analysis_window(
last_date_full_data=end_date.strftime("%Y-%m-%d"),
analysis_start_days=0,
analysis_length_dates=analysis_length_dates,
first_enrollment_date=self.config.experiment.start_date.strftime(
"%Y-%m-%d"),
num_dates_enrollment=dates_enrollment,
)
exp = mozanalysis.experiment.Experiment(
experiment_slug=self.config.experiment.normandy_slug,
start_date=self.config.experiment.start_date.strftime("%Y-%m-%d"),
)
metrics = set()
for v in self.config.metrics.values():
metrics |= {m.metric for m in v}
sql = exp.build_query(
metrics,
limits,
"normandy",
self.config.experiment.enrollment_query,
)
dry_run_query(sql)
def test_time_limits_create1():
# When we have complete data for 2019-01-14...
# ...We have 14 dates of data for those who enrolled on the 1st
tl = TimeLimits.for_single_analysis_window(
first_enrollment_date="2019-01-01",
last_date_full_data="2019-01-14",
analysis_start_days=0,
analysis_length_dates=14,
)
assert tl.first_enrollment_date == "2019-01-01"
assert tl.last_enrollment_date == "2019-01-01"
assert len(tl.analysis_windows) == 1
assert tl.analysis_windows[0].start == 0
assert tl.analysis_windows[0].end == 13
assert tl.first_date_data_required == "2019-01-01"
assert tl.last_date_data_required == "2019-01-14"
def dry_run_query(exp_path):
report = validate_schema(op.join(exp_path, "report.json"))
metric_list = _make_metric_list(report)
exp = Experiment(experiment_slug=report["experiment_slug"],
start_date=report["start_date"],
num_dates_enrollment=report["num_dates_enrollment"])
# create an archive of the sql generating analysis
time_limits = TimeLimits.for_single_analysis_window(
first_enrollment_date=report['start_date'],
last_date_full_data=report['last_date_full_data'],
analysis_start_days=report['analysis_start_days'],
analysis_length_dates=report['analysis_length_days'],
num_dates_enrollment=report['num_dates_enrollment'])
query = exp.build_query(metric_list=metric_list,
time_limits=time_limits,
enrollments_query_type='normandy')
return query
def _get_timelimits_if_ready(
self, period: AnalysisPeriod,
current_date: datetime) -> Optional[TimeLimits]:
"""
Returns a TimeLimits instance if experiment is due for analysis.
Otherwise returns None.
"""
prior_date = current_date - timedelta(days=1)
prior_date_str = prior_date.strftime("%Y-%m-%d")
current_date_str = current_date.strftime("%Y-%m-%d")
if not self.config.experiment.proposed_enrollment:
self.logger.info("Skipping %s; no enrollment period",
self.config.experiment.slug)
return None
dates_enrollment = self.config.experiment.proposed_enrollment + 1
if self.config.experiment.start_date is None:
return None
time_limits_args = {
"first_enrollment_date":
self.config.experiment.start_date.strftime("%Y-%m-%d"),
"num_dates_enrollment":
dates_enrollment,
}
if period != AnalysisPeriod.OVERALL:
try:
current_time_limits = TimeLimits.for_ts(
last_date_full_data=current_date_str,
time_series_period=period.adjective,
**time_limits_args,
)
except ValueError:
# There are no analysis windows yet.
# TODO: Add a more specific check.
return None
try:
prior_time_limits = TimeLimits.for_ts(
last_date_full_data=prior_date_str,
time_series_period=period.adjective,
**time_limits_args,
)
except ValueError:
# We have an analysis window today, and we didn't yesterday,
# so we must have just closed our first window.
return current_time_limits
if len(current_time_limits.analysis_windows) == len(
prior_time_limits.analysis_windows):
# No new data today
return None
return current_time_limits
# Period is OVERALL
if self.config.experiment.end_date != prior_date:
return None
return TimeLimits.for_single_analysis_window(
last_date_full_data=prior_date_str,
analysis_start_days=0,
analysis_length_dates=(self.config.experiment.end_date -
self.config.experiment.start_date).days -
dates_enrollment + 1,
**time_limits_args,
)
def test_get_per_client_data_join(spark):
exp = Experiment('a-stub', '20190101')
enrollments = spark.createDataFrame(
[
['aaaa', 'control', '20190101'],
['bbbb', 'test', '20190101'],
['cccc', 'control', '20190108'],
['dddd', 'test', '20190109'],
['annie-nodata', 'control', '20190101'],
['bob-badtiming', 'test', '20190102'],
['carol-gooddata', 'test', '20190101'],
['derek-lateisok', 'control', '20190110'],
],
[
"client_id",
"branch",
"enrollment_date",
],
)
ex_d = {'a-stub': 'fake-branch-lifes-too-short'}
data_source_df = spark.createDataFrame(
[
# bob-badtiming only has data before/after analysis window
# but missed by `process_data_source`
['bob-badtiming', '20190102', ex_d, 1],
['bob-badtiming', '20190106', ex_d, 2],
# carol-gooddata has data on two days (including a dupe day)
['carol-gooddata', '20190102', ex_d, 3],
['carol-gooddata', '20190102', ex_d, 2],
['carol-gooddata', '20190104', ex_d, 6],
# derek-lateisok has data before and during the analysis window
['derek-lateisok', '20190110', ex_d, 1000],
['derek-lateisok', '20190111', ex_d, 1],
# TODO: exercise the last condition on the join
],
[
"client_id",
"submission_date_s3",
"experiments",
"some_value",
],
)
ds = DataSource.from_dataframe('ds', data_source_df)
metric = Metric.from_col('some_value', agg_sum(data_source_df.some_value),
ds)
res = exp.get_per_client_data(enrollments, [metric],
'20190114',
1,
3,
keep_client_id=True)
# Check that the dataframe has the correct number of rows
assert res.count() == enrollments.count()
# Check that dataless enrollments are handled correctly
annie_nodata = res.filter(res.client_id == 'annie-nodata')
assert annie_nodata.count() == 1
assert annie_nodata.first()['some_value'] == 0
# Check that early and late data were ignored
# i.e. check the join, not just _process_data_source_df
bob_badtiming = res.filter(res.client_id == 'bob-badtiming')
assert bob_badtiming.count() == 1
assert bob_badtiming.first()['some_value'] == 0
# Check that _process_data_source_df didn't do the
# heavy lifting above
time_limits = TimeLimits.for_single_analysis_window(
exp.start_date, '20190114', 1, 3, exp.num_dates_enrollment)
pds = exp._process_data_source_df(data_source_df, time_limits)
assert pds.filter(pds.client_id == 'bob-badtiming').select(
F.sum(pds.some_value).alias('agg_val')).first()['agg_val'] == 3
# Check that relevant data was included appropriately
carol_gooddata = res.filter(res.client_id == 'carol-gooddata')
assert carol_gooddata.count() == 1
assert carol_gooddata.first()['some_value'] == 11
derek_lateisok = res.filter(res.client_id == 'derek-lateisok')
assert derek_lateisok.count() == 1
assert derek_lateisok.first()['some_value'] == 1
# Check that it still works for `data_source`s without an experiments map
ds_df_noexp = data_source_df.drop('experiments')
ds_noexp = DataSource.from_dataframe('ds_noexp', ds_df_noexp)
metric_noexp = Metric.from_col('some_value',
agg_sum(ds_df_noexp.some_value), ds_noexp)
res2 = exp.get_per_client_data(enrollments, [metric_noexp],
'20190114',
1,
3,
keep_client_id=True)
assert res2.count() == enrollments.count()
def test_get_results_for_one_data_source(spark):
exp = Experiment('a-stub', '20190101')
enrollments = spark.createDataFrame(
[
['aaaa', 'control', '20190101'],
['bbbb', 'test', '20190101'],
['cccc', 'control', '20190108'],
['dddd', 'test', '20190109'],
['annie-nodata', 'control', '20190101'],
['bob-badtiming', 'test', '20190102'],
['carol-gooddata', 'test', '20190101'],
['derek-lateisok', 'control', '20190110'],
],
[
"client_id",
"branch",
"enrollment_date",
],
)
ex_d = {'a-stub': 'fake-branch-lifes-too-short'}
data_source = spark.createDataFrame(
[
# bob-badtiming only has data before/after analysis window
# but missed by `process_data_source`
['bob-badtiming', '20190102', ex_d, 1],
['bob-badtiming', '20190106', ex_d, 2],
# carol-gooddata has data on two days (including a dupe day)
['carol-gooddata', '20190102', ex_d, 3],
['carol-gooddata', '20190102', ex_d, 2],
['carol-gooddata', '20190104', ex_d, 6],
# derek-lateisok has data before and during the analysis window
['derek-lateisok', '20190110', ex_d, 1000],
['derek-lateisok', '20190111', ex_d, 1],
# TODO: exercise the last condition on the join
],
[
"client_id",
"submission_date_s3",
"experiments",
"some_value",
],
)
time_limits = TimeLimits.for_single_analysis_window(
exp.start_date,
'20190114',
1,
3,
)
enrollments = exp._add_analysis_windows_to_enrollments(
enrollments, time_limits)
res = exp._get_results_for_one_data_source(
enrollments,
data_source,
[
F.coalesce(F.sum(data_source.some_value),
F.lit(0)).alias('some_value'),
],
)
# Check that the dataframe has the correct number of rows
assert res.count() == enrollments.count()
# Check that dataless enrollments are handled correctly
annie_nodata = res.filter(res.client_id == 'annie-nodata')
assert annie_nodata.count() == 1
assert annie_nodata.first()['some_value'] == 0
# Check that early and late data were ignored
bob_badtiming = res.filter(res.client_id == 'bob-badtiming')
assert bob_badtiming.count() == 1
assert bob_badtiming.first()['some_value'] == 0
# Check that relevant data was included appropriately
carol_gooddata = res.filter(res.client_id == 'carol-gooddata')
assert carol_gooddata.count() == 1
assert carol_gooddata.first()['some_value'] == 11
derek_lateisok = res.filter(res.client_id == 'derek-lateisok')
assert derek_lateisok.count() == 1
assert derek_lateisok.first()['some_value'] == 1
# Check that it still works for `data_source`s without an experiments map
res2 = exp._get_results_for_one_data_source(
enrollments,
data_source.drop('experiments'),
[
F.coalesce(F.sum(data_source.some_value),
F.lit(0)).alias('some_value'),
],
)
assert res2.count() == enrollments.count()
def validate(self) -> None:
self.check_runnable()
assert self.config.experiment.start_date is not None # for mypy
dates_enrollment = self.config.experiment.proposed_enrollment + 1
if self.config.experiment.end_date is not None:
end_date = self.config.experiment.end_date
analysis_length_dates = (
(end_date - self.config.experiment.start_date).days - dates_enrollment + 1
)
else:
analysis_length_dates = 21 # arbitrary
end_date = self.config.experiment.start_date + timedelta(
days=analysis_length_dates + dates_enrollment - 1
)
if analysis_length_dates < 0:
logging.error(
"Proposed enrollment longer than analysis dates length:"
+ f"{self.config.experiment.normandy_slug}"
)
raise Exception("Cannot validate experiment")
limits = TimeLimits.for_single_analysis_window(
last_date_full_data=end_date.strftime("%Y-%m-%d"),
analysis_start_days=0,
analysis_length_dates=analysis_length_dates,
first_enrollment_date=self.config.experiment.start_date.strftime("%Y-%m-%d"),
num_dates_enrollment=dates_enrollment,
)
exp = mozanalysis.experiment.Experiment(
experiment_slug=self.config.experiment.normandy_slug,
start_date=self.config.experiment.start_date.strftime("%Y-%m-%d"),
app_id=self._app_id_to_bigquery_dataset(self.config.experiment.app_id),
)
metrics = set()
for v in self.config.metrics.values():
metrics |= {m.metric.to_mozanalysis_metric() for m in v}
exposure_signal = None
if self.config.experiment.exposure_signal:
exposure_signal = self.config.experiment.exposure_signal.to_mozanalysis_exposure_signal(
limits
)
enrollments_sql = exp.build_enrollments_query(
limits,
self.config.experiment.platform.enrollments_query_type,
self.config.experiment.enrollment_query,
None,
exposure_signal,
self.config.experiment.segments,
)
dry_run_query(enrollments_sql)
metrics_sql = exp.build_metrics_query(
metrics, limits, "enrollments_table", AnalysisBasis.ENROLLMENTS
)
# enrollments_table doesn't get created when performing a dry run;
# the metrics SQL is modified to include a subquery for a mock enrollments_table
# A UNION ALL is required here otherwise the dry run fails with
# "cannot query over table without filter over columns"
metrics_sql = metrics_sql.replace(
"WITH analysis_windows AS (",
"""WITH enrollments_table AS (
SELECT '00000' AS client_id,
'test' AS branch,
DATE('2020-01-01') AS enrollment_date,
DATE('2020-01-01') AS exposure_date,
1 AS num_enrollment_events,
1 AS num_exposure_events
UNION ALL
SELECT '00000' AS client_id,
'test' AS branch,
DATE('2020-01-01') AS enrollment_date,
DATE('2020-01-01') AS exposure_date,
1 AS num_enrollment_events,
1 AS num_exposure_events
), analysis_windows AS (""",
)
dry_run_query(metrics_sql)
def _get_timelimits_if_ready(
self, period: AnalysisPeriod, current_date: datetime
) -> Optional[TimeLimits]:
"""
Returns a TimeLimits instance if experiment is due for analysis.
Otherwise returns None.
"""
prior_date = current_date - timedelta(days=1)
prior_date_str = prior_date.strftime("%Y-%m-%d")
current_date_str = current_date.strftime("%Y-%m-%d")
dates_enrollment = self.config.experiment.proposed_enrollment + 1
if self.config.experiment.start_date is None:
return None
time_limits_args = {
"first_enrollment_date": self.config.experiment.start_date.strftime("%Y-%m-%d"),
"num_dates_enrollment": dates_enrollment,
}
if period != AnalysisPeriod.OVERALL:
try:
current_time_limits = TimeLimits.for_ts(
last_date_full_data=current_date_str,
time_series_period=period.mozanalysis_label,
**time_limits_args,
)
except ValueError:
# There are no analysis windows yet.
# TODO: Add a more specific check.
return None
try:
prior_time_limits = TimeLimits.for_ts(
last_date_full_data=prior_date_str,
time_series_period=period.mozanalysis_label,
**time_limits_args,
)
except ValueError:
# We have an analysis window today, and we didn't yesterday,
# so we must have just closed our first window.
return current_time_limits
if len(current_time_limits.analysis_windows) == len(prior_time_limits.analysis_windows):
# No new data today
return None
return current_time_limits
assert period == AnalysisPeriod.OVERALL
if (
self.config.experiment.end_date is None
or self.config.experiment.end_date.date() != current_date.date()
or self.config.experiment.status != "Complete"
):
return None
if self.config.experiment.end_date is None:
return None
analysis_length_dates = (
(self.config.experiment.end_date - self.config.experiment.start_date).days
- dates_enrollment
+ 1
)
if analysis_length_dates < 0:
raise errors.EnrollmentLongerThanAnalysisException(self.config.experiment.normandy_slug)
return TimeLimits.for_single_analysis_window(
last_date_full_data=prior_date_str,
analysis_start_days=0,
analysis_length_dates=analysis_length_dates,
**time_limits_args,
)