def test_time_limits_create5(): # But not an 8 day window with pytest.raises(ValueError): TimeLimits.for_single_analysis_window( first_enrollment_date="2019-01-01", last_date_full_data="2019-01-14", analysis_start_days=0, analysis_length_dates=8, num_dates_enrollment=8, )
def test_time_limits_has_right_date_in_error_message(): msg_re = r"until we have data for 2020-03-30." with pytest.raises(ValueError, match=msg_re): TimeLimits.for_single_analysis_window( first_enrollment_date="2020-03-03", last_date_full_data="2020-03-23", analysis_start_days=0, analysis_length_dates=21, num_dates_enrollment=8, )
def test_no_analysis_exception_when_shared_parent_dataframe(spark): # Check that we don't fall victim to # https://issues.apache.org/jira/browse/SPARK-10925 df = spark.createDataFrame( [ # Just need the schema, really ['someone', '20190102', 'fake', 1], ], [ "client_id", "submission_date_s3", "branch", "some_value", ]) enrollments = df.groupby('client_id', 'branch').agg( F.min('submission_date_s3').alias('enrollment_date')) exp = Experiment('a-stub', '20180101') time_limits = TimeLimits.for_single_analysis_window( exp.start_date, last_date_full_data='20190522', analysis_start_days=28, analysis_length_dates=7) enrollments = exp._add_analysis_windows_to_enrollments( enrollments, time_limits) exp._get_results_for_one_data_source( enrollments, df, [F.max(F.col('some_value'))], )
def test_process_enrollments(spark): exp = Experiment('a-stub', '20190101') enrollments = exp.get_enrollments(spark, _get_enrollment_view(slug="a-stub")) assert enrollments.count() == 4 # With final data collected on '20190114', we have 7 dates of data # for 'cccc' enrolled on '20190108' but not for 'dddd' enrolled on # '20190109'. tl = TimeLimits.for_single_analysis_window( first_enrollment_date=exp.start_date, last_date_full_data='20190114', analysis_start_days=0, analysis_length_dates=7, num_dates_enrollment=exp.num_dates_enrollment) assert tl.last_enrollment_date == '20190108' assert len(tl.analysis_windows) == 1 assert tl.analysis_windows[0].end == 6 pe = exp._process_enrollments(enrollments, tl) assert pe.count() == 3 pe = exp._process_enrollments(enrollments.alias('main_summary'), tl) assert pe.select(F.col('enrollments.enrollment_date')) with pytest.raises(AnalysisException): assert pe.select(F.col('main_summary.enrollment_date'))
def test_process_data_source_df(spark): start_date = '20190101' exp_8d = Experiment('experiment-with-8-day-cohort', start_date, 8) data_source_df = _get_data_source_df(spark) end_date = '20190114' # Are the fixtures sufficiently complicated that we're actually testing # things? assert _simple_return_agg_date(F.min, data_source_df) < start_date assert _simple_return_agg_date(F.max, data_source_df) > end_date tl_03 = TimeLimits.for_single_analysis_window( first_enrollment_date=exp_8d.start_date, last_date_full_data=end_date, analysis_start_days=0, analysis_length_dates=3, num_dates_enrollment=exp_8d.num_dates_enrollment) assert tl_03.first_date_data_required == start_date assert tl_03.last_date_data_required == '20190110' proc_ds = exp_8d._process_data_source_df(data_source_df, tl_03) assert _simple_return_agg_date(F.min, proc_ds) == tl_03.first_date_data_required assert _simple_return_agg_date(F.max, proc_ds) == tl_03.last_date_data_required tl_23 = TimeLimits.for_single_analysis_window( first_enrollment_date=exp_8d.start_date, last_date_full_data=end_date, analysis_start_days=2, analysis_length_dates=3, num_dates_enrollment=exp_8d.num_dates_enrollment) assert tl_23.first_date_data_required == add_days(start_date, 2) assert tl_23.last_date_data_required == '20190112' p_ds_2 = exp_8d._process_data_source_df(data_source_df, tl_23) assert _simple_return_agg_date(F.min, p_ds_2) == tl_23.first_date_data_required assert _simple_return_agg_date(F.max, p_ds_2) == tl_23.last_date_data_required assert proc_ds.select(F.col('data_source.client_id')) with pytest.raises(AnalysisException): assert data_source_df.select(F.col('data_source.client_id'))
def test_time_limits_create2(): # We don't have 14 dates of data for an 8-day cohort: with pytest.raises(ValueError): TimeLimits.for_single_analysis_window( first_enrollment_date="2019-01-01", last_date_full_data="2019-01-14", analysis_start_days=0, analysis_length_dates=14, num_dates_enrollment=8, ) # We don't have 15 full dates of data for any users with pytest.raises(AssertionError): TimeLimits.for_single_analysis_window( first_enrollment_date="2019-01-01", last_date_full_data="2019-01-14", analysis_start_days=0, analysis_length_dates=15, )
def test_time_limits_create6(): # Of course the flexi-experiment has data for a 1 day window tl = TimeLimits.for_single_analysis_window( first_enrollment_date="2019-01-01", last_date_full_data="2019-01-14", analysis_start_days=0, analysis_length_dates=1, ) assert tl.first_enrollment_date == "2019-01-01" assert tl.last_enrollment_date == "2019-01-14" assert len(tl.analysis_windows) == 1 assert tl.analysis_windows[0].start == 0 assert tl.analysis_windows[0].end == 0 assert tl.first_date_data_required == "2019-01-01" assert tl.last_date_data_required == "2019-01-14"
def test_time_limits_create7(): # If the analysis starts later, so does the data source tl = TimeLimits.for_single_analysis_window( first_enrollment_date="2019-01-01", last_date_full_data="2019-01-14", analysis_start_days=7, analysis_length_dates=1, ) assert tl.first_enrollment_date == "2019-01-01" assert tl.last_enrollment_date == "2019-01-07" assert len(tl.analysis_windows) == 1 assert tl.analysis_windows[0].start == 7 assert tl.analysis_windows[0].end == 7 assert tl.first_date_data_required == "2019-01-08" assert tl.last_date_data_required == "2019-01-14"
def test_time_limits_create3(): # For the 8-day cohort We have enough data for a 7 day window tl = TimeLimits.for_single_analysis_window( first_enrollment_date="2019-01-01", last_date_full_data="2019-01-14", analysis_start_days=0, analysis_length_dates=7, num_dates_enrollment=8, ) assert tl.first_enrollment_date == "2019-01-01" assert tl.last_enrollment_date == "2019-01-08" assert len(tl.analysis_windows) == 1 assert tl.analysis_windows[0].start == 0 assert tl.analysis_windows[0].end == 6 assert tl.first_date_data_required == "2019-01-01" assert tl.last_date_data_required == "2019-01-14"
def test_time_limits_create4(): # Or a 2 day window tl = TimeLimits.for_single_analysis_window( first_enrollment_date="2019-01-01", last_date_full_data="2019-01-14", analysis_start_days=0, analysis_length_dates=2, num_dates_enrollment=8, ) assert tl.first_enrollment_date == "2019-01-01" assert tl.last_enrollment_date == "2019-01-08" assert len(tl.analysis_windows) == 1 assert tl.analysis_windows[0].start == 0 assert tl.analysis_windows[0].end == 1 assert tl.first_date_data_required == "2019-01-01" assert tl.last_date_data_required == "2019-01-09"
def validate(self) -> None: self.check_runnable() dates_enrollment = self.config.experiment.proposed_enrollment + 1 if self.config.experiment.end_date is not None: end_date = self.config.experiment.end_date analysis_length_dates = ( (end_date - self.config.experiment.start_date).days - dates_enrollment + 1) else: analysis_length_dates = 21 # arbitrary end_date = self.config.experiment.start_date + timedelta( days=analysis_length_dates + dates_enrollment - 1) if analysis_length_dates < 0: logging.error( "Proposed enrollment longer than analysis dates length:" + f"{self.config.experiment.normandy_slug}") raise Exception("Cannot validate experiment") limits = TimeLimits.for_single_analysis_window( last_date_full_data=end_date.strftime("%Y-%m-%d"), analysis_start_days=0, analysis_length_dates=analysis_length_dates, first_enrollment_date=self.config.experiment.start_date.strftime( "%Y-%m-%d"), num_dates_enrollment=dates_enrollment, ) exp = mozanalysis.experiment.Experiment( experiment_slug=self.config.experiment.normandy_slug, start_date=self.config.experiment.start_date.strftime("%Y-%m-%d"), ) metrics = set() for v in self.config.metrics.values(): metrics |= {m.metric for m in v} sql = exp.build_query( metrics, limits, "normandy", self.config.experiment.enrollment_query, ) dry_run_query(sql)
def test_time_limits_create1(): # When we have complete data for 2019-01-14... # ...We have 14 dates of data for those who enrolled on the 1st tl = TimeLimits.for_single_analysis_window( first_enrollment_date="2019-01-01", last_date_full_data="2019-01-14", analysis_start_days=0, analysis_length_dates=14, ) assert tl.first_enrollment_date == "2019-01-01" assert tl.last_enrollment_date == "2019-01-01" assert len(tl.analysis_windows) == 1 assert tl.analysis_windows[0].start == 0 assert tl.analysis_windows[0].end == 13 assert tl.first_date_data_required == "2019-01-01" assert tl.last_date_data_required == "2019-01-14"
def dry_run_query(exp_path): report = validate_schema(op.join(exp_path, "report.json")) metric_list = _make_metric_list(report) exp = Experiment(experiment_slug=report["experiment_slug"], start_date=report["start_date"], num_dates_enrollment=report["num_dates_enrollment"]) # create an archive of the sql generating analysis time_limits = TimeLimits.for_single_analysis_window( first_enrollment_date=report['start_date'], last_date_full_data=report['last_date_full_data'], analysis_start_days=report['analysis_start_days'], analysis_length_dates=report['analysis_length_days'], num_dates_enrollment=report['num_dates_enrollment']) query = exp.build_query(metric_list=metric_list, time_limits=time_limits, enrollments_query_type='normandy') return query
def _get_timelimits_if_ready( self, period: AnalysisPeriod, current_date: datetime) -> Optional[TimeLimits]: """ Returns a TimeLimits instance if experiment is due for analysis. Otherwise returns None. """ prior_date = current_date - timedelta(days=1) prior_date_str = prior_date.strftime("%Y-%m-%d") current_date_str = current_date.strftime("%Y-%m-%d") if not self.config.experiment.proposed_enrollment: self.logger.info("Skipping %s; no enrollment period", self.config.experiment.slug) return None dates_enrollment = self.config.experiment.proposed_enrollment + 1 if self.config.experiment.start_date is None: return None time_limits_args = { "first_enrollment_date": self.config.experiment.start_date.strftime("%Y-%m-%d"), "num_dates_enrollment": dates_enrollment, } if period != AnalysisPeriod.OVERALL: try: current_time_limits = TimeLimits.for_ts( last_date_full_data=current_date_str, time_series_period=period.adjective, **time_limits_args, ) except ValueError: # There are no analysis windows yet. # TODO: Add a more specific check. return None try: prior_time_limits = TimeLimits.for_ts( last_date_full_data=prior_date_str, time_series_period=period.adjective, **time_limits_args, ) except ValueError: # We have an analysis window today, and we didn't yesterday, # so we must have just closed our first window. return current_time_limits if len(current_time_limits.analysis_windows) == len( prior_time_limits.analysis_windows): # No new data today return None return current_time_limits # Period is OVERALL if self.config.experiment.end_date != prior_date: return None return TimeLimits.for_single_analysis_window( last_date_full_data=prior_date_str, analysis_start_days=0, analysis_length_dates=(self.config.experiment.end_date - self.config.experiment.start_date).days - dates_enrollment + 1, **time_limits_args, )
def test_get_per_client_data_join(spark): exp = Experiment('a-stub', '20190101') enrollments = spark.createDataFrame( [ ['aaaa', 'control', '20190101'], ['bbbb', 'test', '20190101'], ['cccc', 'control', '20190108'], ['dddd', 'test', '20190109'], ['annie-nodata', 'control', '20190101'], ['bob-badtiming', 'test', '20190102'], ['carol-gooddata', 'test', '20190101'], ['derek-lateisok', 'control', '20190110'], ], [ "client_id", "branch", "enrollment_date", ], ) ex_d = {'a-stub': 'fake-branch-lifes-too-short'} data_source_df = spark.createDataFrame( [ # bob-badtiming only has data before/after analysis window # but missed by `process_data_source` ['bob-badtiming', '20190102', ex_d, 1], ['bob-badtiming', '20190106', ex_d, 2], # carol-gooddata has data on two days (including a dupe day) ['carol-gooddata', '20190102', ex_d, 3], ['carol-gooddata', '20190102', ex_d, 2], ['carol-gooddata', '20190104', ex_d, 6], # derek-lateisok has data before and during the analysis window ['derek-lateisok', '20190110', ex_d, 1000], ['derek-lateisok', '20190111', ex_d, 1], # TODO: exercise the last condition on the join ], [ "client_id", "submission_date_s3", "experiments", "some_value", ], ) ds = DataSource.from_dataframe('ds', data_source_df) metric = Metric.from_col('some_value', agg_sum(data_source_df.some_value), ds) res = exp.get_per_client_data(enrollments, [metric], '20190114', 1, 3, keep_client_id=True) # Check that the dataframe has the correct number of rows assert res.count() == enrollments.count() # Check that dataless enrollments are handled correctly annie_nodata = res.filter(res.client_id == 'annie-nodata') assert annie_nodata.count() == 1 assert annie_nodata.first()['some_value'] == 0 # Check that early and late data were ignored # i.e. check the join, not just _process_data_source_df bob_badtiming = res.filter(res.client_id == 'bob-badtiming') assert bob_badtiming.count() == 1 assert bob_badtiming.first()['some_value'] == 0 # Check that _process_data_source_df didn't do the # heavy lifting above time_limits = TimeLimits.for_single_analysis_window( exp.start_date, '20190114', 1, 3, exp.num_dates_enrollment) pds = exp._process_data_source_df(data_source_df, time_limits) assert pds.filter(pds.client_id == 'bob-badtiming').select( F.sum(pds.some_value).alias('agg_val')).first()['agg_val'] == 3 # Check that relevant data was included appropriately carol_gooddata = res.filter(res.client_id == 'carol-gooddata') assert carol_gooddata.count() == 1 assert carol_gooddata.first()['some_value'] == 11 derek_lateisok = res.filter(res.client_id == 'derek-lateisok') assert derek_lateisok.count() == 1 assert derek_lateisok.first()['some_value'] == 1 # Check that it still works for `data_source`s without an experiments map ds_df_noexp = data_source_df.drop('experiments') ds_noexp = DataSource.from_dataframe('ds_noexp', ds_df_noexp) metric_noexp = Metric.from_col('some_value', agg_sum(ds_df_noexp.some_value), ds_noexp) res2 = exp.get_per_client_data(enrollments, [metric_noexp], '20190114', 1, 3, keep_client_id=True) assert res2.count() == enrollments.count()
def test_get_results_for_one_data_source(spark): exp = Experiment('a-stub', '20190101') enrollments = spark.createDataFrame( [ ['aaaa', 'control', '20190101'], ['bbbb', 'test', '20190101'], ['cccc', 'control', '20190108'], ['dddd', 'test', '20190109'], ['annie-nodata', 'control', '20190101'], ['bob-badtiming', 'test', '20190102'], ['carol-gooddata', 'test', '20190101'], ['derek-lateisok', 'control', '20190110'], ], [ "client_id", "branch", "enrollment_date", ], ) ex_d = {'a-stub': 'fake-branch-lifes-too-short'} data_source = spark.createDataFrame( [ # bob-badtiming only has data before/after analysis window # but missed by `process_data_source` ['bob-badtiming', '20190102', ex_d, 1], ['bob-badtiming', '20190106', ex_d, 2], # carol-gooddata has data on two days (including a dupe day) ['carol-gooddata', '20190102', ex_d, 3], ['carol-gooddata', '20190102', ex_d, 2], ['carol-gooddata', '20190104', ex_d, 6], # derek-lateisok has data before and during the analysis window ['derek-lateisok', '20190110', ex_d, 1000], ['derek-lateisok', '20190111', ex_d, 1], # TODO: exercise the last condition on the join ], [ "client_id", "submission_date_s3", "experiments", "some_value", ], ) time_limits = TimeLimits.for_single_analysis_window( exp.start_date, '20190114', 1, 3, ) enrollments = exp._add_analysis_windows_to_enrollments( enrollments, time_limits) res = exp._get_results_for_one_data_source( enrollments, data_source, [ F.coalesce(F.sum(data_source.some_value), F.lit(0)).alias('some_value'), ], ) # Check that the dataframe has the correct number of rows assert res.count() == enrollments.count() # Check that dataless enrollments are handled correctly annie_nodata = res.filter(res.client_id == 'annie-nodata') assert annie_nodata.count() == 1 assert annie_nodata.first()['some_value'] == 0 # Check that early and late data were ignored bob_badtiming = res.filter(res.client_id == 'bob-badtiming') assert bob_badtiming.count() == 1 assert bob_badtiming.first()['some_value'] == 0 # Check that relevant data was included appropriately carol_gooddata = res.filter(res.client_id == 'carol-gooddata') assert carol_gooddata.count() == 1 assert carol_gooddata.first()['some_value'] == 11 derek_lateisok = res.filter(res.client_id == 'derek-lateisok') assert derek_lateisok.count() == 1 assert derek_lateisok.first()['some_value'] == 1 # Check that it still works for `data_source`s without an experiments map res2 = exp._get_results_for_one_data_source( enrollments, data_source.drop('experiments'), [ F.coalesce(F.sum(data_source.some_value), F.lit(0)).alias('some_value'), ], ) assert res2.count() == enrollments.count()
def validate(self) -> None: self.check_runnable() assert self.config.experiment.start_date is not None # for mypy dates_enrollment = self.config.experiment.proposed_enrollment + 1 if self.config.experiment.end_date is not None: end_date = self.config.experiment.end_date analysis_length_dates = ( (end_date - self.config.experiment.start_date).days - dates_enrollment + 1 ) else: analysis_length_dates = 21 # arbitrary end_date = self.config.experiment.start_date + timedelta( days=analysis_length_dates + dates_enrollment - 1 ) if analysis_length_dates < 0: logging.error( "Proposed enrollment longer than analysis dates length:" + f"{self.config.experiment.normandy_slug}" ) raise Exception("Cannot validate experiment") limits = TimeLimits.for_single_analysis_window( last_date_full_data=end_date.strftime("%Y-%m-%d"), analysis_start_days=0, analysis_length_dates=analysis_length_dates, first_enrollment_date=self.config.experiment.start_date.strftime("%Y-%m-%d"), num_dates_enrollment=dates_enrollment, ) exp = mozanalysis.experiment.Experiment( experiment_slug=self.config.experiment.normandy_slug, start_date=self.config.experiment.start_date.strftime("%Y-%m-%d"), app_id=self._app_id_to_bigquery_dataset(self.config.experiment.app_id), ) metrics = set() for v in self.config.metrics.values(): metrics |= {m.metric.to_mozanalysis_metric() for m in v} exposure_signal = None if self.config.experiment.exposure_signal: exposure_signal = self.config.experiment.exposure_signal.to_mozanalysis_exposure_signal( limits ) enrollments_sql = exp.build_enrollments_query( limits, self.config.experiment.platform.enrollments_query_type, self.config.experiment.enrollment_query, None, exposure_signal, self.config.experiment.segments, ) dry_run_query(enrollments_sql) metrics_sql = exp.build_metrics_query( metrics, limits, "enrollments_table", AnalysisBasis.ENROLLMENTS ) # enrollments_table doesn't get created when performing a dry run; # the metrics SQL is modified to include a subquery for a mock enrollments_table # A UNION ALL is required here otherwise the dry run fails with # "cannot query over table without filter over columns" metrics_sql = metrics_sql.replace( "WITH analysis_windows AS (", """WITH enrollments_table AS ( SELECT '00000' AS client_id, 'test' AS branch, DATE('2020-01-01') AS enrollment_date, DATE('2020-01-01') AS exposure_date, 1 AS num_enrollment_events, 1 AS num_exposure_events UNION ALL SELECT '00000' AS client_id, 'test' AS branch, DATE('2020-01-01') AS enrollment_date, DATE('2020-01-01') AS exposure_date, 1 AS num_enrollment_events, 1 AS num_exposure_events ), analysis_windows AS (""", ) dry_run_query(metrics_sql)
def _get_timelimits_if_ready( self, period: AnalysisPeriod, current_date: datetime ) -> Optional[TimeLimits]: """ Returns a TimeLimits instance if experiment is due for analysis. Otherwise returns None. """ prior_date = current_date - timedelta(days=1) prior_date_str = prior_date.strftime("%Y-%m-%d") current_date_str = current_date.strftime("%Y-%m-%d") dates_enrollment = self.config.experiment.proposed_enrollment + 1 if self.config.experiment.start_date is None: return None time_limits_args = { "first_enrollment_date": self.config.experiment.start_date.strftime("%Y-%m-%d"), "num_dates_enrollment": dates_enrollment, } if period != AnalysisPeriod.OVERALL: try: current_time_limits = TimeLimits.for_ts( last_date_full_data=current_date_str, time_series_period=period.mozanalysis_label, **time_limits_args, ) except ValueError: # There are no analysis windows yet. # TODO: Add a more specific check. return None try: prior_time_limits = TimeLimits.for_ts( last_date_full_data=prior_date_str, time_series_period=period.mozanalysis_label, **time_limits_args, ) except ValueError: # We have an analysis window today, and we didn't yesterday, # so we must have just closed our first window. return current_time_limits if len(current_time_limits.analysis_windows) == len(prior_time_limits.analysis_windows): # No new data today return None return current_time_limits assert period == AnalysisPeriod.OVERALL if ( self.config.experiment.end_date is None or self.config.experiment.end_date.date() != current_date.date() or self.config.experiment.status != "Complete" ): return None if self.config.experiment.end_date is None: return None analysis_length_dates = ( (self.config.experiment.end_date - self.config.experiment.start_date).days - dates_enrollment + 1 ) if analysis_length_dates < 0: raise errors.EnrollmentLongerThanAnalysisException(self.config.experiment.normandy_slug) return TimeLimits.for_single_analysis_window( last_date_full_data=prior_date_str, analysis_start_days=0, analysis_length_dates=analysis_length_dates, **time_limits_args, )