def test_ts_time_limits_create_not_enough_data(): with pytest.raises(ValueError): TimeLimits.for_ts( first_enrollment_date="2019-01-01", last_date_full_data="2019-01-13", time_series_period="weekly", num_dates_enrollment=8, )
def test_metrics_query_based_on_exposure(): exp = Experiment("slug", "2019-01-01", 8) tl = TimeLimits.for_ts( first_enrollment_date="2019-01-01", last_date_full_data="2019-03-01", time_series_period="weekly", num_dates_enrollment=8, ) enrollments_sql = exp.build_enrollments_query( time_limits=tl, enrollments_query_type="fenix-fallback") sql_lint(enrollments_sql) metrics_sql = exp.build_metrics_query( metric_list=[ m for m in mozanalysis.metrics.fenix.__dict__.values() if isinstance(m, Metric) ], time_limits=tl, enrollments_table="enrollments", analysis_basis=AnalysisBasis.EXPOSURES, ) sql_lint(metrics_sql) assert "e.exposure_date" in metrics_sql
def test_exposure_signal_query_custom_windows(): exp = Experiment("slug", "2019-01-01", 8, app_id="my_cool_app") tl = TimeLimits.for_ts( first_enrollment_date="2019-01-01", last_date_full_data="2019-03-01", time_series_period="weekly", num_dates_enrollment=8, ) enrollment_sql = exp.build_enrollments_query( time_limits=tl, enrollments_query_type="glean-event", exposure_signal=ExposureSignal( name="exposures", data_source=mozanalysis.metrics.fenix.baseline, select_expr="metrics.counter.events_total_uri_count > 0", friendly_name="URI visited exposure", description="Exposed when URI visited", window_start=1, window_end=3, ), ) sql_lint(enrollment_sql) assert "exposures" in enrollment_sql assert "metrics.counter.events_total_uri_count > 0" in enrollment_sql assert "DATE_ADD('2019-01-01', INTERVAL 1 DAY)" in enrollment_sql assert "DATE_ADD('2019-01-01', INTERVAL 3 DAY)" in enrollment_sql
def test_add_analysis_windows_to_enrollments(spark): exp = Experiment('a-stub', '20190101', num_dates_enrollment=8) enrollments = exp.get_enrollments(spark, _get_enrollment_view(slug="a-stub")) assert enrollments.count() == 3 tl = TimeLimits.for_ts( first_enrollment_date=exp.start_date, last_date_full_data='20190114', time_series_period='daily', num_dates_enrollment=exp.num_dates_enrollment, ) assert len(tl.analysis_windows) == 7 new_enrollments = exp._add_analysis_windows_to_enrollments(enrollments, tl) nep = new_enrollments.toPandas() assert len(nep) == enrollments.count() * len(tl.analysis_windows) a = nep[nep['client_id'] == 'aaaa'] assert len(a) == len(tl.analysis_windows) assert (a.mozanalysis_analysis_window_start.sort_values() == np.arange( len(tl.analysis_windows))).all() assert (a.mozanalysis_analysis_window_end.sort_values() == np.arange( len(tl.analysis_windows))).all()
def test_segments_megaquery_not_detectably_malformed(): exp = Experiment("slug", "2019-01-01", 8) tl = TimeLimits.for_ts( first_enrollment_date="2019-01-01", last_date_full_data="2019-03-01", time_series_period="weekly", num_dates_enrollment=8, ) enrollments_sql = exp.build_enrollments_query( time_limits=tl, segment_list=[s for s in msd.__dict__.values() if isinstance(s, msd.Segment)], enrollments_query_type="normandy", ) sql_lint(enrollments_sql) metrics_sql = exp.build_metrics_query( metric_list=[m for m in mad.__dict__.values() if isinstance(m, mad.Metric)], time_limits=tl, enrollments_table="enrollments", ) sql_lint(metrics_sql)
def test_ts_time_limits_create3(): tl = TimeLimits.for_ts(first_enrollment_date='2019-01-01', last_date_full_data='2019-01-15', time_series_period='weekly', num_dates_enrollment=8) assert tl.first_enrollment_date == '2019-01-01' assert tl.last_enrollment_date == '2019-01-08' assert len(tl.analysis_windows) == 1 assert tl.analysis_windows[0].start == 0 assert tl.analysis_windows[0].end == 6 assert tl.first_date_data_required == '2019-01-01' assert tl.last_date_data_required == '2019-01-14'
def test_ts_time_limits_create3(): tl = TimeLimits.for_ts( first_enrollment_date="2019-01-01", last_date_full_data="2019-01-15", time_series_period="weekly", num_dates_enrollment=8, ) assert tl.first_enrollment_date == "2019-01-01" assert tl.last_enrollment_date == "2019-01-08" assert len(tl.analysis_windows) == 1 assert tl.analysis_windows[0].start == 0 assert tl.analysis_windows[0].end == 6 assert tl.first_date_data_required == "2019-01-01" assert tl.last_date_data_required == "2019-01-14"
def test_megaquery_not_detectably_malformed(): exp = Experiment('slug', '2019-01-01', 8) tl = TimeLimits.for_ts(first_enrollment_date='2019-01-01', last_date_full_data='2019-03-01', time_series_period='weekly', num_dates_enrollment=8) sql = exp.build_query( metric_list=[ m for m in mad.__dict__.values() if isinstance(m, mad.Metric) ], time_limits=tl, enrollments_query_type='normandy', ) sql_lint(sql)
def test_query_not_detectably_malformed(): exp = Experiment('slug', '2019-01-01', 8) tl = TimeLimits.for_ts( first_enrollment_date='2019-01-01', last_date_full_data='2019-03-01', time_series_period='weekly', num_dates_enrollment=8 ) sql = exp.build_query( metric_list=[], time_limits=tl, enrollments_query_type='normandy', ) sql_lint(sql)
def test_firefox_ios_klar_app_id_propagation(): exp = Experiment("slug", "2019-01-01", 8, app_id="my_cool_app") tl = TimeLimits.for_ts( first_enrollment_date="2019-01-01", last_date_full_data="2019-03-01", time_series_period="weekly", num_dates_enrollment=8, ) sds = SegmentDataSource( name="cool_data_source", from_expr="`moz-fx-data-shared-prod`.{dataset}.cool_table", default_dataset="org_mozilla_ios_klar", ) segment = Segment( name="cool_segment", select_expr="COUNT(*)", data_source=sds, ) enrollments_sql = exp.build_enrollments_query( time_limits=tl, segment_list=[segment], enrollments_query_type="glean-event", ) sql_lint(enrollments_sql) metrics_sql = exp.build_metrics_query( metric_list=[ m for m in mozanalysis.metrics.klar_ios.__dict__.values() if isinstance(m, Metric) ], time_limits=tl, enrollments_table="enrollments", ) sql_lint(metrics_sql) assert "org_mozilla_ios_klar" not in enrollments_sql assert "my_cool_app" in enrollments_sql sql_lint(metrics_sql)
def test_query_not_detectably_malformed(): exp = Experiment('slug', '2019-01-01', 8) tl = TimeLimits.for_ts(first_enrollment_date='2019-01-01', last_date_full_data='2019-03-01', time_series_period='weekly', num_dates_enrollment=8) sql = exp.build_query( metric_list=[], time_limits=tl, enrollments_query_type='normandy', ) # This query is actually slightly malformed, due to a trailing comma. # We should add a metric here if the linter ever improves. sql_lint(sql)
def test_exposure_query(): exp = Experiment("slug", "2019-01-01", 8, app_id="my_cool_app") tl = TimeLimits.for_ts( first_enrollment_date="2019-01-01", last_date_full_data="2019-03-01", time_series_period="weekly", num_dates_enrollment=8, ) enrollment_sql = exp.build_enrollments_query( time_limits=tl, enrollments_query_type="glean-event", ) sql_lint(enrollment_sql) assert "exposures" in enrollment_sql
def test_query_not_detectably_malformed_fenix_fallback(): exp = Experiment("slug", "2019-01-01", 8) tl = TimeLimits.for_ts( first_enrollment_date="2019-01-01", last_date_full_data="2019-03-01", time_series_period="weekly", num_dates_enrollment=8, ) enrollments_sql = exp.build_enrollments_query( time_limits=tl, enrollments_query_type="fenix-fallback") sql_lint(enrollments_sql) metrics_sql = exp.build_metrics_query( metric_list=[], time_limits=tl, enrollments_table="enrollments", ) sql_lint(metrics_sql)
def test_metrics_query_with_exposure_signal_custom_windows(): exp = Experiment("slug", "2019-01-01", 8) tl = TimeLimits.for_ts( first_enrollment_date="2019-01-01", last_date_full_data="2019-03-01", time_series_period="weekly", num_dates_enrollment=8, ) enrollments_sql = exp.build_enrollments_query( time_limits=tl, enrollments_query_type="fenix-fallback") sql_lint(enrollments_sql) metrics_sql = exp.build_metrics_query( metric_list=[ m for m in mozanalysis.metrics.fenix.__dict__.values() if isinstance(m, Metric) ], time_limits=tl, enrollments_table="enrollments", analysis_basis=AnalysisBasis.EXPOSURES, exposure_signal=ExposureSignal( name="exposures", data_source=mozanalysis.metrics.fenix.baseline, select_expr="metrics.counter.events_total_uri_count > 0", friendly_name="URI visited exposure", description="Exposed when URI visited", window_start=1, window_end=3, ), ) sql_lint(metrics_sql) assert "DATE_ADD('2019-01-01', INTERVAL 1 DAY)" in metrics_sql assert "DATE_ADD('2019-01-01', INTERVAL 3 DAY)" in metrics_sql
def test_ts_time_limits_create_not_enough_data(): with pytest.raises(ValueError): TimeLimits.for_ts(first_enrollment_date='20190101', last_date_full_data='20190113', time_series_period='weekly', num_dates_enrollment=8)
def _get_timelimits_if_ready( self, period: AnalysisPeriod, current_date: datetime) -> Optional[TimeLimits]: """ Returns a TimeLimits instance if experiment is due for analysis. Otherwise returns None. """ prior_date = current_date - timedelta(days=1) prior_date_str = prior_date.strftime("%Y-%m-%d") current_date_str = current_date.strftime("%Y-%m-%d") if not self.config.experiment.proposed_enrollment: self.logger.info("Skipping %s; no enrollment period", self.config.experiment.slug) return None dates_enrollment = self.config.experiment.proposed_enrollment + 1 if self.config.experiment.start_date is None: return None time_limits_args = { "first_enrollment_date": self.config.experiment.start_date.strftime("%Y-%m-%d"), "num_dates_enrollment": dates_enrollment, } if period != AnalysisPeriod.OVERALL: try: current_time_limits = TimeLimits.for_ts( last_date_full_data=current_date_str, time_series_period=period.adjective, **time_limits_args, ) except ValueError: # There are no analysis windows yet. # TODO: Add a more specific check. return None try: prior_time_limits = TimeLimits.for_ts( last_date_full_data=prior_date_str, time_series_period=period.adjective, **time_limits_args, ) except ValueError: # We have an analysis window today, and we didn't yesterday, # so we must have just closed our first window. return current_time_limits if len(current_time_limits.analysis_windows) == len( prior_time_limits.analysis_windows): # No new data today return None return current_time_limits # Period is OVERALL if self.config.experiment.end_date != prior_date: return None return TimeLimits.for_single_analysis_window( last_date_full_data=prior_date_str, analysis_start_days=0, analysis_length_dates=(self.config.experiment.end_date - self.config.experiment.start_date).days - dates_enrollment + 1, **time_limits_args, )
def _get_timelimits_if_ready( self, period: AnalysisPeriod, current_date: datetime ) -> Optional[TimeLimits]: """ Returns a TimeLimits instance if experiment is due for analysis. Otherwise returns None. """ prior_date = current_date - timedelta(days=1) prior_date_str = prior_date.strftime("%Y-%m-%d") current_date_str = current_date.strftime("%Y-%m-%d") dates_enrollment = self.config.experiment.proposed_enrollment + 1 if self.config.experiment.start_date is None: return None time_limits_args = { "first_enrollment_date": self.config.experiment.start_date.strftime("%Y-%m-%d"), "num_dates_enrollment": dates_enrollment, } if period != AnalysisPeriod.OVERALL: try: current_time_limits = TimeLimits.for_ts( last_date_full_data=current_date_str, time_series_period=period.mozanalysis_label, **time_limits_args, ) except ValueError: # There are no analysis windows yet. # TODO: Add a more specific check. return None try: prior_time_limits = TimeLimits.for_ts( last_date_full_data=prior_date_str, time_series_period=period.mozanalysis_label, **time_limits_args, ) except ValueError: # We have an analysis window today, and we didn't yesterday, # so we must have just closed our first window. return current_time_limits if len(current_time_limits.analysis_windows) == len(prior_time_limits.analysis_windows): # No new data today return None return current_time_limits assert period == AnalysisPeriod.OVERALL if ( self.config.experiment.end_date is None or self.config.experiment.end_date.date() != current_date.date() or self.config.experiment.status != "Complete" ): return None if self.config.experiment.end_date is None: return None analysis_length_dates = ( (self.config.experiment.end_date - self.config.experiment.start_date).days - dates_enrollment + 1 ) if analysis_length_dates < 0: raise errors.EnrollmentLongerThanAnalysisException(self.config.experiment.normandy_slug) return TimeLimits.for_single_analysis_window( last_date_full_data=prior_date_str, analysis_start_days=0, analysis_length_dates=analysis_length_dates, **time_limits_args, )