def setUp(self): super().setUp() self.user = UserFactory() self.request = RequestFactory().request() self.request.session = {} self.request.site = SiteFactory() self.request.user = self.user self.addCleanup(set_current_request, None) set_current_request(self.request) self.flag = ExperimentWaffleFlag('experiments', 'test', __name__, num_buckets=2, experiment_id=0) self.key = CourseKey.from_string('a/b/c') bucket_patch = patch( 'lms.djangoapps.experiments.flags.stable_bucketing_hash_group', return_value=1) self.addCleanup(bucket_patch.stop) bucket_patch.start() self.addCleanup(RequestCache.clear_all_namespaces)
class ExperimentWaffleFlagTests(SharedModuleStoreTestCase): """ Tests for the ExperimentWaffleFlag class """ def setUp(self): super().setUp() self.user = UserFactory() self.request = RequestFactory().request() self.request.session = {} self.request.site = SiteFactory() self.request.user = self.user self.addCleanup(set_current_request, None) set_current_request(self.request) self.flag = ExperimentWaffleFlag('experiments', 'test', __name__, num_buckets=2, experiment_id=0) self.key = CourseKey.from_string('a/b/c') bucket_patch = patch( 'lms.djangoapps.experiments.flags.stable_bucketing_hash_group', return_value=1) self.addCleanup(bucket_patch.stop) bucket_patch.start() self.addCleanup(RequestCache.clear_all_namespaces) def get_bucket(self, track=False, active=True): # Does not use ExperimentWaffleFlag.override, since that shortcuts get_bucket and we want to test internals with override_waffle_flag(self.flag, active): with override_waffle_flag(self.flag.bucket_flags[1], True): return self.flag.get_bucket(course_key=self.key, track=track) def test_basic_happy_path(self): self.assertEqual(self.get_bucket(), 1) def test_no_request(self): set_current_request(None) self.assertEqual(self.get_bucket(), 0) def test_not_enabled(self): self.assertEqual(self.get_bucket(active=False), 0) @ddt.data( ( '2012-01-06', None, 1 ), # no enrollment, but start is in past (we allow normal bucketing in this case) ( '9999-01-06', None, 0 ), # no enrollment, but start is in future (we give bucket 0 in that case) ('2012-01-06', '2012-01-05', 0), # enrolled before experiment start ('2012-01-06', '2012-01-07', 1), # enrolled after experiment start (None, '2012-01-07', 1), # no experiment date ('not-a-date', '2012-01-07', 0), # bad experiment date ) @ddt.unpack def test_enrollment_start(self, experiment_start, enrollment_created, expected_bucket): if enrollment_created: enrollment = CourseEnrollmentFactory(user=self.user, course_id='a/b/c') enrollment.created = parser.parse(enrollment_created).replace( tzinfo=pytz.UTC) enrollment.save() if experiment_start: ExperimentKeyValueFactory(experiment_id=0, key='enrollment_start', value=experiment_start) self.assertEqual(self.get_bucket(), expected_bucket) @ddt.data( ( '2012-01-06', None, 0 ), # no enrollment, but end is in past (we give bucket 0 in that case) ( '9999-01-06', None, 1 ), # no enrollment, but end is in future (we allow normal bucketing in this case) ('2012-01-06', '2012-01-05', 1), # enrolled before experiment end ('2012-01-06', '2012-01-07', 0), # enrolled after experiment end (None, '2012-01-07', 1), # no experiment date ('not-a-date', '2012-01-07', 0), # bad experiment date ) @ddt.unpack def test_enrollment_end(self, experiment_end, enrollment_created, expected_bucket): if enrollment_created: enrollment = CourseEnrollmentFactory(user=self.user, course_id='a/b/c') enrollment.created = parser.parse(enrollment_created).replace( tzinfo=pytz.UTC) enrollment.save() if experiment_end: ExperimentKeyValueFactory(experiment_id=0, key='enrollment_end', value=experiment_end) self.assertEqual(self.get_bucket(), expected_bucket) @ddt.data( (True, 0), (False, 1), ) @ddt.unpack def test_forcing_bucket(self, active, expected_bucket): bucket_flag = CourseWaffleFlag('experiments', 'test.0', __name__) with bucket_flag.override(active=active): self.assertEqual(self.get_bucket(), expected_bucket) def test_tracking(self): # Run twice, with same request with patch('lms.djangoapps.experiments.flags.segment') as segment_mock: self.assertEqual(self.get_bucket(track=True), 1) RequestCache.clear_all_namespaces( ) # we want to force get_bucket to check session, not early exit self.assertEqual(self.get_bucket(track=True), 1) # Now test that we only sent the signal once, and with the correct properties self.assertEqual(segment_mock.track.call_count, 1) self.assertEqual(segment_mock.track.call_args, ((), { 'user_id': self.user.id, 'event_name': 'edx.bi.experiment.user.bucketed', 'properties': { 'site': self.request.site.domain, 'app_label': 'experiments', 'experiment': 'test', 'bucket': 1, 'course_id': 'a/b/c', 'is_staff': self.user.is_staff, 'nonInteraction': 1, }, })) def test_caching(self): self.assertEqual(self.get_bucket(active=True), 1) self.assertEqual(self.get_bucket(active=False), 1) # still returns 1! def test_is_enabled(self): with patch( 'lms.djangoapps.experiments.flags.ExperimentWaffleFlag.get_bucket', return_value=1): self.assertEqual(self.flag.is_enabled(self.key), True) self.assertEqual(self.flag.is_enabled(), True) with patch( 'lms.djangoapps.experiments.flags.ExperimentWaffleFlag.get_bucket', return_value=0): self.assertEqual(self.flag.is_enabled(self.key), False) self.assertEqual(self.flag.is_enabled(), False) @ddt.data( (True, 1, 1), (True, 0, 0), (False, 1, 0), # bucket is always 0 if the experiment is off (False, 0, 0), ) @ddt.unpack # Test the override method def test_override_method(self, active, bucket_override, expected_bucket): with self.flag.override(active=active, bucket=bucket_override): self.assertEqual(self.flag.get_bucket(), expected_bucket) self.assertEqual(self.flag.is_experiment_on(), active)
class ExperimentWaffleFlagCourseAwarenessTest(SharedModuleStoreTestCase): """ Tests for how course context awareness/unawareness interacts with the ExperimentWaffleFlag class. """ course_aware_flag = ExperimentWaffleFlag( 'exp', 'aware', __name__, num_buckets=20, use_course_aware_bucketing=True, ) course_aware_subflag = CourseWaffleFlag('exp', 'aware.1', __name__) course_unaware_flag = ExperimentWaffleFlag( 'exp', 'unaware', __name__, num_buckets=20, use_course_aware_bucketing=False, ) course_unaware_subflag = CourseWaffleFlag('exp', 'unaware.1', __name__) course_key_1 = CourseKey.from_string("x/y/1") course_key_2 = CourseKey.from_string("x/y/22") course_key_3 = CourseKey.from_string("x/y/333") @classmethod def setUpTestData(cls): super().setUpTestData() # Force all users into Bucket 1 for course at `course_key_1`. WaffleFlagCourseOverrideModel.objects.create( waffle_flag="exp.aware.1", course_id=cls.course_key_1, enabled=True) WaffleFlagCourseOverrideModel.objects.create( waffle_flag="exp.unaware.1", course_id=cls.course_key_1, enabled=True) cls.user = UserFactory() def setUp(self): super().setUp() self.request = RequestFactory().request() self.request.session = {} self.request.site = SiteFactory() self.request.user = self.user self.addCleanup(set_current_request, None) set_current_request(self.request) self.addCleanup(RequestCache.clear_all_namespaces) # Enable all experiment waffle flags. experiment_waffle_flag_patcher = patch.object(ExperimentWaffleFlag, 'is_experiment_on', return_value=True) experiment_waffle_flag_patcher.start() self.addCleanup(experiment_waffle_flag_patcher.stop) # Use our custom fake `stable_bucketing_hash_group` implementation. stable_bucket_patcher = patch( 'lms.djangoapps.experiments.flags.stable_bucketing_hash_group', self._mock_stable_bucket) stable_bucket_patcher.start() self.addCleanup(stable_bucket_patcher.stop) @staticmethod def _mock_stable_bucket(group_name, *_args, **_kwargs): """ A fake version of `stable_bucketing_hash_group` that just returns the length of `group_name`. """ return len(group_name) def test_course_aware_bucketing(self): """ Test behavior of an experiment flag configured wtih course-aware bucket hashing. """ # Expect queries for Course 1 to be forced into Bucket 1 # due to `course_aware_subflag`. assert self.course_aware_flag.get_bucket(self.course_key_1) == 1 # Because we are using course-aware bucket hashing, different # courses may default to different buckets. # In the case of Courses 2 and 3 here, we expect two different buckets. assert self.course_aware_flag.get_bucket(self.course_key_2) == 16 assert self.course_aware_flag.get_bucket(self.course_key_3) == 17 # We can still query a course-aware flag outside of course context, # which has its own default bucket. assert self.course_aware_flag.get_bucket() == 9 def test_course_unaware_bucketing(self): """ Test behavior of an experiment flag configured wtih course-unaware bucket hashing. """ # Expect queries for Course 1 to be forced into Bucket 1 # due to `course_unaware_subflag`. # This should happen in spite of the fact that *default* bucketing # is unaware of courses. assert self.course_unaware_flag.get_bucket(self.course_key_1) == 1 # Expect queries for Course 2, queries for Course 3, and queries outside # the context of the course to all be hashed into the same default bucket. assert self.course_unaware_flag.get_bucket(self.course_key_2) == 11 assert self.course_unaware_flag.get_bucket(self.course_key_3) == 11 assert self.course_unaware_flag.get_bucket() == 11