def setUp(self): super().setUp() self.course = DummyCourse(id='course-v1:testX+tt101+2019') self.course.save() self.course_version = 'TEST_VERSION' self.user = User(username='******', email='*****@*****.**') self.user.save() self.enrollment = DummyEnrollment(user=self.user, course=self.course) self.enrollment.save() self.schedule = DummySchedule( enrollment=self.enrollment, created=datetime(2019, 4, 1), start_date=datetime(2019, 4, 1) ) self.schedule.save() dummy_schedule_patcher = patch('edx_when.utils.Schedule', DummySchedule) dummy_schedule_patcher.start() self.addCleanup(dummy_schedule_patcher.stop) relative_dates_patcher = patch('edx_when.api._are_relative_dates_enabled', return_value=True) relative_dates_patcher.start() self.addCleanup(relative_dates_patcher.stop) self.addCleanup(TieredCache.dangerous_clear_all_tiers) TieredCache.dangerous_clear_all_tiers()
def key_values(cls, *key_fields, **kwargs): """ Get the set of unique values in the configuration table for the given key[s]. Calling cls.current(*value) for each value in the resulting list should always produce an entry, though any such entry may have enabled=False. Arguments: key_fields: The positional arguments are the KEY_FIELDS to return. For example if you had a course embargo configuration where each entry was keyed on (country, course), then you might want to know "What countries have embargoes configured?" with cls.key_values('country'), or "Which courses have country restrictions?" with cls.key_values('course'). You can also leave this unspecified for the default, which returns the distinct combinations of all keys. flat: If you pass flat=True as a kwarg, it has the same effect as in Django's 'values_list' method: Instead of returning a list of lists, you'll get one list of values. This makes sense to use whenever there is only one key being queried. Return value: List of lists of each combination of keys found in the database. e.g. [("Italy", "course-v1:SomeX+some+2015"), ...] for the course embargo example """ flat = kwargs.pop('flat', False) assert not kwargs, "'flat' is the only kwarg accepted" key_fields = key_fields or cls.KEY_FIELDS cache_key = cls.key_values_cache_key_name(*key_fields) cached_response = TieredCache.get_cached_response(cache_key) if cached_response.is_found: return cached_response.value values = list( cls.objects.values_list(*key_fields, flat=flat).order_by().distinct()) TieredCache.set_all_tiers(cache_key, values, cls.cache_timeout) return values
def test_set_all_tiers(self, mock_cache_set): mock_cache_set.return_value = EXPECTED_VALUE TieredCache.set_all_tiers(TEST_KEY, EXPECTED_VALUE, TEST_DJANGO_TIMEOUT_CACHE) mock_cache_set.assert_called_with(TEST_KEY, EXPECTED_VALUE, TEST_DJANGO_TIMEOUT_CACHE) self.assertEqual( self.request_cache.get_cached_response(TEST_KEY).value, EXPECTED_VALUE)
def clear_caches(cls): """ Clear all of the caches defined in settings.CACHES. """ # N.B. As of 2016-04-20, Django won't return any caches # from django.core.cache.caches.all() that haven't been # accessed using caches[name] previously, so we loop # over our list of overridden caches, instead. for cache in settings.CACHES: caches[cache].clear() TieredCache.dangerous_clear_all_tiers()
def test_set_date_for_block(self, initial_date, override_date, expected_date): items = make_items() first = items[0] block_id = first[0] items[0][1]['due'] = initial_date api.set_dates_for_course(str(block_id.course_key), items) api.set_date_for_block(block_id.course_key, block_id, 'due', override_date) TieredCache.dangerous_clear_all_tiers() retrieved = api.get_dates_for_course(block_id.course_key, user=self.user.id) assert len(retrieved) == NUM_OVERRIDES assert retrieved[block_id, 'due'] == expected_date
def test_with_unsupported_routing_strategy(self, mocked_logger, mocked_post): RouterConfigurationFactory.create( backend_name='test_backend', enabled=True, route_url='http://test3.com', auth_scheme=RouterConfiguration.AUTH_BEARER, auth_key='test_key', configurations=ROUTER_CONFIG_FIXTURE[0]) router = EventsRouter(processors=[], backend_name='test_backend') TieredCache.dangerous_clear_all_tiers() router.send(self.transformed_event) mocked_logger.error.assert_called_once_with( 'Unsupported routing strategy detected: INVALID_TYPE') mocked_post.assert_not_called()
def save(self, force_insert=False, force_update=False, using=None, update_fields=None): """ Clear the cached value when saving a new configuration entry """ # Always create a new entry, instead of updating an existing model self.pk = None # pylint: disable=invalid-name super(ConfigurationModel, self).save(force_insert, force_update, using, update_fields) TieredCache.delete_all_tiers( self.cache_key_name( *[getattr(self, key) for key in self.KEY_FIELDS])) if self.KEY_FIELDS: TieredCache.delete_all_tiers(self.key_values_cache_key_name())
def test_get_dates_for_course_query_counts(self, has_schedule, pass_user_object, pass_schedule, item_count): if not has_schedule: self.schedule.delete() items = [ (make_block_id(self.course.id), {'due': datetime(2020, 1, 1) + timedelta(days=i)}) for i in range(item_count) ] api.set_dates_for_course(self.course.id, items) user = self.user if pass_user_object else self.user.id schedule = self.schedule if pass_schedule and has_schedule else None if has_schedule and pass_schedule: query_count = 2 else: query_count = 3 with self.assertNumQueries(query_count): dates = api.get_dates_for_course( course_id=self.course.id, user=user, schedule=schedule ) # Second time, the request cache eliminates all querying (sometimes)... # If a schedule is not provided, we will get the schedule to avoid caching outdated dates with self.assertNumQueries(0 if schedule else 1): cached_dates = api.get_dates_for_course( course_id=self.course.id, user=user, schedule=schedule ) assert dates == cached_dates # Now wipe all cache tiers... TieredCache.dangerous_clear_all_tiers() # No cached values - so will do *all* queries again. with self.assertNumQueries(query_count): externally_cached_dates = api.get_dates_for_course( course_id=self.course.id, user=user, schedule=schedule ) assert dates == externally_cached_dates # Finally, force uncached behavior with used_cache=False with self.assertNumQueries(query_count): uncached_dates = api.get_dates_for_course( course_id=self.course.id, user=user, schedule=schedule, use_cached=False ) assert dates == uncached_dates
def test_with_no_available_hosts(self, mocked_logger, mocked_post): router_config = RouterConfigurationFactory.create( backend_name='test_backend', enabled=True, route_url='http://test3.com', configurations=ROUTER_CONFIG_FIXTURE[1]) router = EventsRouter(processors=[], backend_name='test_backend') TieredCache.dangerous_clear_all_tiers() router.send(self.transformed_event) mocked_post.assert_not_called() self.assertIn( call( 'Event %s is not allowed to be sent to any host for router %s with backend "%s"', self.transformed_event['name'], router_config.route_url, 'test_backend'), mocked_logger.info.mock_calls)
def test_get_cached_response_django_cache_hit(self, mock_cache_get): mock_cache_get.return_value = EXPECTED_VALUE cached_response = TieredCache.get_cached_response(TEST_KEY) self.assertTrue(cached_response.is_found) self.assertEqual(cached_response.value, EXPECTED_VALUE) cached_response = self.request_cache.get_cached_response(TEST_KEY) self.assertTrue( cached_response.is_found, 'Django cache hit should cache value in request cache.')
def current(cls, *args): """ Return the active configuration entry, either from cache, from the database, or by creating a new empty entry (which is not persisted). """ cache_key = cls.cache_key_name(*args) cached_response = TieredCache.get_cached_response(cache_key) if cached_response.is_found: return cached_response.value key_dict = dict(zip(cls.KEY_FIELDS, args)) try: current = cls.objects.filter( **key_dict).order_by('-change_date')[0] except IndexError: current = cls(**key_dict) TieredCache.set_all_tiers(cache_key, current, cls.cache_timeout) return current
def get_routers(self, backend_name): """ Bring active routers of a backend. A queryset for the active configuration entries only. Only useful if backend_name is passed. This function will return all active routers of a backend. """ if not backend_name: return [] cache_key = get_cache_key(namespace="event_routing_backends", resource=backend_name) cached_response = TieredCache.get_cached_response(cache_key) if cached_response.is_found: return cached_response.value current = self.current_set().filter( backend_name=backend_name, enabled=True).order_by('-change_date') TieredCache.set_all_tiers(cache_key, current, backend_cache_ttl()) return current
def test_get_cached_response_force_cache_miss(self, mock_cache_get): self.request_cache.set(SHOULD_FORCE_CACHE_MISS_KEY, True) mock_cache_get.return_value = EXPECTED_VALUE cached_response = TieredCache.get_cached_response(TEST_KEY) self.assertFalse(cached_response.is_found) cached_response = self.request_cache.get_cached_response(TEST_KEY) self.assertFalse( cached_response.is_found, 'Forced Django cache miss should not cache value in request cache.' )
def test_relative_date_past_cutoff_date(self): course_key = CourseLocator('testX', 'tt101', '2019') start_block = make_block_id(course_key, block_type='course') start_date = datetime(2019, 3, 15) first_block = make_block_id(course_key) first_delta = timedelta(days=1) second_block = make_block_id(course_key) second_delta = timedelta(days=10) end_block = make_block_id(course_key, block_type='course') end_date = datetime(2019, 4, 20) items = [ (start_block, {'start': start_date}), # start dates are always absolute (first_block, {'due': first_delta}), # relative (second_block, {'due': second_delta}), # relative (end_block, {'end': end_date}), # end dates are always absolute ] api.set_dates_for_course(course_key, items) # Try one with just enough as a sanity check self.schedule.created = end_date - second_delta self.schedule.save() dates = [ ((start_block, 'start'), start_date), ((first_block, 'due'), self.schedule.start_date + first_delta), ((second_block, 'due'), self.schedule.start_date + second_delta), ((end_block, 'end'), end_date), ] assert api.get_dates_for_course(course_key, schedule=self.schedule) == dict(dates) TieredCache.dangerous_clear_all_tiers() # Now set schedule start date too close to the end date and verify that we no longer get due dates self.schedule.created = datetime(2019, 4, 15) self.schedule.save() dates = [ ((start_block, 'start'), start_date), ((first_block, 'due'), None), ((second_block, 'due'), None), ((end_block, 'end'), end_date), ] assert api.get_dates_for_course(course_key, schedule=self.schedule) == dict(dates)
def test_enabled_router_is_returned(self): first_router = RouterConfigurationFactory(configurations='{}', enabled=True, route_url='http://test2.com', backend_name='first') second_router = RouterConfigurationFactory( configurations='{}', enabled=False, route_url='http://test3.com', backend_name='second') self.assertEqual( RouterConfiguration.get_enabled_routers('first')[0], first_router) self.assertEqual(RouterConfiguration.get_enabled_routers('second'), None) second_router.enabled = True second_router.save() TieredCache.dangerous_clear_all_tiers() self.assertEqual( RouterConfiguration.get_enabled_routers('second')[0], second_router)
def test_set_user_override(self, initial_date, override_date, expected_date): items = make_items() first = items[0] block_id = first[0] items[0][1]['due'] = initial_date api.set_dates_for_course(str(block_id.course_key), items) api.set_date_for_block(block_id.course_key, block_id, 'due', override_date, user=self.user) TieredCache.dangerous_clear_all_tiers() retrieved = api.get_dates_for_course(block_id.course_key, user=self.user.id) assert len(retrieved) == NUM_OVERRIDES assert retrieved[block_id, 'due'] == expected_date overrides = api.get_overrides_for_block(block_id.course_key, block_id) assert len(overrides) == 1 assert overrides[0][2] == expected_date overrides = list(api.get_overrides_for_user(block_id.course_key, self.user)) assert len(overrides) == 1 assert overrides[0] == {'location': block_id, 'actual_date': expected_date}
def test_remove_user_override(self, initial_date, override_date, expected_date): items = make_items() first = items[0] block_id = first[0] items[0][1]['due'] = initial_date api.set_dates_for_course(str(block_id.course_key), items) api.set_date_for_block(block_id.course_key, block_id, 'due', override_date, user=self.user) TieredCache.dangerous_clear_all_tiers() retrieved = api.get_dates_for_course(block_id.course_key, user=self.user.id) assert len(retrieved) == NUM_OVERRIDES assert retrieved[block_id, 'due'] == expected_date api.set_date_for_block(block_id.course_key, block_id, 'due', None, user=self.user) TieredCache.dangerous_clear_all_tiers() retrieved = api.get_dates_for_course(block_id.course_key, user=self.user.id) assert len(retrieved) == NUM_OVERRIDES if isinstance(initial_date, timedelta): user_initial_date = self.schedule.start_date + initial_date else: user_initial_date = initial_date assert retrieved[block_id, 'due'] == user_initial_date
def test_delete(self, mock_cache_delete): TieredCache.set_all_tiers(TEST_KEY, EXPECTED_VALUE) TieredCache.set_all_tiers(TEST_KEY_2, EXPECTED_VALUE) TieredCache.delete_all_tiers(TEST_KEY) self.assertFalse( self.request_cache.get_cached_response(TEST_KEY).is_found) self.assertEqual( self.request_cache.get_cached_response(TEST_KEY_2).value, EXPECTED_VALUE) mock_cache_delete.assert_called_with(TEST_KEY)
def setUp(self): super().setUp() self.request_cache = RequestCache() TieredCache.dangerous_clear_all_tiers()
def test_successful_routing_of_event( self, auth_scheme, auth_key, username, password, backend_name, route_url, mocked_lrs, mocked_post, ): TieredCache.dangerous_clear_all_tiers() mocked_oauth_client = MagicMock() mocked_api_key_client = MagicMock() MOCKED_MAP = { 'AUTH_HEADERS': HttpClient, 'OAUTH2': mocked_oauth_client, 'API_KEY': mocked_api_key_client, 'XAPI_LRS': LrsClient, } RouterConfigurationFactory.create( backend_name=backend_name, enabled=True, route_url=route_url, auth_scheme=auth_scheme, auth_key=auth_key, username=username, password=password, configurations=ROUTER_CONFIG_FIXTURE[0]) router = EventsRouter(processors=[], backend_name=backend_name) with patch.dict('event_routing_backends.tasks.ROUTER_STRATEGY_MAPPING', MOCKED_MAP): router.send(self.transformed_event) overridden_event = self.transformed_event.copy() overridden_event['new_key'] = 'new_value' if backend_name == RouterConfiguration.XAPI_BACKEND: # test LRS Client mocked_lrs().save_statement.assert_has_calls([ call(overridden_event), ]) else: # test the HTTP client if auth_scheme == RouterConfiguration.AUTH_BASIC: mocked_post.assert_has_calls([ call(url=route_url, json=overridden_event, headers={}, auth=(username, password)), ]) elif auth_scheme == RouterConfiguration.AUTH_BEARER: mocked_post.assert_has_calls([ call(url=route_url, json=overridden_event, headers={ 'Authorization': RouterConfiguration.AUTH_BEARER + ' ' + auth_key }), ]) else: mocked_post.assert_has_calls([ call( url=route_url, json=overridden_event, headers={}, ), ]) # test mocked oauth client mocked_oauth_client.assert_not_called()
def test_get_cached_response_all_tier_miss(self): cached_response = TieredCache.get_cached_response(TEST_KEY) self.assertFalse(cached_response.is_found)
def test_dangerous_clear_all_tiers_and_namespaces(self, mock_cache_clear): TieredCache.set_all_tiers(TEST_KEY, EXPECTED_VALUE) TieredCache.dangerous_clear_all_tiers() self.assertFalse( self.request_cache.get_cached_response(TEST_KEY).is_found) mock_cache_clear.assert_called_once_with()
def test_get_cached_response_request_cache_hit(self): self.request_cache.set(TEST_KEY, EXPECTED_VALUE) cached_response = TieredCache.get_cached_response(TEST_KEY) self.assertTrue(cached_response.is_found) self.assertEqual(cached_response.value, EXPECTED_VALUE)