def test_multiple_students(self): # Register two students user = actions.login(self.STUDENT_EMAIL) actions.register(self, user.email(), course=self.COURSE) other_user = actions.login('*****@*****.**') actions.register(self, other_user.email(), course=self.COURSE) # Get IDs of those students; make an event for each. with common_utils.Namespace(self.NAMESPACE): student1_id = ( models.Student.get_by_user(user).user_id) student2_id = ( models.Student.get_by_user(other_user).user_id) models.EventEntity(user_id=student1_id, source='test').put() models.EventEntity(user_id=student2_id, source='test').put() # Unregister one of them. actions.login(self.STUDENT_EMAIL) self._unregister_and_request_data_removal(self.COURSE) self._complete_removal() # Unregistered student and his data are gone; still-registered # student's data is still present. with common_utils.Namespace(self.NAMESPACE): self.assertIsNone(models.Student.get_by_user(user)) self.assertIsNotNone(models.Student.get_by_user(other_user)) entities = list(models.EventEntity.all().run()) self.assertEquals(1, len(entities)) self.assertEquals(student2_id, entities[0].user_id)
def test_utf8_datastore(self): """Test writing to and reading from datastore using UTF-8 content.""" event = models.EventEntity() event.source = 'test-source' event.user_id = 'test-user-id' event.data = u'Test Data (тест данные)' event.put() stored_event = models.EventEntity().get_by_id([event.key().id()]) assert 1 == len(stored_event) assert event.data == stored_event[0].data
def test_for_export_transforms_correctly(self): event = models.EventEntity(source='source', user_id='1') key = event.put() exported = event.for_export(self.transform) self.assert_blacklisted_properties_removed(event, exported) self.assertEqual('source', event.source) self.assertEqual('transformed_1', exported.user_id) self.assertEqual(key, models.EventEntity.safe_key(key, self.transform))
def test_multiple_students(self): user = self.make_test_user(self.STUDENT_EMAIL) other_user = self.make_test_user('*****@*****.**') # Register two students actions.login(user.email()) actions.register(self, user.email(), course=self.COURSE) actions.login(other_user.email()) actions.register(self, other_user.email(), course=self.COURSE) # Get IDs of those students; make an event for each. with common_utils.Namespace(self.NAMESPACE): student1_id = (models.Student.get_by_user(user).user_id) student2_id = (models.Student.get_by_user(other_user).user_id) models.EventEntity(user_id=student1_id, source='test').put() models.EventEntity(user_id=student2_id, source='test').put() # Unregister one of them. actions.login(self.STUDENT_EMAIL) actions.unregister(self, self.COURSE, do_data_removal=True) # Complete all data removal tasks. self.execute_all_deferred_tasks( models.StudentLifecycleObserver.QUEUE_NAME) self.get(data_removal.DataRemovalCronHandler.URL, headers={'X-AppEngine-Cron': 'True'}) self.execute_all_deferred_tasks() # Unregistered student and his data are gone; still-registered # student's data is still present. with common_utils.Namespace(self.NAMESPACE): self.assertIsNone(models.Student.get_by_user(user)) self.assertIsNotNone(models.Student.get_by_user(other_user)) entities = list(models.EventEntity.all().run()) self.assertEquals(1, len(entities)) self.assertEquals(student2_id, entities[0].user_id)
def test_immediate_removal_policy(self): user = self.make_test_user(self.STUDENT_EMAIL) actions.login(user.email()) actions.register(self, self.STUDENT_EMAIL, course=self.COURSE) task_count = self.execute_all_deferred_tasks( models.StudentLifecycleObserver.QUEUE_NAME) self.assertEquals(1, task_count) # registration. user_id = None with common_utils.Namespace(self.NAMESPACE): # After registration, we should have a student object, and # a ImmediateRemovalState instance. student = models.Student.get_by_user(user) self.assertIsNotNone(student) user_id = student.user_id removal_state = removal_models.ImmediateRemovalState.get_by_user_id( user_id) self.assertIsNotNone(removal_state) self.assertEquals( removal_models.ImmediateRemovalState.STATE_REGISTERED, removal_state.state) r = removal_models.BatchRemovalState.get_by_user_ids([user_id]) self.assertEqual([None], r) # Add an EventEntity record so we can see it being removed. event = models.EventEntity(user_id=user_id, source='test') event.put() actions.unregister(self, self.COURSE, do_data_removal=True) with common_utils.Namespace(self.NAMESPACE): # Immediately upon unregistration, we should still have the student # record, and removal state should be pending deletion. student = models.Student.get_by_user(user) self.assertIsNotNone(student) removal_state = removal_models.ImmediateRemovalState.get_by_user_id( user_id) self.assertIsNotNone(removal_state) self.assertEquals( removal_models.ImmediateRemovalState.STATE_DELETION_PENDING, removal_state.state) r = removal_models.BatchRemovalState.get_by_user_ids([user_id]) self.assertEqual([None], r) events = list(models.EventEntity.all().run()) self.assertEquals(1, len(events)) # We should have gotten a to-do item on the task queue for student # removal. task_count = self.execute_all_deferred_tasks( models.StudentLifecycleObserver.QUEUE_NAME) self.assertEquals(1, task_count) # unregistration. with common_utils.Namespace(self.NAMESPACE): # Having processed the queue item, the student record should now # be gone. students = list(models.Student.all().run()) student = models.Student.get_by_user(user) self.assertIsNone(student) # But the record tracking removal should not yet be gone. removal_state = removal_models.ImmediateRemovalState.get_by_user_id( user_id) self.assertIsNotNone(removal_state) self.assertEquals( removal_models.ImmediateRemovalState.STATE_DELETION_PENDING, removal_state.state) # And we should have a to-do item for the cron batch cleanup. r = removal_models.BatchRemovalState.get_by_user_ids([user_id]) self.assertEquals(1, len(r)) removal_record = r[0] self.assertEquals( models_data_removal.Registry.get_unindexed_class_names(), removal_record.resource_types) # Events won't have been cleaned up yet; need cron batch to run. events = list(models.EventEntity.all().run()) self.assertEquals(1, len(events)) # Call the cron handler to schedule batch removal tasks. This, in # turn, will schedule map/reduce jobs to remove records for that # student. response = self.get(data_removal.DataRemovalCronHandler.URL, headers={'X-AppEngine-Cron': 'True'}) self.assertEquals(200, response.status_int) self.assertEquals('OK.', response.body) # Run the map/reduce jobs to completion. self.execute_all_deferred_tasks() # We should now be completely clean; the M/R job that finishes last # should also clean up the to-do tracking item. with common_utils.Namespace(self.NAMESPACE): student = models.Student.get_by_user(user) self.assertIsNone(student) removal_state = removal_models.ImmediateRemovalState.get_by_user_id( user_id) self.assertIsNone(removal_state) # Events should now be gone. events = list(models.EventEntity.all().run()) self.assertEquals(0, len(events))
def test_immediate_removal_policy(self): user = actions.login(self.STUDENT_EMAIL) actions.register(self, self.STUDENT_EMAIL, course=self.COURSE) task_count = self.execute_all_deferred_tasks( models.StudentLifecycleObserver.QUEUE_NAME) self.assertEquals(1, task_count) # registration. user_id = None with common_utils.Namespace(self.NAMESPACE): # After registration, we should have a student object, and # a ImmediateRemovalState instance, and no to-do deletion work. student = models.Student.get_by_user(user) self.assertIsNotNone(student) user_id = student.user_id removal_state = removal_models.ImmediateRemovalState.get_by_user_id( user_id) self.assertIsNotNone(removal_state) self.assertEquals( removal_models.ImmediateRemovalState.STATE_REGISTERED, removal_state.state) r = removal_models.BatchRemovalState.get_by_user_ids([user_id]) self.assertEqual([None], r) # Add an EventEntity record so we can see it being removed. event = models.EventEntity(user_id=user_id, source='test') event.put() self._unregister_and_request_data_removal(self.COURSE) with common_utils.Namespace(self.NAMESPACE): # Immediately upon unregistration, we should still have the student # record, and removal state should be pending deletion. student = models.Student.get_by_user(user) self.assertIsNotNone(student) removal_state = removal_models.ImmediateRemovalState.get_by_user_id( user_id) self.assertIsNotNone(removal_state) self.assertEquals( removal_models.ImmediateRemovalState.STATE_DELETION_PENDING, removal_state.state) r = removal_models.BatchRemovalState.get_by_user_ids([user_id]) self.assertEqual([None], r) events = list(models.EventEntity.all().run()) self.assertEquals(1, len(events)) # We should have gotten a to-do item on the task queue for student # removal. task_count = self.execute_all_deferred_tasks( models.StudentLifecycleObserver.QUEUE_NAME) self.assertEquals(1, task_count) # unregistration. with common_utils.Namespace(self.NAMESPACE): # Having processed the queue item, the student record should now # be gone. students = list(models.Student.all().run()) student = models.Student.get_by_user(user) self.assertIsNone(student) # But the record tracking removal should not yet be gone. removal_state = removal_models.ImmediateRemovalState.get_by_user_id( user_id) self.assertIsNotNone(removal_state) self.assertEquals( removal_models.ImmediateRemovalState.STATE_DELETION_PENDING, removal_state.state) # And we should have a to-do item for the cron batch cleanup. r = removal_models.BatchRemovalState.get_by_user_ids([user_id]) self.assertEquals(1, len(r)) removal_record = r[0] self.assertEquals( models_data_removal.Registry.get_unindexed_class_names(), removal_record.resource_types) # Events won't have been cleaned up yet; need cron batch to run. events = list(models.EventEntity.all().run()) self.assertEquals(1, len(events)) # Call the cron handler to schedule batch removal tasks. This, in # turn, will schedule map/reduce jobs to remove records for that # student. response = self.get( data_removal.DataRemovalCronHandler.URL, headers={'X-AppEngine-Cron': 'True'}) self.assertEquals(200, response.status_int) self.assertEquals('OK.', response.body) # Run the map/reduce jobs to completion. self.execute_all_deferred_tasks() # We should now be nearly clean; in the normal course of events, only # the ImmediateRemovalState should still be present. However, due to # race conditions, an analysis map/reduce job may have finished in the # meantime, and written a per-student record. Add such a record. with common_utils.Namespace(self.NAMESPACE): student = models.Student.get_by_user(user) self.assertIsNone(student) removal_state = removal_models.ImmediateRemovalState.get_by_user_id( user_id) self.assertIsNotNone(removal_state) # Events should now be gone. events = list(models.EventEntity.all().run()) self.assertEquals(0, len(events)) # Cron batch cleanup record should be present, but now empty. r = removal_models.BatchRemovalState.get_by_user_ids([user_id]) self.assertEquals(1, len(r)) removal_record = r[0] self.assertEquals([], removal_record.resource_types) # Simulate map/reduce finishing asychronously & adding a per-student # item. Verify that the record is present so we know the test # below that checks for it being gone is correct. student_aggregate.StudentAggregateEntity(key_name=user_id).put() a = student_aggregate.StudentAggregateEntity.get_by_key_name( user_id) self.assertIsNotNone(a) # Call the cron handler one more time. Because the batch work item # is empty, this should do one more round of cleanup on items indexed # by user id. response = self.get( data_removal.DataRemovalCronHandler.URL, headers={'X-AppEngine-Cron': 'True'}) self.assertEquals(200, response.status_int) self.assertEquals('OK.', response.body) # We should now have zero data about the user. with common_utils.Namespace(self.NAMESPACE): student = models.Student.get_by_user(user) self.assertIsNone(student) removal_state = removal_models.ImmediateRemovalState.get_by_user_id( user_id) self.assertIsNone(removal_state) # Events should now be gone. events = list(models.EventEntity.all().run()) self.assertEquals(0, len(events)) # Cron batch cleanup record should be gone. r = removal_models.BatchRemovalState.get_by_user_ids([user_id]) self.assertEqual([None], r) # Map/reduce results should be gone. a = student_aggregate.StudentAggregateEntity.get_by_key_name( user_id) self.assertIsNone(a)