def test_multiple_students(self):
# Register two students
user = actions.login(self.STUDENT_EMAIL)
actions.register(self, user.email(), course=self.COURSE)
other_user = actions.login('*****@*****.**')
actions.register(self, other_user.email(), course=self.COURSE)
# Get IDs of those students; make an event for each.
with common_utils.Namespace(self.NAMESPACE):
student1_id = (
models.Student.get_by_user(user).user_id)
student2_id = (
models.Student.get_by_user(other_user).user_id)
models.EventEntity(user_id=student1_id, source='test').put()
models.EventEntity(user_id=student2_id, source='test').put()
# Unregister one of them.
actions.login(self.STUDENT_EMAIL)
self._unregister_and_request_data_removal(self.COURSE)
self._complete_removal()
# Unregistered student and his data are gone; still-registered
# student's data is still present.
with common_utils.Namespace(self.NAMESPACE):
self.assertIsNone(models.Student.get_by_user(user))
self.assertIsNotNone(models.Student.get_by_user(other_user))
entities = list(models.EventEntity.all().run())
self.assertEquals(1, len(entities))
self.assertEquals(student2_id, entities[0].user_id)
def test_utf8_datastore(self):
"""Test writing to and reading from datastore using UTF-8 content."""
event = models.EventEntity()
event.source = 'test-source'
event.user_id = 'test-user-id'
event.data = u'Test Data (тест данные)'
event.put()
stored_event = models.EventEntity().get_by_id([event.key().id()])
assert 1 == len(stored_event)
assert event.data == stored_event[0].data
def test_for_export_transforms_correctly(self):
event = models.EventEntity(source='source', user_id='1')
key = event.put()
exported = event.for_export(self.transform)
self.assert_blacklisted_properties_removed(event, exported)
self.assertEqual('source', event.source)
self.assertEqual('transformed_1', exported.user_id)
self.assertEqual(key, models.EventEntity.safe_key(key, self.transform))
def test_multiple_students(self):
user = self.make_test_user(self.STUDENT_EMAIL)
other_user = self.make_test_user('*****@*****.**')
# Register two students
actions.login(user.email())
actions.register(self, user.email(), course=self.COURSE)
actions.login(other_user.email())
actions.register(self, other_user.email(), course=self.COURSE)
# Get IDs of those students; make an event for each.
with common_utils.Namespace(self.NAMESPACE):
student1_id = (models.Student.get_by_user(user).user_id)
student2_id = (models.Student.get_by_user(other_user).user_id)
models.EventEntity(user_id=student1_id, source='test').put()
models.EventEntity(user_id=student2_id, source='test').put()
# Unregister one of them.
actions.login(self.STUDENT_EMAIL)
actions.unregister(self, self.COURSE, do_data_removal=True)
# Complete all data removal tasks.
self.execute_all_deferred_tasks(
models.StudentLifecycleObserver.QUEUE_NAME)
self.get(data_removal.DataRemovalCronHandler.URL,
headers={'X-AppEngine-Cron': 'True'})
self.execute_all_deferred_tasks()
# Unregistered student and his data are gone; still-registered
# student's data is still present.
with common_utils.Namespace(self.NAMESPACE):
self.assertIsNone(models.Student.get_by_user(user))
self.assertIsNotNone(models.Student.get_by_user(other_user))
entities = list(models.EventEntity.all().run())
self.assertEquals(1, len(entities))
self.assertEquals(student2_id, entities[0].user_id)
def test_immediate_removal_policy(self):
user = self.make_test_user(self.STUDENT_EMAIL)
actions.login(user.email())
actions.register(self, self.STUDENT_EMAIL, course=self.COURSE)
task_count = self.execute_all_deferred_tasks(
models.StudentLifecycleObserver.QUEUE_NAME)
self.assertEquals(1, task_count) # registration.
user_id = None
with common_utils.Namespace(self.NAMESPACE):
# After registration, we should have a student object, and
# a ImmediateRemovalState instance.
student = models.Student.get_by_user(user)
self.assertIsNotNone(student)
user_id = student.user_id
removal_state = removal_models.ImmediateRemovalState.get_by_user_id(
user_id)
self.assertIsNotNone(removal_state)
self.assertEquals(
removal_models.ImmediateRemovalState.STATE_REGISTERED,
removal_state.state)
r = removal_models.BatchRemovalState.get_by_user_ids([user_id])
self.assertEqual([None], r)
# Add an EventEntity record so we can see it being removed.
event = models.EventEntity(user_id=user_id, source='test')
event.put()
actions.unregister(self, self.COURSE, do_data_removal=True)
with common_utils.Namespace(self.NAMESPACE):
# Immediately upon unregistration, we should still have the student
# record, and removal state should be pending deletion.
student = models.Student.get_by_user(user)
self.assertIsNotNone(student)
removal_state = removal_models.ImmediateRemovalState.get_by_user_id(
user_id)
self.assertIsNotNone(removal_state)
self.assertEquals(
removal_models.ImmediateRemovalState.STATE_DELETION_PENDING,
removal_state.state)
r = removal_models.BatchRemovalState.get_by_user_ids([user_id])
self.assertEqual([None], r)
events = list(models.EventEntity.all().run())
self.assertEquals(1, len(events))
# We should have gotten a to-do item on the task queue for student
# removal.
task_count = self.execute_all_deferred_tasks(
models.StudentLifecycleObserver.QUEUE_NAME)
self.assertEquals(1, task_count) # unregistration.
with common_utils.Namespace(self.NAMESPACE):
# Having processed the queue item, the student record should now
# be gone.
students = list(models.Student.all().run())
student = models.Student.get_by_user(user)
self.assertIsNone(student)
# But the record tracking removal should not yet be gone.
removal_state = removal_models.ImmediateRemovalState.get_by_user_id(
user_id)
self.assertIsNotNone(removal_state)
self.assertEquals(
removal_models.ImmediateRemovalState.STATE_DELETION_PENDING,
removal_state.state)
# And we should have a to-do item for the cron batch cleanup.
r = removal_models.BatchRemovalState.get_by_user_ids([user_id])
self.assertEquals(1, len(r))
removal_record = r[0]
self.assertEquals(
models_data_removal.Registry.get_unindexed_class_names(),
removal_record.resource_types)
# Events won't have been cleaned up yet; need cron batch to run.
events = list(models.EventEntity.all().run())
self.assertEquals(1, len(events))
# Call the cron handler to schedule batch removal tasks. This, in
# turn, will schedule map/reduce jobs to remove records for that
# student.
response = self.get(data_removal.DataRemovalCronHandler.URL,
headers={'X-AppEngine-Cron': 'True'})
self.assertEquals(200, response.status_int)
self.assertEquals('OK.', response.body)
# Run the map/reduce jobs to completion.
self.execute_all_deferred_tasks()
# We should now be completely clean; the M/R job that finishes last
# should also clean up the to-do tracking item.
with common_utils.Namespace(self.NAMESPACE):
student = models.Student.get_by_user(user)
self.assertIsNone(student)
removal_state = removal_models.ImmediateRemovalState.get_by_user_id(
user_id)
self.assertIsNone(removal_state)
# Events should now be gone.
events = list(models.EventEntity.all().run())
self.assertEquals(0, len(events))
def test_immediate_removal_policy(self):
user = actions.login(self.STUDENT_EMAIL)
actions.register(self, self.STUDENT_EMAIL, course=self.COURSE)
task_count = self.execute_all_deferred_tasks(
models.StudentLifecycleObserver.QUEUE_NAME)
self.assertEquals(1, task_count) # registration.
user_id = None
with common_utils.Namespace(self.NAMESPACE):
# After registration, we should have a student object, and
# a ImmediateRemovalState instance, and no to-do deletion work.
student = models.Student.get_by_user(user)
self.assertIsNotNone(student)
user_id = student.user_id
removal_state = removal_models.ImmediateRemovalState.get_by_user_id(
user_id)
self.assertIsNotNone(removal_state)
self.assertEquals(
removal_models.ImmediateRemovalState.STATE_REGISTERED,
removal_state.state)
r = removal_models.BatchRemovalState.get_by_user_ids([user_id])
self.assertEqual([None], r)
# Add an EventEntity record so we can see it being removed.
event = models.EventEntity(user_id=user_id, source='test')
event.put()
self._unregister_and_request_data_removal(self.COURSE)
with common_utils.Namespace(self.NAMESPACE):
# Immediately upon unregistration, we should still have the student
# record, and removal state should be pending deletion.
student = models.Student.get_by_user(user)
self.assertIsNotNone(student)
removal_state = removal_models.ImmediateRemovalState.get_by_user_id(
user_id)
self.assertIsNotNone(removal_state)
self.assertEquals(
removal_models.ImmediateRemovalState.STATE_DELETION_PENDING,
removal_state.state)
r = removal_models.BatchRemovalState.get_by_user_ids([user_id])
self.assertEqual([None], r)
events = list(models.EventEntity.all().run())
self.assertEquals(1, len(events))
# We should have gotten a to-do item on the task queue for student
# removal.
task_count = self.execute_all_deferred_tasks(
models.StudentLifecycleObserver.QUEUE_NAME)
self.assertEquals(1, task_count) # unregistration.
with common_utils.Namespace(self.NAMESPACE):
# Having processed the queue item, the student record should now
# be gone.
students = list(models.Student.all().run())
student = models.Student.get_by_user(user)
self.assertIsNone(student)
# But the record tracking removal should not yet be gone.
removal_state = removal_models.ImmediateRemovalState.get_by_user_id(
user_id)
self.assertIsNotNone(removal_state)
self.assertEquals(
removal_models.ImmediateRemovalState.STATE_DELETION_PENDING,
removal_state.state)
# And we should have a to-do item for the cron batch cleanup.
r = removal_models.BatchRemovalState.get_by_user_ids([user_id])
self.assertEquals(1, len(r))
removal_record = r[0]
self.assertEquals(
models_data_removal.Registry.get_unindexed_class_names(),
removal_record.resource_types)
# Events won't have been cleaned up yet; need cron batch to run.
events = list(models.EventEntity.all().run())
self.assertEquals(1, len(events))
# Call the cron handler to schedule batch removal tasks. This, in
# turn, will schedule map/reduce jobs to remove records for that
# student.
response = self.get(
data_removal.DataRemovalCronHandler.URL,
headers={'X-AppEngine-Cron': 'True'})
self.assertEquals(200, response.status_int)
self.assertEquals('OK.', response.body)
# Run the map/reduce jobs to completion.
self.execute_all_deferred_tasks()
# We should now be nearly clean; in the normal course of events, only
# the ImmediateRemovalState should still be present. However, due to
# race conditions, an analysis map/reduce job may have finished in the
# meantime, and written a per-student record. Add such a record.
with common_utils.Namespace(self.NAMESPACE):
student = models.Student.get_by_user(user)
self.assertIsNone(student)
removal_state = removal_models.ImmediateRemovalState.get_by_user_id(
user_id)
self.assertIsNotNone(removal_state)
# Events should now be gone.
events = list(models.EventEntity.all().run())
self.assertEquals(0, len(events))
# Cron batch cleanup record should be present, but now empty.
r = removal_models.BatchRemovalState.get_by_user_ids([user_id])
self.assertEquals(1, len(r))
removal_record = r[0]
self.assertEquals([], removal_record.resource_types)
# Simulate map/reduce finishing asychronously & adding a per-student
# item. Verify that the record is present so we know the test
# below that checks for it being gone is correct.
student_aggregate.StudentAggregateEntity(key_name=user_id).put()
a = student_aggregate.StudentAggregateEntity.get_by_key_name(
user_id)
self.assertIsNotNone(a)
# Call the cron handler one more time. Because the batch work item
# is empty, this should do one more round of cleanup on items indexed
# by user id.
response = self.get(
data_removal.DataRemovalCronHandler.URL,
headers={'X-AppEngine-Cron': 'True'})
self.assertEquals(200, response.status_int)
self.assertEquals('OK.', response.body)
# We should now have zero data about the user.
with common_utils.Namespace(self.NAMESPACE):
student = models.Student.get_by_user(user)
self.assertIsNone(student)
removal_state = removal_models.ImmediateRemovalState.get_by_user_id(
user_id)
self.assertIsNone(removal_state)
# Events should now be gone.
events = list(models.EventEntity.all().run())
self.assertEquals(0, len(events))
# Cron batch cleanup record should be gone.
r = removal_models.BatchRemovalState.get_by_user_ids([user_id])
self.assertEqual([None], r)
# Map/reduce results should be gone.
a = student_aggregate.StudentAggregateEntity.get_by_key_name(
user_id)
self.assertIsNone(a)