def test_active_job_not_cleaned(self):
mapper = rest_providers.LabelsOnStudentsGenerator(self.course_one)
mapper.submit()
self.execute_all_deferred_tasks(iteration_limit=1)
self.assertEquals(1, self._get_num_root_jobs(COURSE_ONE))
self.assertEquals(0, self._clean_jobs(datetime.timedelta(minutes=1)))
def test_completed_job_is_not_cleaned(self):
mapper = rest_providers.LabelsOnStudentsGenerator(self.course_one)
mapper.submit()
self.execute_all_deferred_tasks()
self._force_finalize(mapper)
self.assertEquals(1, self._get_num_root_jobs(COURSE_ONE))
self.assertEquals(0, self._clean_jobs(datetime.timedelta(minutes=1)))
def test_cleanup_modifies_incomplete_status(self):
mapper = rest_providers.LabelsOnStudentsGenerator(self.course_one)
mapper.submit()
self.execute_all_deferred_tasks(iteration_limit=1)
self.assertEquals(jobs.STATUS_CODE_STARTED, mapper.load().status_code)
self.assertEquals(1, self._clean_jobs(datetime.timedelta(seconds=0)))
self.assertEquals(jobs.STATUS_CODE_FAILED, mapper.load().status_code)
self.assertIn('assumed to have failed', mapper.load().output)
def test_incomplete_job_cleaned_if_time_expired(self):
mapper = rest_providers.LabelsOnStudentsGenerator(self.course_one)
mapper.submit()
self.execute_all_deferred_tasks(iteration_limit=1)
self.assertEquals(1, self._get_num_root_jobs(COURSE_ONE))
self.assertEquals(1, self._clean_jobs(datetime.timedelta(seconds=0)))
self.execute_all_deferred_tasks() # Run deferred deletion task.
self.assertEquals(0, self._get_num_root_jobs(COURSE_ONE))
def test_cleanup_does_not_modify_completed_status(self):
mapper = rest_providers.LabelsOnStudentsGenerator(self.course_one)
mapper.submit()
self.execute_all_deferred_tasks()
self.assertEquals(jobs.STATUS_CODE_COMPLETED,
mapper.load().status_code)
self.assertEquals(1, self._clean_jobs(datetime.timedelta(seconds=0)))
self.assertEquals(jobs.STATUS_CODE_COMPLETED,
mapper.load().status_code)
def test_completed_job_cleaned_if_time_expired(self):
mapper = rest_providers.LabelsOnStudentsGenerator(self.course_one)
mapper.submit()
self.execute_all_deferred_tasks()
self.assertEquals(1, self._get_num_root_jobs(COURSE_ONE))
self.assertEquals(1, self._clean_jobs(datetime.timedelta(seconds=0)))
paths = self._get_cloudstore_paths(COURSE_ONE)
self.assertTrue(len(paths) == 6 or len(paths) == 3)
self.execute_all_deferred_tasks() # Run deferred deletion task.
self.assertEquals(0, self._get_num_root_jobs(COURSE_ONE))
self._assert_cloudstore_paths_removed(COURSE_ONE, paths)
def test_cleanup_in_multiple_namespaces(self):
mapper_one = rest_providers.LabelsOnStudentsGenerator(self.course_one)
mapper_two = rest_providers.LabelsOnStudentsGenerator(self.course_two)
for _ in range(0, 2):
mapper_one.submit()
mapper_two.submit()
self.execute_all_deferred_tasks()
self.assertEquals(2, self._get_num_root_jobs(COURSE_ONE))
course_one_paths = self._get_blobstore_paths(COURSE_ONE)
self.assertEquals(16, len(course_one_paths))
self.assertEquals(2, self._get_num_root_jobs(COURSE_TWO))
course_two_paths = self._get_blobstore_paths(COURSE_TWO)
self.assertEquals(16, len(course_two_paths))
self.assertEquals(4, self._clean_jobs(datetime.timedelta(seconds=0)))
self.execute_all_deferred_tasks() # Run deferred deletion task.
self.assertEquals(0, self._get_num_root_jobs(COURSE_ONE))
self.assertEquals(0, self._get_num_root_jobs(COURSE_TWO))
self._assert_blobstore_paths_removed(COURSE_ONE, course_one_paths)
self._assert_blobstore_paths_removed(COURSE_TWO, course_two_paths)
def test_multiple_runs_cleaned(self):
mapper = rest_providers.LabelsOnStudentsGenerator(self.course_one)
for _ in range(0, 3):
mapper.submit()
self.execute_all_deferred_tasks()
self.assertEquals(3, self._get_num_root_jobs(COURSE_ONE))
self.assertEquals(3, self._clean_jobs(datetime.timedelta(seconds=0)))
paths = self._get_cloudstore_paths(COURSE_ONE)
self.assertTrue(len(paths) == 18 or len(paths) == 9)
self.execute_all_deferred_tasks() # Run deferred deletion task.
self.assertEquals(0, self._get_num_root_jobs(COURSE_ONE))
self._assert_cloudstore_paths_removed(COURSE_ONE, paths)
def test_cleanup_handler(self):
mapper = rest_providers.LabelsOnStudentsGenerator(self.course_one)
mapper.submit()
self.execute_all_deferred_tasks(iteration_limit=1)
mapper.cancel()
self.execute_all_deferred_tasks()
self.assertEquals(1, self._get_num_root_jobs(COURSE_ONE))
# Check that hitting the cron handler via GET works as well.
# Note that since the actual handler uses a max time limit of
# a few days, we need to set up a canceled job which, having
# no defined start-time will be cleaned up immediately.
self.get('/cron/mapreduce/cleanup')
self.execute_all_deferred_tasks(iteration_limit=1)
self.assertEquals(0, self._get_num_root_jobs(COURSE_ONE))
def test_unstarted_job_not_cleaned(self):
mapper = rest_providers.LabelsOnStudentsGenerator(self.course_one)
mapper.submit()
self.assertEquals(1, self._get_num_root_jobs(COURSE_ONE))
self.assertEquals(0, self._clean_jobs(datetime.timedelta(minutes=1)))
