def __do_run(): """ Resume a suspended batch job if it exists, or create a new one if not. """ logging.info("Loading batch job") job = BatchJob.get_or_create() config = DeploymentConfig.get_instance() if not job.student_data_loaded: logging.info("Loading students to DB") load_students_to_db(config) job.student_data_loaded = True job.put() # The report generation step might take awhile for large data sets, # requeue here just in case to reset the task queue timer. # (Because of the sorting, it has to be completed atomically) queue() return if not job.reports_complete: logging.info("Generating reports") print_reports(config) job.reports_complete = True job.put() logging.info("Cleaning up temporary records") Student.delete_all() logging.info("Anonymizer run complete, terminating batch job") job.key.delete()
def __unstick(): """ DO NOT call this method directly, use queue_unstick() instead. """ logging.info("Clearing stuck batch job") logging.info("Wiping temporary records from student table") Student.delete_all() logging.info("Deleting batch job entity") BatchJob.get_or_create().key.delete() logging.info("Batch job cleared")
def test_student_put(self): # Ensure empty starting DB result = Student().query().fetch(limit=None) self.assertFalse(result) # Test basic db put stu1 = get_dummy_student(1) stu1.put() result = Student().query().fetch(limit=None) self.assertEqual(1, len(result)) # Test get_all and delete_all stu2 = get_dummy_student(2) stu3 = get_dummy_student(3) stu4 = get_dummy_student(4) stu2.put() stu3.put() stu4.put() id_set1 = Student.get_all_id_numbers() id_set2 = set(get_dummy_ids()) self.assertEqual(id_set1, id_set2) Student.delete_all() result = Student().query().fetch(limit=None) self.assertFalse(result) # Same as above, with circular sharding setup_dummy_students("circular") id_set1 = Student.get_all_id_numbers() id_set2 = set(get_dummy_ids()) self.assertEqual(id_set1, id_set2) Student.delete_all() result = Student().query().fetch(limit=None) self.assertFalse(result) # Same as above, with linear sharding setup_dummy_students("linear") id_set1 = Student.get_all_id_numbers() id_set2 = set(get_dummy_ids()) self.assertEqual(id_set1, id_set2) Student.delete_all() result = Student().query().fetch(limit=None) self.assertFalse(result)