def __do_run(): """ Resume a suspended batch job if it exists, or create a new one if not. """ logging.info("Loading batch job") job = BatchJob.get_or_create() config = DeploymentConfig.get_instance() if not job.student_data_loaded: logging.info("Loading students to DB") load_students_to_db(config) job.student_data_loaded = True job.put() # The report generation step might take awhile for large data sets, # requeue here just in case to reset the task queue timer. # (Because of the sorting, it has to be completed atomically) queue() return if not job.reports_complete: logging.info("Generating reports") print_reports(config) job.reports_complete = True job.put() logging.info("Cleaning up temporary records") Student.delete_all() logging.info("Anonymizer run complete, terminating batch job") job.key.delete()
def get_dummy_config(): # Set up a dummy DeploymentConfig instance. # Test cases will likely edit the member vars to facilitate testing; # creating the instance in setUp ensures we always get a fresh copy. config = DeploymentConfig.get_instance() config.put() return config
def get_dummy_config(): # Set up a dummy DeploymentConfig instance. SPS==1 means that we will # create one shard per student during the test; it's important to force # multiple shards so we can test things like the generators. config = DeploymentConfig.get_instance() config.estimated_student_count = len(get_dummy_ids()) config.students_per_shard = 1 config.put() return config
def get_dummy_config(): # Set up a dummy DeploymentConfig instance. # Test cases will likely edit the member vars to facilitate testing; # creating the instance in setUp ensures we always get a fresh copy. config = DeploymentConfig.get_instance() config.import_filename = "/input_bucket/sm.csv" config.output_bucket = "output_bucket" config.output_anon_subdir = "/anon-subdir" config.output_school_subdir = "/school-subdir" config.put() return config
def get_dummy_config(): # Set up a dummy DeploymentConfig instance. # Test cases will likely edit the member vars to facilitate testing; # creating the instance in setUp ensures we always get a fresh copy. config = DeploymentConfig.get_instance() config.output_bucket = "bucket" config.output_anon_subdir = "/anon-subdir" config.output_school_subdir = "/school-subdir" config.school_info = [("1", "school1"), ("2", "school2"), ("default", "misc")] config.put() return config
def post_auth_success(self): config = DeploymentConfig.get_instance() # Importers tab config.importer = self.request.get("importer") config.import_filename = self.request.get("import_filename") # Accounts tab config.email_domain = self.request.get("email_domain") config.anon_fname_config = self.request.get("anon_fname_config") config.anon_lname_config = self.request.get("anon_lname_config") config.username_config = self.request.get("username_config") config.anon_username_config = self.request.get("anon_username_config") config.password_config = self.request.get("password_config") config.suppress_first_username = self.request.get( "suppress_first_username") == "True" config.suppress_first_anon_username = self.request.get( "suppress_first_anon_username") == "True" # Schools tab school_count = 0 for i in self.request.arguments(): if re.match(r"school_", i): school_count += 1 # Two fields per school with the school_ prefix; # one for school number, one for abbreviation. school_count /= 2 config.school_info = [] for i in range(school_count): num = self.request.get("school_num_{}".format(i)) abbrev = self.request.get("school_abbrev_{}".format(i)) if num and abbrev: config.school_info.append((num, abbrev)) # Exporters tab config.output_bucket = self.request.get("output_bucket") config.output_school_subdir = self.request.get("output_school_subdir") config.output_anon_subdir = self.request.get("output_anon_subdir") config.dummy_teacher_username = self.request.get("dummy_teacher_username") config.dummy_teacher_pass = self.request.get("dummy_teacher_pass") # Advanced tab config.estimated_student_count = int(self.request.get("est_student_count")) config.students_per_shard = int(self.request.get("students_per_shard")) config.unique_strings_case_sensitive = self.request.get("case_sensitive") == "True" config.importer_batch_size = int(self.request.get("importer_batch_size")) config.driver_batch_size = int(self.request.get("driver_batch_size")) config.filter_json = self.request.get("filter_json") config.put() self.render_front(config=config, result="Changes saved")
def post_auth_success(self): config = DeploymentConfig.get_instance() config.key.delete() self.redirect("/config")
def render_front(self, config=None, result=None): if not config: config = DeploymentConfig.get_instance() self.render("config.html", dc=config, result=result)