def __do_run():
"""
Resume a suspended batch job if it exists, or create a new one if not.
"""
logging.info("Loading batch job")
job = BatchJob.get_or_create()
config = DeploymentConfig.get_instance()
if not job.student_data_loaded:
logging.info("Loading students to DB")
load_students_to_db(config)
job.student_data_loaded = True
job.put()
# The report generation step might take awhile for large data sets,
# requeue here just in case to reset the task queue timer.
# (Because of the sorting, it has to be completed atomically)
queue()
return
if not job.reports_complete:
logging.info("Generating reports")
print_reports(config)
job.reports_complete = True
job.put()
logging.info("Cleaning up temporary records")
Student.delete_all()
logging.info("Anonymizer run complete, terminating batch job")
job.key.delete()
def get_dummy_config():
# Set up a dummy DeploymentConfig instance.
# Test cases will likely edit the member vars to facilitate testing;
# creating the instance in setUp ensures we always get a fresh copy.
config = DeploymentConfig.get_instance()
config.put()
return config
def get_dummy_config():
# Set up a dummy DeploymentConfig instance. SPS==1 means that we will
# create one shard per student during the test; it's important to force
# multiple shards so we can test things like the generators.
config = DeploymentConfig.get_instance()
config.estimated_student_count = len(get_dummy_ids())
config.students_per_shard = 1
config.put()
return config
def get_dummy_config():
# Set up a dummy DeploymentConfig instance.
# Test cases will likely edit the member vars to facilitate testing;
# creating the instance in setUp ensures we always get a fresh copy.
config = DeploymentConfig.get_instance()
config.import_filename = "/input_bucket/sm.csv"
config.output_bucket = "output_bucket"
config.output_anon_subdir = "/anon-subdir"
config.output_school_subdir = "/school-subdir"
config.put()
return config
def get_dummy_config():
# Set up a dummy DeploymentConfig instance.
# Test cases will likely edit the member vars to facilitate testing;
# creating the instance in setUp ensures we always get a fresh copy.
config = DeploymentConfig.get_instance()
config.output_bucket = "bucket"
config.output_anon_subdir = "/anon-subdir"
config.output_school_subdir = "/school-subdir"
config.school_info = [("1", "school1"), ("2", "school2"), ("default", "misc")]
config.put()
return config
def post_auth_success(self):
config = DeploymentConfig.get_instance()
# Importers tab
config.importer = self.request.get("importer")
config.import_filename = self.request.get("import_filename")
# Accounts tab
config.email_domain = self.request.get("email_domain")
config.anon_fname_config = self.request.get("anon_fname_config")
config.anon_lname_config = self.request.get("anon_lname_config")
config.username_config = self.request.get("username_config")
config.anon_username_config = self.request.get("anon_username_config")
config.password_config = self.request.get("password_config")
config.suppress_first_username = self.request.get(
"suppress_first_username") == "True"
config.suppress_first_anon_username = self.request.get(
"suppress_first_anon_username") == "True"
# Schools tab
school_count = 0
for i in self.request.arguments():
if re.match(r"school_", i):
school_count += 1
# Two fields per school with the school_ prefix;
# one for school number, one for abbreviation.
school_count /= 2
config.school_info = []
for i in range(school_count):
num = self.request.get("school_num_{}".format(i))
abbrev = self.request.get("school_abbrev_{}".format(i))
if num and abbrev:
config.school_info.append((num, abbrev))
# Exporters tab
config.output_bucket = self.request.get("output_bucket")
config.output_school_subdir = self.request.get("output_school_subdir")
config.output_anon_subdir = self.request.get("output_anon_subdir")
config.dummy_teacher_username = self.request.get("dummy_teacher_username")
config.dummy_teacher_pass = self.request.get("dummy_teacher_pass")
# Advanced tab
config.estimated_student_count = int(self.request.get("est_student_count"))
config.students_per_shard = int(self.request.get("students_per_shard"))
config.unique_strings_case_sensitive = self.request.get("case_sensitive") == "True"
config.importer_batch_size = int(self.request.get("importer_batch_size"))
config.driver_batch_size = int(self.request.get("driver_batch_size"))
config.filter_json = self.request.get("filter_json")
config.put()
self.render_front(config=config, result="Changes saved")
def post_auth_success(self):
config = DeploymentConfig.get_instance()
config.key.delete()
self.redirect("/config")
def render_front(self, config=None, result=None):
if not config:
config = DeploymentConfig.get_instance()
self.render("config.html", dc=config, result=result)
