def _update_run_state(settings, grading_run, state):
"""
Updates the state for a grading run
"""
grading_run_dao = daos.GradingRunDao(settings)
grading_run.state = state
grading_run_dao.update(grading_run)
def post(self, *args, **kwargs):
assignment_id = self.get_assignment_id(**kwargs)
config_dao = daos.AssignmentConfigDao(self.settings)
config = config_dao.find_by_id(assignment_id)
if not config:
self.abort({"message": "assignment configuration not found"})
return
if not self._assert_run_valid(config):
# abort in valid-run method for conciseness
return
run_attrs = {
**self.body,
"assignment_id": assignment_id,
"started_at": get_time(),
"state": models.GradingRunState.READY,
"student_jobs_left": len(self.body.get("students_env")),
}
run = models.GradingRun(**run_attrs)
run_dao = daos.GradingRunDao(self.settings)
run.id = str(run_dao.insert(run).inserted_id)
if not continue_grading_run(self.settings, run):
self.abort({"message": "failed to start grading run"}, status=500)
return
# trigger schedule event
tornado.ioloop.IOLoop.current().add_callback(worker_schedule_job,
self.settings)
return {"grading_run_id": run.id}
def job_update_callback(settings, grading_job_id, grading_run_id):
job_dao = daos.GradingJobDao(settings)
job = job_dao.find_by_id(grading_job_id)
if job is None:
logger.critical("cannot update non-existent job with ID '{}'".format(
grading_job_id))
return
run_dao = daos.GradingRunDao(settings)
run = run_dao.find_by_id(grading_run_id)
if run is None:
logger.critical("cannot update non-existent run with ID '{}'".format(
grading_run_id))
return
if run.finished_at is not None:
logger.critical(
"cannot update run with ID '{}' (already finished)".format(
grading_run_id))
return
stream_queue = settings["STREAM_QUEUE"]
if job.success:
stream_queue.update_job_state(job.id, GradingRunState.FINISHED.name)
else:
stream_queue.update_job_state(job.id, GradingRunState.FAILED.name)
stream_queue.send_close_event(job.id)
if job.type == GradingJobType.PRE_PROCESSING:
if job.success:
continue_grading_run(settings, run)
else:
fail_grading_run(settings, run)
elif job.type == GradingJobType.POST_PROCESSING:
if run.student_jobs_left != 0:
logger.critical(
"post-processing job finished when {} student jobs remain".
format(run.student_jobs_left))
return
if job.success:
continue_grading_run(settings, run)
else:
fail_grading_run(settings, run)
elif job.type == GradingJobType.STUDENT:
if run.student_jobs_left <= 0:
logger.critical(
"student job finished when {} student jobs remain".format(
run.student_jobs_left))
return
run.student_jobs_left -= 1
run_dao.update(run)
if run.student_jobs_left == 0:
# last job in this stage is complete
continue_grading_run(settings, run)
else:
logger.critical("cannot update run with last job type '{}'".format(
job.type))
def fail_grading_run(settings, run):
run_dao = daos.GradingRunDao(settings)
if run is None:
logger.critical("cannot fail non-existent run with ID '{}'".format(
run.id))
return
run.finished_at = get_time()
run.state = GradingRunState.FAILED
run.success = False
run_dao.update(run)
def get(self, *args, **kwargs):
grading_run_id = kwargs.get("run_id")
grading_run_dao = daos.GradingRunDao(self.settings)
grading_run = grading_run_dao.find_by_id(grading_run_id)
if grading_run is None:
self.abort({"message": "grading run with the given ID not found"})
return
grading_job_dao = daos.GradingJobDao(self.settings)
grading_jobs = grading_job_dao.find_by_run_id(grading_run_id)
pre_processing_job = next(
filter(lambda j: j.type == models.GradingJobType.PRE_PROCESSING,
grading_jobs),
None,
)
post_processing_job = next(
filter(lambda j: j.type == models.GradingJobType.POST_PROCESSING,
grading_jobs),
None,
)
student_jobs = filter(
lambda j: j.type == models.GradingJobType.STUDENT, grading_jobs)
# [jobs] -> { job_id: job_state }
def get_job_id_to_state_map(jobs):
if jobs is None:
return None
else:
return {job.id: job.get_state().value for job in jobs}
return {
"state":
grading_run.state.value,
"pre_processing_job_state":
get_job_id_to_state_map(
[pre_processing_job] if pre_processing_job else None),
"post_processing_job_state":
get_job_id_to_state_map(
[post_processing_job] if post_processing_job else None),
"student_jobs_state":
get_job_id_to_state_map(student_jobs),
}
def get(self, *args, **kwargs):
grading_run_id = kwargs.get("run_id")
grading_run_dao = daos.GradingRunDao(self.settings)
grading_run = grading_run_dao.find_by_id(grading_run_id)
if grading_run is None:
self.abort({"message": "grading run with the given ID not found"})
return
grading_job_dao = daos.GradingJobDao(self.settings)
grading_jobs = grading_job_dao.find_by_run_id(grading_run_id)
# Helper method for making a dictionary from the grading_run
def get_job_id_to_env_map(jobs):
job_id_to_env_map = {}
for job in jobs:
env_dict = {}
for stage in job.stages:
env = stage["env"]
for key in env:
env_dict[key] = env_dict.get(key, set())
env_dict[key].add(env[key])
# Convert each set into a list for JSON
for key in env_dict:
env_values = list(env_dict[key])
env_dict[key] = env_values
job_id_to_env_map[job.id] = env_dict
return job_id_to_env_map
pre_processing_job = next(
filter(lambda j: j.type == models.GradingJobType.PRE_PROCESSING,
grading_jobs),
None,
)
post_processing_job = next(
filter(lambda j: j.type == models.GradingJobType.POST_PROCESSING,
grading_jobs),
None,
)
student_jobs = filter(
lambda j: j.type == models.GradingJobType.STUDENT, grading_jobs)
# Make sure pre_processing_env exists
if pre_processing_job is None:
pre_processing_dict = None
else:
pre_processing_dict = get_job_id_to_env_map([pre_processing_job])
# Make sure post_processing_env exists
if post_processing_job is None:
post_processing_dict = None
else:
post_processing_dict = get_job_id_to_env_map([post_processing_job])
# We are guaranteed that this dict exists in the run, so no need to check
student_dict = get_job_id_to_env_map(student_jobs)
return {
"pre_processing_env": pre_processing_dict,
"post_processing_env": post_processing_dict,
"student_env": student_dict,
}
def _finish_grading_run(settings, grading_run):
grading_run_dao = daos.GradingRunDao(settings)
grading_run.state = GradingRunState.FINISHED
grading_run.finished_at = get_time()
grading_run.success = True
grading_run_dao.update(grading_run)