def get_peer_grading_data_for_location(request):
if request.method != 'GET':
return util._error_response("Request type must be GET",
_INTERFACE_VERSION)
for tag in ['student_id', 'location']:
if tag not in request.GET:
return util._error_response("Missing required key {0}".format(tag),
_INTERFACE_VERSION)
location = request.GET.get('location')
student_id = request.GET.get('student_id')
pl = peer_grading_util.PeerLocation(location, student_id)
student_sub_count = pl.submitted_count()
submissions_graded = pl.graded_count()
submissions_required = pl.required_count()
submissions_available = pl.pending_count()
peer_data = {
'count_graded': submissions_graded,
'count_required': submissions_required,
'student_sub_count': student_sub_count,
'count_available': submissions_available
}
util.log_connection_data()
return util._success_response(peer_data, _INTERFACE_VERSION)
def get_problem_list(request):
"""
Get the list of problems that need grading in course request.GET['course_id'].
Returns:
list of dicts with keys
'location'
'problem_name'
'num_graded' -- number graded
'num_pending' -- number pending in the queue
'min_for_ml' -- minimum needed to make ML model
"""
if request.method!="GET":
error_message="Request needs to be GET."
log.error(error_message)
return util._error_response(error_message, _INTERFACE_VERSION)
course_id=request.GET.get("course_id")
if not course_id:
error_message="Missing needed tag course_id"
log.error(error_message)
return util._error_response(error_message, _INTERFACE_VERSION)
locations_for_course = [x['location'] for x in
list(Submission.objects.filter(course_id=course_id).values('location').distinct())]
if len(locations_for_course)==0:
error_message="No problems associated with course."
log.error(error_message)
return util._error_response(error_message, _INTERFACE_VERSION)
location_info=[]
for location in locations_for_course:
problem_name = Submission.objects.filter(location=location)[0].problem_id
submissions_pending = staff_grading_util.submissions_pending_instructor(location, state_in=[SubmissionState.waiting_to_be_graded]).count()
finished_instructor_graded = staff_grading_util.finished_submissions_graded_by_instructor(location).count()
min_scored_for_location=settings.MIN_TO_USE_PEER
location_ml_count = Submission.objects.filter(location=location, preferred_grader_type="ML").count()
if location_ml_count>0:
min_scored_for_location=settings.MIN_TO_USE_ML
submissions_required = max([0,min_scored_for_location-finished_instructor_graded])
problem_name_from_location=location.split("://")[1]
location_dict={
'location' : location,
'problem_name' : problem_name,
'problem_name_from_location' : problem_name_from_location,
'num_graded' : finished_instructor_graded,
'num_pending' : submissions_pending,
'num_required' : submissions_required,
'min_for_ml' : settings.MIN_TO_USE_ML,
}
location_info.append(location_dict)
util.log_connection_data()
return util._success_response({'problem_list' : location_info},
_INTERFACE_VERSION)
def get_peer_grading_data_for_location(request):
if request.method != 'GET':
return util._error_response("Request type must be GET", _INTERFACE_VERSION)
for tag in ['student_id', 'location']:
if tag not in request.GET:
return util._error_response("Missing required key {0}".format(tag), _INTERFACE_VERSION)
location = request.GET.get('location')
student_id = request.GET.get('student_id')
pl = peer_grading_util.PeerLocation(location,student_id)
student_sub_count = pl.submitted_count()
submissions_graded = pl.graded_count()
submissions_required = pl.required_count()
submissions_available = pl.pending_count()
peer_data = {
'count_graded' : submissions_graded,
'count_required' : submissions_required,
'student_sub_count' : student_sub_count,
'count_available' : submissions_available
}
util.log_connection_data()
return util._success_response(peer_data, _INTERFACE_VERSION)
def is_student_calibrated(request):
"""
Decides if student has fulfilled criteria for peer grading calibration for a given location (problem id).
Input:
student id, problem_id
Output:
Dictionary with boolean calibrated indicating whether or not student has finished calibration.
Note: Location in the database is currently being used as the problem id.
"""
if request.method != "GET":
raise Http404
problem_id = request.GET.get("problem_id")
student_id = request.GET.get("student_id")
success, data = calibration.check_calibration_status(
problem_id, student_id)
if not success:
return util._error_response(data, _INTERFACE_VERSION)
util.log_connection_data()
return util._success_response(data, _INTERFACE_VERSION)
def get_peer_grading_data_for_location(request):
if request.method != 'GET':
return util._error_response("Request type must be GET", _INTERFACE_VERSION)
for tag in ['student_id', 'location']:
if tag not in request.GET:
return util._error_response("Missing required key {0}".format(tag), _INTERFACE_VERSION)
location = request.GET.get('location')
student_id = request.GET.get('student_id')
student_sub_count= peer_grading_util.get_required_peer_grading_for_location({'student_id' : student_id, 'location' : location, 'preferred_grader_type' : "PE"})
submissions_graded = peer_grading_util.peer_grading_submissions_graded_for_location(location,student_id).count()
submissions_required = settings.REQUIRED_PEER_GRADING_PER_STUDENT*student_sub_count
##Check to see if submissions were available to grade in the past week
notification_seen_recently = NotificationsSeen.check_for_recent_notifications(
student_id = student_id,
location = location,
notification_type=NotificationTypes.peer_grading,
recent_notification_interval=settings.PEER_GRADING_TIMEOUT_INTERVAL
)
if not notification_seen_recently:
submissions_required = submissions_graded
peer_data = {
'count_graded' : submissions_graded,
'count_required' : submissions_required,
'student_sub_count' : student_sub_count,
}
util.log_connection_data()
return util._success_response(peer_data, _INTERFACE_VERSION)
def get_problem_list(request):
"""
Get the list of problems that need grading in course_id request.GET['course_id'].
Returns:
list of dicts with keys
'location'
'problem_name'
'num_graded' -- number graded
'num_pending' -- number pending in the queue
"""
if request.method != "GET":
error_message = "Request needs to be GET."
log.error(error_message)
return util._error_response(error_message, _INTERFACE_VERSION)
course_id = request.GET.get("course_id")
student_id = request.GET.get("student_id")
if not course_id or not student_id:
error_message = "Missing needed tag course_id or student_id"
log.error(error_message)
return util._error_response(error_message, _INTERFACE_VERSION)
locations_for_course = [
x["location"] for x in list(Submission.objects.filter(course_id=course_id).values("location").distinct())
]
location_info = []
for location in locations_for_course:
student_sub_count = peer_grading_util.get_required_peer_grading_for_location(
{"student_id": student_id, "location": location, "preferred_grader_type": "PE"}
)
if student_sub_count > 0:
problem_name = Submission.objects.filter(location=location)[0].problem_id
submissions_pending = peer_grading_util.peer_grading_submissions_pending_for_location(
location, student_id
).count()
submissions_graded = peer_grading_util.peer_grading_submissions_graded_for_location(
location, student_id
).count()
submissions_required = max(
[0, (settings.REQUIRED_PEER_GRADING_PER_STUDENT * student_sub_count) - submissions_graded]
)
problem_name_from_location = location.split("://")[1]
if submissions_graded > 0 or submissions_pending > 0:
location_dict = {
"location": location,
"problem_name": problem_name,
"num_graded": submissions_graded,
"num_required": submissions_required,
"num_pending": submissions_pending,
}
location_info.append(location_dict)
util.log_connection_data()
return util._success_response({"problem_list": location_info}, _INTERFACE_VERSION)
def get_problem_list(request):
"""
Get the list of problems that need grading in course request.GET['course_id'].
Returns:
list of dicts with keys
'location'
'problem_name'
'num_graded' -- number graded
'num_pending' -- number pending in the queue
'min_for_ml' -- minimum needed to make ML model
"""
if request.method!="GET":
error_message="Request needs to be GET."
log.error(error_message)
return util._error_response(error_message, _INTERFACE_VERSION)
course_id=request.GET.get("course_id")
if not course_id:
error_message="Missing needed tag course_id"
log.error(error_message)
return util._error_response(error_message, _INTERFACE_VERSION)
locations_for_course = [x['location'] for x in
list(Submission.objects.filter(course_id=course_id).values('location').distinct())]
if len(locations_for_course)==0:
error_message="No problems associated with course."
log.error(error_message)
return util._error_response(error_message, _INTERFACE_VERSION)
location_info=[]
for location in locations_for_course:
sl = staff_grading_util.StaffLocation(location)
control = SubmissionControl(sl.latest_submission())
problem_name = sl.problem_name()
submissions_pending = sl.pending_count()
finished_instructor_graded = sl.graded_count()
submissions_required = max([0, sl.minimum_to_score() - finished_instructor_graded])
problem_name_from_location=location.split("://")[1]
location_dict={
'location': location,
'problem_name': problem_name,
'problem_name_from_location': problem_name_from_location,
'num_graded': finished_instructor_graded,
'num_pending': submissions_pending,
'num_required': submissions_required,
'min_for_ml': control.minimum_to_use_ai,
}
location_info.append(location_dict)
util.log_connection_data()
return util._success_response({'problem_list' : location_info},
_INTERFACE_VERSION)
def get_problem_list(request):
"""
Get the list of problems that need grading in course_id request.GET['course_id'].
Returns:
list of dicts with keys
'location'
'problem_name'
'num_graded' -- number graded
'num_pending' -- number pending in the queue
"""
if request.method != "GET":
error_message = "Request needs to be GET."
log.error(error_message)
return util._error_response(error_message, _INTERFACE_VERSION)
course_id = request.GET.get("course_id")
student_id = request.GET.get("student_id")
if not course_id or not student_id:
error_message = "Missing needed tag course_id or student_id"
log.error(error_message)
return util._error_response(error_message, _INTERFACE_VERSION)
locations_for_course = [
x['location'] for x in Submission.objects.filter(
course_id=course_id).values('location').distinct()
]
location_info = []
for location in locations_for_course:
pl = peer_grading_util.PeerLocation(location, student_id)
if pl.submitted_count() > 0:
problem_name = pl.problem_name()
submissions_pending = pl.pending_count()
submissions_graded = pl.graded_count()
submissions_required = max(
[0, pl.required_count() - submissions_graded])
if (submissions_graded > 0 or submissions_pending > 0
or control_util.SubmissionControl.peer_grade_finished_subs(
pl)):
location_dict = {
'location': location,
'problem_name': problem_name,
'num_graded': submissions_graded,
'num_required': submissions_required,
'num_pending': submissions_pending,
}
location_info.append(location_dict)
util.log_connection_data()
return util._success_response({'problem_list': location_info},
_INTERFACE_VERSION)
def get_problem_list(request):
"""
Get the list of problems that need grading in course_id request.GET['course_id'].
Returns:
list of dicts with keys
'location'
'problem_name'
'num_graded' -- number graded
'num_pending' -- number pending in the queue
"""
if request.method!="GET":
error_message="Request needs to be GET."
log.error(error_message)
return util._error_response(error_message, _INTERFACE_VERSION)
course_id=request.GET.get("course_id")
student_id = request.GET.get("student_id")
if not course_id or not student_id:
error_message="Missing needed tag course_id or student_id"
log.error(error_message)
return util._error_response(error_message, _INTERFACE_VERSION)
locations_for_course = [x['location'] for x in
Submission.objects.filter(course_id=course_id).values('location').distinct()]
location_info=[]
for location in locations_for_course:
pl = peer_grading_util.PeerLocation(location,student_id)
if pl.submitted_count()>0:
problem_name = pl.problem_name()
submissions_pending = pl.pending_count()
submissions_graded = pl.graded_count()
submissions_required = max([0,pl.required_count() - submissions_graded])
if (submissions_graded > 0 or
submissions_pending > 0 or
control_util.SubmissionControl.peer_grade_finished_subs(pl)):
location_dict={
'location' : location,
'problem_name' : problem_name,
'num_graded' : submissions_graded,
'num_required' : submissions_required,
'num_pending' : submissions_pending,
}
location_info.append(location_dict)
util.log_connection_data()
return util._success_response({'problem_list' : location_info},
_INTERFACE_VERSION)
def get_notifications(request):
if request.method!="GET":
error_message="Request needs to be GET."
log.error(error_message)
return util._error_response(error_message, _INTERFACE_VERSION)
course_id=request.GET.get("course_id")
if not course_id:
error_message="Missing needed tag course_id"
log.error(error_message)
return util._error_response(error_message, _INTERFACE_VERSION)
success, staff_needs_to_grade = staff_grading_util.get_staff_grading_notifications(course_id)
if not success:
return util._error_response(staff_needs_to_grade, _INTERFACE_VERSION)
util.log_connection_data()
return util._success_response({'staff_needs_to_grade' : staff_needs_to_grade}, _INTERFACE_VERSION)
def get_peer_grading_data_for_location(request):
if request.method != 'GET':
return util._error_response("Request type must be GET",
_INTERFACE_VERSION)
for tag in ['student_id', 'location']:
if tag not in request.GET:
return util._error_response("Missing required key {0}".format(tag),
_INTERFACE_VERSION)
location = request.GET.get('location')
student_id = request.GET.get('student_id')
student_sub_count = peer_grading_util.get_required_peer_grading_for_location(
{
'student_id': student_id,
'location': location,
'preferred_grader_type': "PE"
})
submissions_graded = peer_grading_util.peer_grading_submissions_graded_for_location(
location, student_id).count()
submissions_required = settings.REQUIRED_PEER_GRADING_PER_STUDENT * student_sub_count
##Check to see if submissions were available to grade in the past week
notification_seen_recently = NotificationsSeen.check_for_recent_notifications(
student_id=student_id,
location=location,
notification_type=NotificationTypes.peer_grading,
recent_notification_interval=settings.PEER_GRADING_TIMEOUT_INTERVAL)
if not notification_seen_recently:
submissions_required = submissions_graded
peer_data = {
'count_graded': submissions_graded,
'count_required': submissions_required,
'student_sub_count': student_sub_count,
}
util.log_connection_data()
return util._success_response(peer_data, _INTERFACE_VERSION)
def get_notifications(request):
if request.method!="GET":
error_message="Request needs to be GET."
log.error(error_message)
return util._error_response(error_message, _INTERFACE_VERSION)
course_id=request.GET.get("course_id")
student_id = request.GET.get("student_id")
if not course_id or not student_id:
error_message="Missing needed tag course_id or student_id"
log.error(error_message)
return util._error_response(error_message, _INTERFACE_VERSION)
pc = peer_grading_util.PeerCourse(course_id,student_id)
success, student_needs_to_peer_grade = pc.notifications()
if not success:
return util._error_response(student_needs_to_peer_grade, _INTERFACE_VERSION)
util.log_connection_data()
return util._success_response({'student_needs_to_peer_grade' : student_needs_to_peer_grade}, _INTERFACE_VERSION)
def get_notifications(request):
if request.method != "GET":
error_message = "Request needs to be GET."
log.error(error_message)
return util._error_response(error_message, _INTERFACE_VERSION)
course_id = request.GET.get("course_id")
if not course_id:
error_message = "Missing needed tag course_id"
log.error(error_message)
return util._error_response(error_message, _INTERFACE_VERSION)
success, staff_needs_to_grade = staff_grading_util.get_staff_grading_notifications(
course_id)
if not success:
return util._error_response(staff_needs_to_grade, _INTERFACE_VERSION)
util.log_connection_data()
return util._success_response(
{'staff_needs_to_grade': staff_needs_to_grade}, _INTERFACE_VERSION)
def show_calibration_essay(request):
"""
Shows a calibration essay when it receives a GET request.
Input:
Http request containing problem_id and student_id
Output:
Http response containing essay data (submission id, submission key, student response, prompt, rubric, max_score)
Or error
"""
if request.method != "GET":
raise Http404
problem_id = request.GET.get("problem_id")
student_id = request.GET.get("student_id")
success, data = calibration.get_calibration_essay(problem_id, student_id)
if not success:
return util._error_response(data, _INTERFACE_VERSION)
util.log_connection_data()
return util._success_response(data, _INTERFACE_VERSION)
def show_calibration_essay(request):
"""
Shows a calibration essay when it receives a GET request.
Input:
Http request containing problem_id and student_id
Output:
Http response containing essay data (submission id, submission key, student response, prompt, rubric, max_score)
Or error
"""
if request.method != "GET":
raise Http404
problem_id = request.GET.get("problem_id")
student_id = request.GET.get("student_id")
success, data = calibration.get_calibration_essay(problem_id, student_id)
if not success:
return util._error_response(data, _INTERFACE_VERSION)
util.log_connection_data()
return util._success_response(data, _INTERFACE_VERSION)
def is_student_calibrated(request):
"""
Decides if student has fulfilled criteria for peer grading calibration for a given location (problem id).
Input:
student id, problem_id
Output:
Dictionary with boolean calibrated indicating whether or not student has finished calibration.
Note: Location in the database is currently being used as the problem id.
"""
if request.method != "GET":
raise Http404
problem_id = request.GET.get("problem_id")
student_id = request.GET.get("student_id")
success, data = calibration.check_calibration_status(problem_id, student_id)
if not success:
return util._error_response(data, _INTERFACE_VERSION)
util.log_connection_data()
return util._success_response(data, _INTERFACE_VERSION)
def get_notifications(request):
if request.method != "GET":
error_message = "Request needs to be GET."
log.error(error_message)
return util._error_response(error_message, _INTERFACE_VERSION)
course_id = request.GET.get("course_id")
student_id = request.GET.get("student_id")
if not course_id or not student_id:
error_message = "Missing needed tag course_id or student_id"
log.error(error_message)
return util._error_response(error_message, _INTERFACE_VERSION)
pc = peer_grading_util.PeerCourse(course_id, student_id)
success, student_needs_to_peer_grade = pc.notifications()
if not success:
return util._error_response(student_needs_to_peer_grade,
_INTERFACE_VERSION)
util.log_connection_data()
return util._success_response(
{'student_needs_to_peer_grade': student_needs_to_peer_grade},
_INTERFACE_VERSION)
def get_problem_list(request):
"""
Get the list of problems that need grading in course_id request.GET['course_id'].
Returns:
list of dicts with keys
'location'
'problem_name'
'num_graded' -- number graded
'num_pending' -- number pending in the queue
"""
if request.method != "GET":
error_message = "Request needs to be GET."
log.error(error_message)
return util._error_response(error_message, _INTERFACE_VERSION)
course_id = request.GET.get("course_id")
student_id = request.GET.get("student_id")
if not course_id or not student_id:
error_message = "Missing needed tag course_id or student_id"
log.error(error_message)
return util._error_response(error_message, _INTERFACE_VERSION)
locations_for_course = [
x['location'] for x in list(
Submission.objects.filter(
course_id=course_id).values('location').distinct())
]
location_info = []
for location in locations_for_course:
student_sub_count = peer_grading_util.get_required_peer_grading_for_location(
{
'student_id': student_id,
'location': location,
'preferred_grader_type': "PE"
})
if student_sub_count > 0:
problem_name = Submission.objects.filter(
location=location)[0].problem_id
submissions_pending = peer_grading_util.peer_grading_submissions_pending_for_location(
location, student_id).count()
submissions_graded = peer_grading_util.peer_grading_submissions_graded_for_location(
location, student_id).count()
submissions_required = max([
0,
(settings.REQUIRED_PEER_GRADING_PER_STUDENT *
student_sub_count) - submissions_graded
])
problem_name_from_location = location.split("://")[1]
if submissions_graded > 0 or submissions_pending > 0:
location_dict = {
'location': location,
'problem_name': problem_name,
'num_graded': submissions_graded,
'num_required': submissions_required,
'num_pending': submissions_pending,
}
location_info.append(location_dict)
util.log_connection_data()
return util._success_response({'problem_list': location_info},
_INTERFACE_VERSION)
def save_grade(request):
"""
Supports POST requests with the following arguments:
location: string
grader_id: int
submission_id: int
score: int
feedback: string
submission_key : string
Returns json dict with keys
version: int
success: bool
error: string, present if not success
"""
if request.method != "POST":
raise Http404
post_data = request.POST.dict().copy()
log.debug(post_data)
for tag in ["location", "grader_id", "submission_id", "submission_key", "score", "feedback", "submission_flagged"]:
if not tag in post_data:
return util._error_response("Cannot find needed key {0} in request.".format(tag), _INTERFACE_VERSION)
location = post_data["location"]
grader_id = post_data["grader_id"]
submission_id = post_data["submission_id"]
# Submission key currently unused, but plan to use it for validation in the future.
submission_key = post_data["submission_key"]
score = post_data["score"]
# This is done to ensure that response is properly formatted on the lms side.
feedback_dict = post_data["feedback"]
rubric_scores_complete = request.POST.get("rubric_scores_complete", False)
rubric_scores = request.POST.getlist("rubric_scores", [])
is_submission_flagged = request.POST.get("submission_flagged", False)
if isinstance(is_submission_flagged, basestring):
is_submission_flagged = is_submission_flagged.lower() == "true"
status = GraderStatus.success
confidence = 1.0
is_answer_unknown = request.POST.get("answer_unknown", False)
if isinstance(is_answer_unknown, basestring):
is_answer_unknown = is_answer_unknown.lower() == "true"
if is_answer_unknown:
status = GraderStatus.failure
confidence = 0.0
try:
score = int(score)
except ValueError:
return util._error_response("Expected integer score. Got {0}".format(score), _INTERFACE_VERSION)
try:
sub = Submission.objects.get(id=submission_id)
except:
return util._error_response(
"grade_save_error",
_INTERFACE_VERSION,
data={"msg": "Submission id {0} is not valid.".format(submission_id)},
)
success, error_message = grader_util.validate_rubric_scores(rubric_scores, rubric_scores_complete, sub)
if not success:
return util._error_response("grade_save_error", _INTERFACE_VERSION, data={"msg": error_message})
d = {
"submission_id": submission_id,
"score": score,
"feedback": feedback_dict,
"grader_id": grader_id,
"grader_type": "PE",
# Humans always succeed (if they grade at all)...
"status": status,
# ...and they're always confident too.
"confidence": confidence,
# And they don't make any errors
"errors": "",
"rubric_scores_complete": rubric_scores_complete,
"rubric_scores": rubric_scores,
"is_submission_flagged": is_submission_flagged,
}
# Currently not posting back to LMS. Only saving grader object, and letting controller decide when to post back.
(success, header) = grader_util.create_and_handle_grader_object(d)
if not success:
return util._error_response("There was a problem saving the grade. Contact support.", _INTERFACE_VERSION)
# xqueue_session=util.xqueue_login()
# error,msg = util.post_results_to_xqueue(xqueue_session,json.dumps(header),json.dumps(post_data))
util.log_connection_data()
return util._success_response({"msg": "Posted to queue."}, _INTERFACE_VERSION)
def get_next_submission(request):
"""
Supports GET request with the following arguments:
course_id -- the course for which to return a submission.
grader_id -- LMS user_id of the requesting user
Returns json dict with the following keys:
version: '1' (number)
success: bool
if success:
'submission_id': a unique identifier for the submission, to be passed
back with the grade.
'submission': the submission, rendered as read-only html for grading
'rubric': the rubric, also rendered as html.
'prompt': the question prompt, also rendered as html.
'message': if there was no submission available, but nothing went wrong,
there will be a message field.
else:
'error': if success is False, will have an error message with more info.
}
"""
if request.method != "GET":
raise Http404
course_id = request.GET.get('course_id')
grader_id = request.GET.get('grader_id')
location = request.GET.get('location')
found = False
if not (course_id or location) or not grader_id:
return util._error_response("required_parameter_missing", _INTERFACE_VERSION)
if location:
sl = staff_grading_util.StaffLocation(location)
(found, sid) = sl.next_item()
# TODO: save the grader id and match it in save_grade to make sure things
# are consistent.
if not location:
sc = staff_grading_util.StaffCourse(course_id)
(found, sid) = sc.next_item()
if not found:
return util._success_response({'message': 'No more submissions to grade.'},
_INTERFACE_VERSION)
try:
submission = Submission.objects.get(id=int(sid))
except Submission.DoesNotExist:
log.error("Couldn't find submission %s for instructor grading", sid)
return util._error_response('failed_to_load_submission',
_INTERFACE_VERSION,
data={'submission_id': sid})
#Get error metrics from ml grading, and get into dictionary form to pass down to staff grading view
success, ml_error_info=ml_grading_util.get_ml_errors(submission.location)
if success:
ml_error_message=staff_grading_util.generate_ml_error_message(ml_error_info)
else:
ml_error_message=ml_error_info
ml_error_message="Machine learning error information: " + ml_error_message
sl = staff_grading_util.StaffLocation(submission.location)
control = SubmissionControl(sl.latest_submission())
if submission.state != SubmissionState.being_graded:
log.error("Instructor grading got submission {0} in an invalid state {1} ".format(sid, submission.state))
return util._error_response('wrong_internal_state',
_INTERFACE_VERSION,
data={'submission_id': sid,
'submission_state': submission.state})
response = {'submission_id': sid,
'submission': submission.student_response,
'rubric': submission.rubric,
'prompt': submission.prompt,
'max_score': submission.max_score,
'ml_error_info': ml_error_message,
'problem_name': submission.problem_id,
'num_graded': sl.graded_count(),
'num_pending': sl.pending_count(),
'min_for_ml': control.minimum_to_use_ai,
}
util.log_connection_data()
return util._success_response(response, _INTERFACE_VERSION)
def handle_single_item(controller_session):
sub_get_success, content = get_item_from_controller(controller_session)
#Grade and handle here
if sub_get_success:
transaction.commit()
sub = Submission.objects.get(id=int(content['submission_id']))
sl = staff_grading_util.StaffLocation(sub.location)
subs_graded_by_instructor = sl.graded()
first_sub = subs_graded_by_instructor.order_by('date_created')[0]
parsed_rubric=rubric_functions.parse_rubric(first_sub.rubric)
#strip out unicode and other characters in student response
#Needed, or grader may potentially fail
#TODO: Handle unicode in student responses properly
student_response = sub.student_response.encode('ascii', 'ignore')
#Get the latest created model for the given location
transaction.commit()
location_suffixes=ml_grading_util.generate_rubric_location_suffixes(subs_graded_by_instructor, grading = True)
if len(location_suffixes)>0:
rubric_scores_complete=True
rubric_scores=[]
for m in xrange(0,len(location_suffixes)):
suffix = location_suffixes[m]
success, created_model=ml_grading_util.get_latest_created_model(sub.location + suffix)
if not success:
log.error("Could not identify a valid created model!")
if m==0:
results= RESULT_FAILURE_DICT
formatted_feedback="error"
status=GraderStatus.failure
statsd.increment("open_ended_assessment.grading_controller.call_ml_grader",
tags=["success:False"])
else:
#Create grader path from location in submission
grader_path = os.path.join(settings.ML_MODEL_PATH,created_model.model_relative_path)
model_stored_in_s3=created_model.model_stored_in_s3
success, grader_data=load_model_file(created_model,use_full_path=False)
if success:
results = grade.grade(grader_data, student_response)
else:
results=RESULT_FAILURE_DICT
#If the above fails, try using the full path in the created_model object
if not results['success'] and not created_model.model_stored_in_s3:
grader_path=created_model.model_full_path
try:
success, grader_data=load_model_file(created_model,use_full_path=True)
if success:
results = grade.grade(grader_data, student_response)
else:
results=RESULT_FAILURE_DICT
except Exception:
error_message="Could not find a valid model file."
log.exception(error_message)
results=RESULT_FAILURE_DICT
log.info("ML Grader: Success: {0} Errors: {1}".format(results['success'], results['errors']))
statsd.increment("open_ended_assessment.grading_controller.call_ml_grader",
tags=["success:{0}".format(results['success']), 'location:{0}'.format(sub.location)])
#Set grader status according to success/fail
if results['success']:
status = GraderStatus.success
else:
status = GraderStatus.failure
if m==0:
final_results=results
elif results['success']==False:
rubric_scores_complete = False
else:
rubric_scores.append(int(results['score']))
if len(rubric_scores)==0:
rubric_scores_complete=False
grader_dict = {
'score': int(final_results['score']),
'feedback': json.dumps(results['feedback']),
'status': status,
'grader_id': 1,
'grader_type': "ML",
'confidence': results['confidence'],
'submission_id': sub.id,
'errors' : ' ' .join(results['errors']),
'rubric_scores_complete' : rubric_scores_complete,
'rubric_scores' : json.dumps(rubric_scores),
}
#Create grader object in controller by posting back results
created, msg = util._http_post(
controller_session,
urlparse.urljoin(settings.GRADING_CONTROLLER_INTERFACE['url'],
project_urls.ControllerURLs.put_result),
grader_dict,
settings.REQUESTS_TIMEOUT,
)
else:
log.error("Error getting item from controller or no items to get.")
statsd.increment("open_ended_assessment.grading_controller.call_ml_grader",
tags=["success:False"])
util.log_connection_data()
return sub_get_success
def save_grade(request):
"""
Supports POST requests with the following arguments:
course_id: int
grader_id: int
submission_id: int
score: int
feedback: string
Returns json dict with keys
version: int
success: bool
error: string, present if not success
"""
if request.method != "POST":
return util._error_response("Request needs to be GET", _INTERFACE_VERSION)
course_id = request.POST.get('course_id')
grader_id = request.POST.get('grader_id')
submission_id = request.POST.get('submission_id')
score = request.POST.get('score')
feedback = request.POST.get('feedback')
skipped = request.POST.get('skipped')=="True"
rubric_scores_complete = request.POST.get('rubric_scores_complete', False)
rubric_scores = request.POST.getlist('rubric_scores', [])
is_submission_flagged = request.POST.get('submission_flagged', False)
if isinstance(is_submission_flagged, basestring):
is_submission_flagged = is_submission_flagged.lower() == 'true'
if (# These have to be truthy
not (course_id and grader_id and submission_id) or
# These have to be non-None
score is None or feedback is None):
return util._error_response("required_parameter_missing", _INTERFACE_VERSION)
if skipped:
success, sub = staff_grading_util.set_instructor_grading_item_back_to_preferred_grader(submission_id)
if not success:
return util._error_response(sub, _INTERFACE_VERSION)
return util._success_response({}, _INTERFACE_VERSION)
try:
score = int(score)
except ValueError:
return util._error_response(
"grade_save_error",
_INTERFACE_VERSION,
data={"msg": "Expected integer score. Got {0}".format(score)})
try:
sub=Submission.objects.get(id=submission_id)
except Exception:
return util._error_response(
"grade_save_error",
_INTERFACE_VERSION,
data={"msg": "Submission id {0} is not valid.".format(submission_id)}
)
first_sub_for_location=Submission.objects.filter(location=sub.location).order_by('date_created')[0]
rubric= first_sub_for_location.rubric
rubric_success, parsed_rubric = rubric_functions.parse_rubric(rubric)
if rubric_success:
success, error_message = grader_util.validate_rubric_scores(rubric_scores, rubric_scores_complete, sub)
if not success:
return util._error_response(
"grade_save_error",
_INTERFACE_VERSION,
data={"msg": error_message}
)
d = {'submission_id': submission_id,
'score': score,
'feedback': feedback,
'grader_id': grader_id,
'grader_type': 'IN',
# Humans always succeed (if they grade at all)...
'status': GraderStatus.success,
# ...and they're always confident too.
'confidence': 1.0,
#And they don't make errors
'errors' : "",
'rubric_scores_complete' : rubric_scores_complete,
'rubric_scores' : rubric_scores,
'is_submission_flagged' : is_submission_flagged,
}
success, header = grader_util.create_and_handle_grader_object(d)
if not success:
return util._error_response("grade_save_error", _INTERFACE_VERSION,
data={'msg': 'Internal error'})
util.log_connection_data()
return util._success_response({}, _INTERFACE_VERSION)
def save_calibration_essay(request):
"""
Saves a calibration essay sent back from LMS.
Input:
request dict containing keys student_id, location, calibration_essay_id, score, submission_key, feedback
Output:
Boolean indicating success in saving calibration essay or not.
"""
if request.method != "POST":
raise Http404
post_data = request.POST.dict().copy()
for tag in ['location', 'student_id', 'calibration_essay_id', 'submission_key', 'score', 'feedback']:
if not tag in post_data:
return util._error_response("Cannot find needed key {0} in request.".format(tag), _INTERFACE_VERSION)
location = post_data['location']
student_id = post_data['student_id']
submission_id = post_data['calibration_essay_id']
score = post_data['score']
feedback = post_data['feedback']
#Submission key currently unused, but plan to use it for validation in the future.
submission_key = post_data['submission_key']
rubric_scores_complete = request.POST.get('rubric_scores_complete', False)
rubric_scores = request.POST.getlist('rubric_scores', [])
try:
score = int(score)
except ValueError:
return util._error_response("Expected integer score. Got {0}".format(score), _INTERFACE_VERSION)
try:
sub=Submission.objects.get(id=submission_id)
except Exception:
return util._error_response(
"grade_save_error",
_INTERFACE_VERSION,
data={"msg": "Submission id {0} is not valid.".format(submission_id)}
)
d = {'submission_id': submission_id,
'score': score,
'feedback': feedback,
'student_id': student_id,
'location': location,
'rubric_scores_complete' : rubric_scores_complete,
'rubric_scores' : rubric_scores,
}
(success, data) = calibration.create_and_save_calibration_record(d)
if not success:
error_msg = "Failed to create and save calibration record. {0}".format(data)
log.error(error_msg)
return util._error_response(error_msg, _INTERFACE_VERSION)
util.log_connection_data()
return util._success_response({'message' : "Successfully saved calibration record.", 'actual_score' : data['actual_score'], 'actual_rubric' : data['actual_rubric'], 'actual_feedback' : data['actual_feedback']}, _INTERFACE_VERSION)
def save_calibration_essay(request):
"""
Saves a calibration essay sent back from LMS.
Input:
request dict containing keys student_id, location, calibration_essay_id, score, submission_key, feedback
Output:
Boolean indicating success in saving calibration essay or not.
"""
if request.method != "POST":
raise Http404
post_data = request.POST.dict().copy()
for tag in [
'location', 'student_id', 'calibration_essay_id', 'submission_key',
'score', 'feedback'
]:
if not tag in post_data:
return util._error_response(
"Cannot find needed key {0} in request.".format(tag),
_INTERFACE_VERSION)
location = post_data['location']
student_id = post_data['student_id']
submission_id = post_data['calibration_essay_id']
score = post_data['score']
feedback = post_data['feedback']
#Submission key currently unused, but plan to use it for validation in the future.
submission_key = post_data['submission_key']
rubric_scores_complete = request.POST.get('rubric_scores_complete', False)
rubric_scores = request.POST.getlist('rubric_scores', [])
try:
score = int(score)
except ValueError:
return util._error_response(
"Expected integer score. Got {0}".format(score),
_INTERFACE_VERSION)
try:
sub = Submission.objects.get(id=submission_id)
except:
return util._error_response(
"grade_save_error",
_INTERFACE_VERSION,
data={
"msg": "Submission id {0} is not valid.".format(submission_id)
})
d = {
'submission_id': submission_id,
'score': score,
'feedback': feedback,
'student_id': student_id,
'location': location,
'rubric_scores_complete': rubric_scores_complete,
'rubric_scores': rubric_scores,
}
(success, data) = calibration.create_and_save_calibration_record(d)
if not success:
error_msg = "Failed to create and save calibration record. {0}".format(
data)
log.error(error_msg)
return util._error_response(error_msg, _INTERFACE_VERSION)
util.log_connection_data()
return util._success_response(
{
'message': "Successfully saved calibration record.",
'actual_score': data['actual_score'],
'actual_rubric': data['actual_rubric'],
'actual_feedback': data['actual_feedback']
}, _INTERFACE_VERSION)
def get_problem_list(request):
"""
Get the list of problems that need grading in course request.GET['course_id'].
Returns:
list of dicts with keys
'location'
'problem_name'
'num_graded' -- number graded
'num_pending' -- number pending in the queue
'min_for_ml' -- minimum needed to make ML model
"""
if request.method != "GET":
error_message = "Request needs to be GET."
log.error(error_message)
return util._error_response(error_message, _INTERFACE_VERSION)
course_id = request.GET.get("course_id")
if not course_id:
error_message = "Missing needed tag course_id"
log.error(error_message)
return util._error_response(error_message, _INTERFACE_VERSION)
locations_for_course = [
x['location'] for x in list(
Submission.objects.filter(
course_id=course_id).values('location').distinct())
]
if len(locations_for_course) == 0:
error_message = "No problems associated with course."
log.error(error_message)
return util._error_response(error_message, _INTERFACE_VERSION)
location_info = []
for location in locations_for_course:
problem_name = Submission.objects.filter(
location=location)[0].problem_id
submissions_pending = staff_grading_util.submissions_pending_instructor(
location, state_in=[SubmissionState.waiting_to_be_graded]).count()
finished_instructor_graded = staff_grading_util.finished_submissions_graded_by_instructor(
location).count()
min_scored_for_location = settings.MIN_TO_USE_PEER
location_ml_count = Submission.objects.filter(
location=location, preferred_grader_type="ML").count()
if location_ml_count > 0:
min_scored_for_location = settings.MIN_TO_USE_ML
submissions_required = max(
[0, min_scored_for_location - finished_instructor_graded])
problem_name_from_location = location.split("://")[1]
location_dict = {
'location': location,
'problem_name': problem_name,
'problem_name_from_location': problem_name_from_location,
'num_graded': finished_instructor_graded,
'num_pending': submissions_pending,
'num_required': submissions_required,
'min_for_ml': settings.MIN_TO_USE_ML,
}
location_info.append(location_dict)
util.log_connection_data()
log.debug(location_info)
return util._success_response({'problem_list': location_info},
_INTERFACE_VERSION)
def save_grade(request):
"""
Supports POST requests with the following arguments:
course_id: int
grader_id: int
submission_id: int
score: int
feedback: string
Returns json dict with keys
version: int
success: bool
error: string, present if not success
"""
if request.method != "POST":
return util._error_response("Request needs to be GET",
_INTERFACE_VERSION)
course_id = request.POST.get('course_id')
grader_id = request.POST.get('grader_id')
submission_id = request.POST.get('submission_id')
score = request.POST.get('score')
feedback = request.POST.get('feedback')
skipped = request.POST.get('skipped') == "True"
rubric_scores_complete = request.POST.get('rubric_scores_complete', False)
rubric_scores = request.POST.getlist('rubric_scores', [])
is_submission_flagged = request.POST.get('submission_flagged', False)
if isinstance(is_submission_flagged, basestring):
is_submission_flagged = is_submission_flagged.lower() == 'true'
if ( # These have to be truthy
not (course_id and grader_id and submission_id) or
# These have to be non-None
score is None or feedback is None):
return util._error_response("required_parameter_missing",
_INTERFACE_VERSION)
if skipped:
log.debug(submission_id)
success, sub = staff_grading_util.set_instructor_grading_item_back_to_ml(
submission_id)
if not success:
return util._error_response(sub, _INTERFACE_VERSION)
return util._success_response({}, _INTERFACE_VERSION)
try:
score = int(score)
except ValueError:
return util._error_response(
"grade_save_error",
_INTERFACE_VERSION,
data={"msg": "Expected integer score. Got {0}".format(score)})
try:
sub = Submission.objects.get(id=submission_id)
except:
return util._error_response(
"grade_save_error",
_INTERFACE_VERSION,
data={
"msg": "Submission id {0} is not valid.".format(submission_id)
})
first_sub_for_location = Submission.objects.filter(
location=sub.location).order_by('date_created')[0]
rubric = first_sub_for_location.rubric
rubric_success, parsed_rubric = rubric_functions.parse_rubric(rubric)
if rubric_success:
success, error_message = grader_util.validate_rubric_scores(
rubric_scores, rubric_scores_complete, sub)
if not success:
return util._error_response("grade_save_error",
_INTERFACE_VERSION,
data={"msg": error_message})
d = {
'submission_id': submission_id,
'score': score,
'feedback': feedback,
'grader_id': grader_id,
'grader_type': 'IN',
# Humans always succeed (if they grade at all)...
'status': GraderStatus.success,
# ...and they're always confident too.
'confidence': 1.0,
#And they don't make errors
'errors': "",
'rubric_scores_complete': rubric_scores_complete,
'rubric_scores': rubric_scores,
'is_submission_flagged': is_submission_flagged,
}
success, header = grader_util.create_and_handle_grader_object(d)
if not success:
return util._error_response("grade_save_error",
_INTERFACE_VERSION,
data={'msg': 'Internal error'})
util.log_connection_data()
return util._success_response({}, _INTERFACE_VERSION)
def save_grade(request):
"""
Supports POST requests with the following arguments:
location: string
grader_id: int
submission_id: int
score: int
feedback: string
submission_key : string
Returns json dict with keys
version: int
success: bool
error: string, present if not success
"""
if request.method != "POST":
raise Http404
post_data = request.POST.dict().copy()
log.debug(post_data)
for tag in [
'location', 'grader_id', 'submission_id', 'submission_key',
'score', 'feedback', 'submission_flagged'
]:
if not tag in post_data:
return util._error_response(
"Cannot find needed key {0} in request.".format(tag),
_INTERFACE_VERSION)
location = post_data['location']
grader_id = post_data['grader_id']
submission_id = post_data['submission_id']
#Submission key currently unused, but plan to use it for validation in the future.
submission_key = post_data['submission_key']
score = post_data['score']
#This is done to ensure that response is properly formatted on the lms side.
feedback_dict = post_data['feedback']
rubric_scores_complete = request.POST.get('rubric_scores_complete', False)
rubric_scores = request.POST.getlist('rubric_scores', [])
is_submission_flagged = request.POST.get('submission_flagged', False)
if isinstance(is_submission_flagged, basestring):
is_submission_flagged = (is_submission_flagged.lower() == "true")
status = GraderStatus.success
confidence = 1.0
is_answer_unknown = request.POST.get('answer_unknown', False)
if isinstance(is_answer_unknown, basestring):
is_answer_unknown = (is_answer_unknown.lower() == "true")
if is_answer_unknown:
status = GraderStatus.failure
confidence = 0.0
try:
score = int(score)
except ValueError:
return util._error_response(
"Expected integer score. Got {0}".format(score),
_INTERFACE_VERSION)
try:
sub = Submission.objects.get(id=submission_id)
except:
return util._error_response(
"grade_save_error",
_INTERFACE_VERSION,
data={
"msg": "Submission id {0} is not valid.".format(submission_id)
})
success, error_message = grader_util.validate_rubric_scores(
rubric_scores, rubric_scores_complete, sub)
if not success:
return util._error_response("grade_save_error",
_INTERFACE_VERSION,
data={"msg": error_message})
d = {
'submission_id': submission_id,
'score': score,
'feedback': feedback_dict,
'grader_id': grader_id,
'grader_type': 'PE',
# Humans always succeed (if they grade at all)...
'status': status,
# ...and they're always confident too.
'confidence': confidence,
#And they don't make any errors
'errors': "",
'rubric_scores_complete': rubric_scores_complete,
'rubric_scores': rubric_scores,
'is_submission_flagged': is_submission_flagged,
}
#Currently not posting back to LMS. Only saving grader object, and letting controller decide when to post back.
(success, header) = grader_util.create_and_handle_grader_object(d)
if not success:
return util._error_response(
"There was a problem saving the grade. Contact support.",
_INTERFACE_VERSION)
#xqueue_session=util.xqueue_login()
#error,msg = util.post_results_to_xqueue(xqueue_session,json.dumps(header),json.dumps(post_data))
util.log_connection_data()
return util._success_response({'msg': "Posted to queue."},
_INTERFACE_VERSION)
def save_grade(request):
"""
Supports POST requests with the following arguments:
location: string
grader_id: int
submission_id: int
score: int
feedback: string
submission_key : string
Returns json dict with keys
version: int
success: bool
error: string, present if not success
"""
if request.method != "POST":
raise Http404
post_data = request.POST.dict().copy()
for tag in ['location', 'grader_id', 'submission_id', 'submission_key', 'score', 'feedback', 'submission_flagged']:
if not tag in post_data:
return util._error_response("Cannot find needed key {0} in request.".format(tag), _INTERFACE_VERSION)
location = post_data['location']
grader_id = post_data['grader_id']
submission_id = post_data['submission_id']
#Submission key currently unused, but plan to use it for validation in the future.
submission_key = post_data['submission_key']
score = post_data['score']
#This is done to ensure that response is properly formatted on the lms side.
feedback_dict = post_data['feedback']
rubric_scores_complete = request.POST.get('rubric_scores_complete', False)
rubric_scores = request.POST.getlist('rubric_scores', [])
is_submission_flagged = request.POST.get('submission_flagged', False)
if isinstance(is_submission_flagged, basestring):
is_submission_flagged = (is_submission_flagged.lower()=="true")
status = GraderStatus.success
confidence = 1.0
is_answer_unknown = request.POST.get('answer_unknown', False)
if isinstance(is_answer_unknown, basestring):
is_answer_unknown = (is_answer_unknown.lower()=="true")
if is_answer_unknown:
status = GraderStatus.failure
confidence = 0.0
try:
score = int(score)
except ValueError:
#Score may not be filled out if answer_unknown or flagged
if is_answer_unknown or is_submission_flagged:
score = 0
else:
return util._error_response("Expected integer score. Got {0}".format(score), _INTERFACE_VERSION)
try:
sub=Submission.objects.get(id=submission_id)
except Exception:
return util._error_response(
"grade_save_error",
_INTERFACE_VERSION,
data={"msg": "Submission id {0} is not valid.".format(submission_id)}
)
#Patch to handle rubric scores in the case of "I don't know" or flagging if scores aren't filled out
if is_answer_unknown or is_submission_flagged and len(rubric_scores)==0:
success, targets=rubric_functions.generate_targets_from_rubric(sub.rubric)
rubric_scores = [0 for l in targets]
rubric_scores_complete = True
success, error_message = grader_util.validate_rubric_scores(rubric_scores, rubric_scores_complete, sub)
if not success:
return util._error_response(
"grade_save_error",
_INTERFACE_VERSION,
data={"msg": error_message}
)
d = {'submission_id': submission_id,
'score': score,
'feedback': feedback_dict,
'grader_id': grader_id,
'grader_type': 'PE',
# Humans always succeed (if they grade at all)...
'status': status,
# ...and they're always confident too.
'confidence': confidence,
#And they don't make any errors
'errors' : "",
'rubric_scores_complete' : rubric_scores_complete,
'rubric_scores' : rubric_scores,
'is_submission_flagged' : is_submission_flagged,
}
#Currently not posting back to LMS. Only saving grader object, and letting controller decide when to post back.
(success, header) = grader_util.create_and_handle_grader_object(d)
if not success:
return util._error_response("There was a problem saving the grade. Contact support.", _INTERFACE_VERSION)
#xqueue_session=util.xqueue_login()
#error,msg = util.post_results_to_xqueue(xqueue_session,json.dumps(header),json.dumps(post_data))
util.log_connection_data()
return util._success_response({'msg': "Posted to queue."}, _INTERFACE_VERSION)
def handle_single_location(location):
try:
transaction.commit()
gc.collect()
sl = staff_grading_util.StaffLocation(location)
subs_graded_by_instructor = sl.graded()
log.info("Checking location {0} to see if essay count {1} greater than min {2}".format(
location,
subs_graded_by_instructor.count(),
settings.MIN_TO_USE_ML,
))
graded_sub_count=subs_graded_by_instructor.count()
#check to see if there are enough instructor graded essays for location
if graded_sub_count >= settings.MIN_TO_USE_ML:
location_suffixes=ml_grading_util.generate_rubric_location_suffixes(subs_graded_by_instructor, grading=False)
if settings.MAX_TO_USE_ML<graded_sub_count:
graded_sub_count = settings.MAX_TO_USE_ML
subs_graded_by_instructor = subs_graded_by_instructor[:settings.MAX_TO_USE_ML]
sub_rubric_scores=[]
if len(location_suffixes)>0:
for sub in subs_graded_by_instructor:
success, scores = controller.rubric_functions.get_submission_rubric_instructor_scores(sub)
sub_rubric_scores.append(scores)
for m in xrange(0,len(location_suffixes)):
log.info("Currently on location {0}. Greater than zero is a rubric item.".format(m))
suffix=location_suffixes[m]
#Get paths to ml model from database
relative_model_path, full_model_path= ml_grading_util.get_model_path(location + suffix)
#Get last created model for given location
transaction.commit()
success, latest_created_model=ml_grading_util.get_latest_created_model(location + suffix)
if success:
sub_count_diff=graded_sub_count-latest_created_model.number_of_essays
else:
sub_count_diff = graded_sub_count
#Retrain if no model exists, or every 5 graded essays.
if not success or sub_count_diff>=5:
text = [str(i.student_response.encode('ascii', 'ignore')) for i in subs_graded_by_instructor]
ids=[i.id for i in subs_graded_by_instructor]
#TODO: Make queries more efficient
#This is for the basic overall score
if m==0:
scores = [z.get_last_grader().score for z in list(subs_graded_by_instructor)]
else:
scores=[z[m-1] for z in sub_rubric_scores]
#Get the first graded submission, so that we can extract metadata like rubric, etc, from it
first_sub=subs_graded_by_instructor[0]
prompt = str(first_sub.prompt.encode('ascii', 'ignore'))
rubric = str(first_sub.rubric.encode('ascii', 'ignore'))
transaction.commit()
#Checks to see if another model creator process has started amodel for this location
success, model_started, created_model = ml_grading_util.check_if_model_started(location + suffix)
#Checks to see if model was started a long time ago, and removes and retries if it was.
if model_started:
now = timezone.now()
second_difference = (now - created_model.date_modified).total_seconds()
if second_difference > settings.TIME_BEFORE_REMOVING_STARTED_MODEL:
log.error("Model for location {0} started over {1} seconds ago, removing and re-attempting.".format(
location + suffix, settings.TIME_BEFORE_REMOVING_STARTED_MODEL))
created_model.delete()
model_started = False
if not model_started:
created_model_dict_initial={
'max_score' : first_sub.max_score,
'prompt' : prompt,
'rubric' : rubric,
'location' : location + suffix,
'course_id' : first_sub.course_id,
'submission_ids_used' : json.dumps(ids),
'problem_id' : first_sub.problem_id,
'model_relative_path' : relative_model_path,
'model_full_path' : full_model_path,
'number_of_essays' : graded_sub_count,
'creation_succeeded': False,
'creation_started' : True,
'creation_finished' : False,
}
transaction.commit()
success, initial_id = ml_grading_util.save_created_model(created_model_dict_initial)
transaction.commit()
results = create.create(text, scores, prompt)
scores = [int(score_item) for score_item in scores]
#Add in needed stuff that ml creator does not pass back
results.update({'text' : text, 'score' : scores, 'model_path' : full_model_path,
'relative_model_path' : relative_model_path, 'prompt' : prompt})
#Try to create model if ml model creator was successful
if results['success']:
try:
success, s3_public_url = save_model_file(results,settings.USE_S3_TO_STORE_MODELS)
results.update({'s3_public_url' : s3_public_url, 'success' : success})
if not success:
results['errors'].append("Could not save model.")
except Exception:
results['errors'].append("Could not save model.")
results['s3_public_url'] = ""
log.exception("Problem saving ML model.")
created_model_dict_final={
'cv_kappa' : results['cv_kappa'],
'cv_mean_absolute_error' : results['cv_mean_absolute_error'],
'creation_succeeded': results['success'],
's3_public_url' : results['s3_public_url'],
'model_stored_in_s3' : settings.USE_S3_TO_STORE_MODELS,
's3_bucketname' : str(settings.S3_BUCKETNAME),
'creation_finished' : True,
'model_relative_path' : relative_model_path,
'model_full_path' : full_model_path,
'location' : location + suffix,
}
transaction.commit()
success, id = ml_grading_util.save_created_model(created_model_dict_final,update_model=True,update_id=initial_id)
else:
log.error("Could not create an ML model. Have you installed all the needed requirements for ease? This is for location {0} and rubric item {1}".format(location, m))
if not success:
log.error("ModelCreator creation failed. Error: {0}".format(id))
statsd.increment("open_ended_assessment.grading_controller.call_ml_creator",
tags=["success:False", "location:{0}".format(location)])
log.info("Location: {0} Creation Status: {1} Errors: {2}".format(
full_model_path,
results['success'],
results['errors'],
))
statsd.increment("open_ended_assessment.grading_controller.call_ml_creator",
tags=["success:{0}".format(results['success']), "location:{0}".format(location)])
util.log_connection_data()
except Exception:
log.exception("Problem creating model for location {0}".format(location))
statsd.increment("open_ended_assessment.grading_controller.call_ml_creator",
tags=["success:Exception", "location:{0}".format(location)])
def handle_single_item(controller_session):
sub_get_success, content = get_item_from_controller(controller_session)
#Grade and handle here
if sub_get_success:
transaction.commit()
sub = Submission.objects.get(id=int(content['submission_id']))
sl = staff_grading_util.StaffLocation(sub.location)
subs_graded_by_instructor = sl.graded()
first_sub = subs_graded_by_instructor.order_by('date_created')[0]
parsed_rubric = rubric_functions.parse_rubric(first_sub.rubric)
#strip out unicode and other characters in student response
#Needed, or grader may potentially fail
#TODO: Handle unicode in student responses properly
student_response = sub.student_response.encode('ascii', 'ignore')
#Get the latest created model for the given location
transaction.commit()
location_suffixes = ml_grading_util.generate_rubric_location_suffixes(
subs_graded_by_instructor, grading=True)
if len(location_suffixes) > 0:
rubric_scores_complete = True
rubric_scores = []
for m in xrange(0, len(location_suffixes)):
suffix = location_suffixes[m]
success, created_model = ml_grading_util.get_latest_created_model(
sub.location + suffix)
if not success:
log.error("Could not identify a valid created model!")
if m == 0:
results = RESULT_FAILURE_DICT
formatted_feedback = "error"
status = GraderStatus.failure
statsd.increment(
"open_ended_assessment.grading_controller.call_ml_grader",
tags=["success:False"])
else:
#Create grader path from location in submission
grader_path = os.path.join(settings.ML_MODEL_PATH,
created_model.model_relative_path)
model_stored_in_s3 = created_model.model_stored_in_s3
success, grader_data = load_model_file(created_model,
use_full_path=False)
if success:
results = grade.grade(grader_data, student_response)
else:
results = RESULT_FAILURE_DICT
#If the above fails, try using the full path in the created_model object
if not results[
'success'] and not created_model.model_stored_in_s3:
grader_path = created_model.model_full_path
try:
success, grader_data = load_model_file(
created_model, use_full_path=True)
if success:
results = grade.grade(grader_data,
student_response)
else:
results = RESULT_FAILURE_DICT
except Exception:
error_message = "Could not find a valid model file."
log.exception(error_message)
results = RESULT_FAILURE_DICT
log.info("ML Grader: Success: {0} Errors: {1}".format(
results['success'], results['errors']))
statsd.increment(
"open_ended_assessment.grading_controller.call_ml_grader",
tags=[
"success:{0}".format(results['success']),
'location:{0}'.format(sub.location)
])
#Set grader status according to success/fail
if results['success']:
status = GraderStatus.success
else:
status = GraderStatus.failure
if m == 0:
final_results = results
elif results['success'] == False:
rubric_scores_complete = False
else:
rubric_scores.append(int(results['score']))
if len(rubric_scores) == 0:
rubric_scores_complete = False
grader_dict = {
'score': int(final_results['score']),
'feedback': json.dumps(results['feedback']),
'status': status,
'grader_id': 1,
'grader_type': "ML",
'confidence': results['confidence'],
'submission_id': sub.id,
'errors': ' '.join(results['errors']),
'rubric_scores_complete': rubric_scores_complete,
'rubric_scores': json.dumps(rubric_scores),
}
#Create grader object in controller by posting back results
created, msg = util._http_post(
controller_session,
urlparse.urljoin(settings.GRADING_CONTROLLER_INTERFACE['url'],
project_urls.ControllerURLs.put_result),
grader_dict,
settings.REQUESTS_TIMEOUT,
)
else:
log.error("Error getting item from controller or no items to get.")
statsd.increment(
"open_ended_assessment.grading_controller.call_ml_grader",
tags=["success:False"])
util.log_connection_data()
return sub_get_success
def get_next_submission(request):
"""
Supports GET request with the following arguments:
course_id -- the course for which to return a submission.
grader_id -- LMS user_id of the requesting user
Returns json dict with the following keys:
version: '1' (number)
success: bool
if success:
'submission_id': a unique identifier for the submission, to be passed
back with the grade.
'submission': the submission, rendered as read-only html for grading
'rubric': the rubric, also rendered as html.
'prompt': the question prompt, also rendered as html.
'message': if there was no submission available, but nothing went wrong,
there will be a message field.
else:
'error': if success is False, will have an error message with more info.
}
"""
if request.method != "GET":
raise Http404
course_id = request.GET.get('course_id')
grader_id = request.GET.get('grader_id')
location = request.GET.get('location')
log.debug(
"Getting next submission for instructor grading for course: {0}.".
format(course_id))
if not (course_id or location) or not grader_id:
return util._error_response("required_parameter_missing",
_INTERFACE_VERSION)
if location:
(
found, id
) = staff_grading_util.get_single_instructor_grading_item_for_location(
location)
# TODO: save the grader id and match it in save_grade to make sure things
# are consistent.
if not location:
(found,
id) = staff_grading_util.get_single_instructor_grading_item(course_id)
if not found:
return util._success_response(
{'message': 'No more submissions to grade.'}, _INTERFACE_VERSION)
try:
submission = Submission.objects.get(id=int(id))
except Submission.DoesNotExist:
log.error("Couldn't find submission %s for instructor grading", id)
return util._error_response('failed_to_load_submission',
_INTERFACE_VERSION,
data={'submission_id': id})
#Get error metrics from ml grading, and get into dictionary form to pass down to staff grading view
success, ml_error_info = ml_grading_util.get_ml_errors(submission.location)
if success:
ml_error_message = staff_grading_util.generate_ml_error_message(
ml_error_info)
else:
ml_error_message = ml_error_info
ml_error_message = "Machine learning error information: " + ml_error_message
if submission.state != 'C':
log.error(
"Instructor grading got a submission (%s) in an invalid state: ",
id, submission.state)
return util._error_response('wrong_internal_state',
_INTERFACE_VERSION,
data={
'submission_id': id,
'submission_state': submission.state
})
num_graded, num_pending = staff_grading_util.count_submissions_graded_and_pending_instructor(
submission.location)
response = {
'submission_id':
id,
'submission':
submission.student_response,
'rubric':
submission.rubric,
'prompt':
submission.prompt,
'max_score':
submission.max_score,
'ml_error_info':
ml_error_message,
'problem_name':
submission.problem_id,
'num_graded':
staff_grading_util.finished_submissions_graded_by_instructor(
submission.location).count(),
'num_pending':
staff_grading_util.submissions_pending_instructor(
submission.location,
state_in=[SubmissionState.waiting_to_be_graded]).count(),
'min_for_ml':
settings.MIN_TO_USE_ML,
}
util.log_connection_data()
log.debug("Sending success response back to instructor grading!")
log.debug("Sub id from get next: {0}".format(submission.id))
return util._success_response(response, _INTERFACE_VERSION)
def handle_single_location(location):
try:
transaction.commit()
gc.collect()
subs_graded_by_instructor = staff_grading_util.finished_submissions_graded_by_instructor(location)
log.debug("Checking location {0} to see if essay count {1} greater than min {2}".format(
location,
subs_graded_by_instructor.count(),
settings.MIN_TO_USE_ML,
))
graded_sub_count=subs_graded_by_instructor.count()
#check to see if there are enough instructor graded essays for location
if graded_sub_count >= settings.MIN_TO_USE_ML:
location_suffixes=ml_grading_util.generate_rubric_location_suffixes(subs_graded_by_instructor, grading=False)
sub_rubric_scores=[]
if len(location_suffixes)>0:
for sub in subs_graded_by_instructor:
success, scores = controller.rubric_functions.get_submission_rubric_instructor_scores(sub)
sub_rubric_scores.append(scores)
if settings.MAX_TO_USE_ML<graded_sub_count:
graded_sub_count = settings.MAX_TO_USE_ML
subs_graded_by_instructor = subs_graded_by_instructor[:settings.MAX_TO_USE_ML]
for m in xrange(0,len(location_suffixes)):
log.debug("Currently on location {0}. Greater than zero is a rubric item.".format(m))
suffix=location_suffixes[m]
#Get paths to ml model from database
relative_model_path, full_model_path= ml_grading_util.get_model_path(location + suffix)
#Get last created model for given location
transaction.commit()
success, latest_created_model=ml_grading_util.get_latest_created_model(location + suffix)
if success:
sub_count_diff=graded_sub_count-latest_created_model.number_of_essays
else:
sub_count_diff = graded_sub_count
#Retrain if no model exists, or every 5 graded essays.
if not success or sub_count_diff>=5:
text = [str(i.student_response.encode('ascii', 'ignore')) for i in subs_graded_by_instructor]
ids=[i.id for i in subs_graded_by_instructor]
#TODO: Make queries more efficient
#This is for the basic overall score
if m==0:
scores = [z.get_last_grader().score for z in list(subs_graded_by_instructor)]
else:
scores=[z[m-1] for z in sub_rubric_scores]
#Get the first graded submission, so that we can extract metadata like rubric, etc, from it
first_sub=subs_graded_by_instructor[0]
prompt = str(first_sub.prompt.encode('ascii', 'ignore'))
rubric = str(first_sub.rubric.encode('ascii', 'ignore'))
transaction.commit()
#Checks to see if another model creator process has started amodel for this location
success, model_started, created_model = ml_grading_util.check_if_model_started(location + suffix)
#Checks to see if model was started a long time ago, and removes and retries if it was.
if model_started:
now = timezone.now()
second_difference = (now - created_model.date_modified).total_seconds()
if second_difference > settings.TIME_BEFORE_REMOVING_STARTED_MODEL:
log.error("Model for location {0} started over {1} seconds ago, removing and re-attempting.".format(
location + suffix, settings.TIME_BEFORE_REMOVING_STARTED_MODEL))
created_model.delete()
model_started = False
if not model_started:
created_model_dict_initial={
'max_score' : first_sub.max_score,
'prompt' : prompt,
'rubric' : rubric,
'location' : location + suffix,
'course_id' : first_sub.course_id,
'submission_ids_used' : json.dumps(ids),
'problem_id' : first_sub.problem_id,
'model_relative_path' : relative_model_path,
'model_full_path' : full_model_path,
'number_of_essays' : graded_sub_count,
'creation_succeeded': False,
'creation_started' : True,
'creation_finished' : False,
}
transaction.commit()
success, initial_id = ml_grading_util.save_created_model(created_model_dict_initial)
transaction.commit()
results = create.create(text, scores, prompt)
scores = [int(score_item) for score_item in scores]
#Add in needed stuff that ml creator does not pass back
results.update({'text' : text, 'score' : scores, 'model_path' : full_model_path,
'relative_model_path' : relative_model_path, 'prompt' : prompt})
#Try to create model if ml model creator was successful
if results['success']:
try:
success, s3_public_url = save_model_file(results,settings.USE_S3_TO_STORE_MODELS)
results.update({'s3_public_url' : s3_public_url, 'success' : success})
if not success:
results['errors'].append("Could not save model.")
except:
results['errors'].append("Could not save model.")
results['s3_public_url'] = ""
log.exception("Problem saving ML model.")
created_model_dict_final={
'cv_kappa' : results['cv_kappa'],
'cv_mean_absolute_error' : results['cv_mean_absolute_error'],
'creation_succeeded': results['success'],
's3_public_url' : results['s3_public_url'],
'model_stored_in_s3' : settings.USE_S3_TO_STORE_MODELS,
's3_bucketname' : str(settings.S3_BUCKETNAME),
'creation_finished' : True,
'model_relative_path' : relative_model_path,
'model_full_path' : full_model_path,
'location' : location + suffix,
}
transaction.commit()
success, id = ml_grading_util.save_created_model(created_model_dict_final,update_model=True,update_id=initial_id)
if not success:
log.error("ModelCreator creation failed. Error: {0}".format(id))
statsd.increment("open_ended_assessment.grading_controller.call_ml_creator",
tags=["success:False", "location:{0}".format(location)])
log.debug("Location: {0} Creation Status: {1} Errors: {2}".format(
full_model_path,
results['success'],
results['errors'],
))
statsd.increment("open_ended_assessment.grading_controller.call_ml_creator",
tags=["success:{0}".format(results['success']), "location:{0}".format(location)])
util.log_connection_data()
except:
log.exception("Problem creating model for location {0}".format(location))
statsd.increment("open_ended_assessment.grading_controller.call_ml_creator",
tags=["success:Exception", "location:{0}".format(location)])
def get_next_submission(request):
"""
Supports GET request with the following arguments:
course_id -- the course for which to return a submission.
grader_id -- LMS user_id of the requesting user
Returns json dict with the following keys:
version: '1' (number)
success: bool
if success:
'submission_id': a unique identifier for the submission, to be passed
back with the grade.
'submission': the submission, rendered as read-only html for grading
'rubric': the rubric, also rendered as html.
'prompt': the question prompt, also rendered as html.
'message': if there was no submission available, but nothing went wrong,
there will be a message field.
else:
'error': if success is False, will have an error message with more info.
}
"""
if request.method != "GET":
raise Http404
course_id = request.GET.get('course_id')
grader_id = request.GET.get('grader_id')
location = request.GET.get('location')
log.debug("Getting next submission for instructor grading for course: {0}."
.format(course_id))
if not (course_id or location) or not grader_id:
return util._error_response("required_parameter_missing", _INTERFACE_VERSION)
if location:
(found, id) = staff_grading_util.get_single_instructor_grading_item_for_location(location)
# TODO: save the grader id and match it in save_grade to make sure things
# are consistent.
if not location:
(found, id) = staff_grading_util.get_single_instructor_grading_item(course_id)
if not found:
return util._success_response({'message': 'No more submissions to grade.'},
_INTERFACE_VERSION)
try:
submission = Submission.objects.get(id=int(id))
except Submission.DoesNotExist:
log.error("Couldn't find submission %s for instructor grading", id)
return util._error_response('failed_to_load_submission',
_INTERFACE_VERSION,
data={'submission_id': id})
#Get error metrics from ml grading, and get into dictionary form to pass down to staff grading view
success, ml_error_info=ml_grading_util.get_ml_errors(submission.location)
if success:
ml_error_message=staff_grading_util.generate_ml_error_message(ml_error_info)
else:
ml_error_message=ml_error_info
ml_error_message="Machine learning error information: " + ml_error_message
if submission.state != 'C':
log.error("Instructor grading got a submission (%s) in an invalid state: ",
id, submission.state)
return util._error_response('wrong_internal_state',
_INTERFACE_VERSION,
data={'submission_id': id,
'submission_state': submission.state})
num_graded, num_pending = staff_grading_util.count_submissions_graded_and_pending_instructor(submission.location)
response = {'submission_id': id,
'submission': submission.student_response,
'rubric': submission.rubric,
'prompt': submission.prompt,
'max_score': submission.max_score,
'ml_error_info' : ml_error_message,
'problem_name' : submission.problem_id,
'num_graded' : staff_grading_util.finished_submissions_graded_by_instructor(submission.location).count(),
'num_pending' : staff_grading_util.submissions_pending_instructor(submission.location,
state_in=[SubmissionState.waiting_to_be_graded]).count(),
'min_for_ml' : settings.MIN_TO_USE_ML,
}
util.log_connection_data()
log.debug("Sending success response back to instructor grading!")
log.debug("Sub id from get next: {0}".format(submission.id))
return util._success_response(response, _INTERFACE_VERSION)