def get_submission_ml(request): """ Gets a submission for the ML grader Input: Get request with no parameters """ unique_locations = [x["location"] for x in list(Submission.objects.values("location").distinct())] for location in unique_locations: subs_graded_by_instructor = staff_grading_util.finished_submissions_graded_by_instructor(location).count() success = ml_grading_util.check_for_all_model_and_rubric_success(location) if subs_graded_by_instructor >= settings.MIN_TO_USE_ML and success: to_be_graded = Submission.objects.filter( location=location, state=SubmissionState.waiting_to_be_graded, next_grader_type="ML" ) if to_be_graded.count() > 0: to_be_graded = to_be_graded[0] if to_be_graded is not None: to_be_graded.state = SubmissionState.being_graded to_be_graded.save() # Insert timing initialization code initialize_timing(to_be_graded) return util._success_response({"submission_id": to_be_graded.id}, _INTERFACE_VERSION) util.log_connection_data() return util._error_response("Nothing to grade.", _INTERFACE_VERSION)
def get_submission_ml(request): """ Gets a submission for the ML grader Input: Get request with no parameters """ unique_locations = [x['location'] for x in list(Submission.objects.values('location').distinct())] for location in unique_locations: subs_graded_by_instructor = staff_grading_util.finished_submissions_graded_by_instructor(location).count() success = ml_grading_util.check_for_all_model_and_rubric_success(location) if subs_graded_by_instructor >= settings.MIN_TO_USE_ML and success: to_be_graded = Submission.objects.filter( location=location, state=SubmissionState.waiting_to_be_graded, next_grader_type="ML", ) if(to_be_graded.count() > 0): to_be_graded = to_be_graded[0] if to_be_graded is not None: to_be_graded.state = SubmissionState.being_graded to_be_graded.save() #Insert timing initialization code initialize_timing(to_be_graded) return util._success_response({'submission_id' : to_be_graded.id}, _INTERFACE_VERSION) util.log_connection_data() return util._error_response("Nothing to grade.", _INTERFACE_VERSION)
def get_single_instructor_grading_item_for_location_with_options(location,check_for_ml=True,types_to_check_for=None, submission_state_to_check_for=SubmissionState.waiting_to_be_graded): """ Returns a single instructor grading item for a given location Input: Problem location, boolean check_for_ML, which dictates whether or not problems should be returned to the instructor if there is already an ML model trained for this location or not. If True, then it does not return submissions for which an ML model has already been trained. Output: Boolean success/fail, and then either error message or submission id of a valid submission. """ if not types_to_check_for: types_to_check_for="IN" log.debug("Looking for location {0}, state {1}, next_grader_type {2}".format(location, submission_state_to_check_for, types_to_check_for)) subs_graded = finished_submissions_graded_by_instructor(location).count() subs_pending = submissions_pending_instructor(location, state_in=[SubmissionState.being_graded]).count() success= ml_grading_util.check_for_all_model_and_rubric_success(location) if ((subs_graded + subs_pending) < settings.MIN_TO_USE_ML or not success) or not check_for_ml: to_be_graded = Submission.objects.filter( location=location, state=submission_state_to_check_for, next_grader_type=types_to_check_for, ) #Order by confidence if we are looking for finished ML submissions finished_submission_text=submission_text_graded_by_instructor(location) if types_to_check_for == "ML" and submission_state_to_check_for == SubmissionState.finished: to_be_graded = to_be_graded.filter(grader__status_code=GraderStatus.success).order_by('grader__confidence') to_be_graded_count=to_be_graded.count() log.debug("Looking for location {0} and got count {1}".format(location,to_be_graded_count)) for i in xrange(0,to_be_graded_count): #In some cases, this causes a model query error without the try/except block due to the checked out state try: to_be_graded_obj = to_be_graded[i] except: return False, 0 if to_be_graded_obj is not None and to_be_graded_obj.student_response not in finished_submission_text: to_be_graded_obj.state = SubmissionState.being_graded to_be_graded_obj.next_grader_type="IN" to_be_graded_obj.save() found = True sub_id = to_be_graded_obj.id #Insert timing initialization code initialize_timing(sub_id) return found, sub_id #If nothing is found, return false return False, 0
def get_submission_instructor(request): """ Gets a submission for the Instructor grading view """ try: course_id = util._value_or_default(request.GET["course_id"], None) except: return util._error_response("'get_submission' requires parameter 'course_id'", _INTERFACE_VERSION) found, sub_id = staff_grading_util.get_single_instructor_grading_item(course_id) if not found: return util._error_response("Nothing to grade.", _INTERFACE_VERSION) # Insert timing initialization code initialize_timing(sub_id) util.log_connection_data() return util._success_response({"submission_id": sub_id}, _INTERFACE_VERSION)
def get_submission_instructor(request): """ Gets a submission for the Instructor grading view """ try: course_id = util._value_or_default(request.GET['course_id'], None) except: return util._error_response("'get_submission' requires parameter 'course_id'", _INTERFACE_VERSION) found, sub_id = staff_grading_util.get_single_instructor_grading_item(course_id) if not found: return util._error_response("Nothing to grade.", _INTERFACE_VERSION) #Insert timing initialization code initialize_timing(sub_id) util.log_connection_data() return util._success_response({'submission_id' : sub_id}, _INTERFACE_VERSION)
def get_submission_peer(request): """ Gets a submission for the Peer grading view """ try: location = util._value_or_default(request.GET["location"], None) grader_id = util._value_or_default(request.GET["grader_id"], None) except KeyError: return util._error_response("'get_submission' requires parameters 'location', 'grader_id'", _INTERFACE_VERSION) found, sub_id = peer_grading_util.get_single_peer_grading_item(location, grader_id) if not found: return util._error_response("Nothing to grade.", _INTERFACE_VERSION) # Insert timing initialization code initialize_timing(sub_id) util.log_connection_data() return util._success_response({"submission_id": sub_id}, _INTERFACE_VERSION)
def get_submission_peer(request): """ Gets a submission for the Peer grading view """ try: location = util._value_or_default(request.GET['location'], None) grader_id = util._value_or_default(request.GET['grader_id'], None) except KeyError: return util._error_response("'get_submission' requires parameters 'location', 'grader_id'", _INTERFACE_VERSION) found, sub_id = peer_grading_util.get_single_peer_grading_item(location, grader_id) if not found: return util._error_response("Nothing to grade.", _INTERFACE_VERSION) #Insert timing initialization code initialize_timing(sub_id) util.log_connection_data() return util._success_response({'submission_id' : sub_id}, _INTERFACE_VERSION)
def get_single_peer_grading_item(location, grader_id): """ Gets peer grading for a given location and grader. Returns one submission id corresponding to the location and the grader. Input: location - problem location. grader_id - student id of the peer grader Returns: found - Boolean indicating whether or not something to grade was found sub_id - If found, the id of a submission to grade """ found = False sub_id = 0 to_be_graded = peer_grading_submissions_pending_for_location( location, grader_id) #Do some checks to ensure that there are actually items to grade if to_be_graded is not None: to_be_graded_length = to_be_graded.count() if to_be_graded_length > 0: course_id = to_be_graded[0].course_id submissions_to_grade = (to_be_graded.filter( grader__status_code=GraderStatus.success, grader__grader_type__in=[ "PE", "BC" ]).exclude(grader__grader_id=grader_id).annotate( num_graders=Count('grader')).values( "num_graders", "id").order_by("num_graders")[:50]) if submissions_to_grade is not None: submission_grader_counts = [ p['num_graders'] for p in submissions_to_grade ] #log.debug("Submissions to grade with graders: {0} {1}".format(submission_grader_counts, submissions_to_grade)) submission_ids = [p['id'] for p in submissions_to_grade] student_profile_success, profile_dict = utilize_student_metrics.get_student_profile( grader_id, course_id) #Ensure that student hasn't graded this submission before! #Also ensures that all submissions are searched through if student has graded the minimum one fallback_sub_id = None for i in xrange(0, len(submission_ids)): #log.debug("Looping through graders, on {0}".format(i)) minimum_index = submission_grader_counts.index( min(submission_grader_counts)) grade_item = Submission.objects.get( id=int(submission_ids[minimum_index])) previous_graders = [ p.grader_id for p in grade_item.get_successful_peer_graders() ] if grader_id not in previous_graders: found = True sub_id = grade_item.id #Insert timing initialization code if fallback_sub_id is None: fallback_sub_id = grade_item.id if not student_profile_success: initialize_timing(sub_id) grade_item.state = SubmissionState.being_graded grade_item.save() return found, sub_id else: success, similarity_score = utilize_student_metrics.get_similarity_score( profile_dict, grade_item.student_id, course_id) log.debug(similarity_score) if similarity_score <= settings.PEER_GRADER_MIN_SIMILARITY_FOR_MATCHING: initialize_timing(sub_id) grade_item.state = SubmissionState.being_graded grade_item.save() return found, sub_id else: if len(submission_ids) > 1: submission_ids.pop(minimum_index) submission_grader_counts.pop(minimum_index) if found: initialize_timing(fallback_sub_id) grade_item = Submission.objects.get(id=fallback_sub_id) grade_item.state = SubmissionState.being_graded grade_item.save() return found, fallback_sub_id return found, sub_id
def handle_submission(sub): """ Handles a new submission. Decides what the next grader should be and saves it. Input: sub - A Submission object from controller.models Output: True/False status code """ try: #Run some basic sanity checks on submission. Also check to see if student is banned, and fail them if they are. sub.next_grader_type = "BC" sub.save() transaction.commit_unless_managed() timing_functions.initialize_timing(sub.id) success, check_dict = basic_check_util.simple_quality_check(sub.student_response, sub.initial_display, sub.student_id, sub.skip_basic_checks) if not success: log.exception("could not run basic checks on {0}".format(sub.student_response)) #add additional tags needed to create a grader object check_dict = grader_util.add_additional_tags_to_dict(check_dict, sub.id) if check_dict['score']==0: success, max_rubric_scores = rubric_functions.generate_targets_from_rubric(sub.rubric) log.debug(max_rubric_scores) if success: check_dict['rubric_scores_complete'] = True check_dict['rubric_scores'] = [0 for i in xrange(0,len(max_rubric_scores))] log.debug(check_dict) #Create and handle the grader, and return grader_util.create_and_handle_grader_object(check_dict) #If the checks result in a score of 0 (out of 1), then the submission fails basic sanity checks #Return to student and don't process further if check_dict['score'] == 0: return True else: sub.state = SubmissionState.waiting_to_be_graded #Assign whether grader should be ML or IN based on number of graded examples. subs_graded_by_instructor, subs_pending_instructor = staff_grading_util.count_submissions_graded_and_pending_instructor( sub.location) #TODO: abstract out logic for assigning which grader to go with. grader_settings_path = os.path.join(settings.GRADER_SETTINGS_DIRECTORY, sub.grader_settings) grader_settings = grader_util.get_grader_settings(grader_settings_path) if grader_settings['grader_type'] == "ML": success= ml_grading_util.check_for_all_model_and_rubric_success(sub.location) if(((subs_graded_by_instructor + subs_pending_instructor) >= settings.MIN_TO_USE_ML) and success): sub.next_grader_type = "ML" else: sub.next_grader_type = "IN" elif grader_settings['grader_type'] == "PE": #Ensures that there will be some calibration essays before peer grading begins! #Calibration essays can be added using command line utility, or through normal instructor grading. if((subs_graded_by_instructor + subs_pending_instructor) >= settings.MIN_TO_USE_PEER): sub.next_grader_type = "PE" else: sub.next_grader_type = "IN" elif grader_settings['grader_type'] == "IN": sub.next_grader_type = "IN" else: log.exception("Invalid grader type specified in settings file.") return False sub.preferred_grader_type=grader_settings['grader_type'] #Do duplicate checks is_duplicate, is_plagiarized, duplicate_id = grader_util.check_is_duplicate_and_plagiarized(sub.student_response, sub.location, sub.student_id, sub.preferred_grader_type) sub.is_duplicate=is_duplicate sub.is_plagiarized = is_plagiarized sub.duplicate_submission_id = duplicate_id sub.has_been_duplicate_checked = True statsd.increment("open_ended_assessment.grading_controller.controller.xqueue_interface.handle_submission.duplicates", tags=[ "duplicate:{0}".format(is_duplicate), "is_plagiarized:{0}".format(is_plagiarized) ]) sub.save() log.debug("Submission object created successfully!") except: log.exception("Submission creation failed!") return False transaction.commit_unless_managed() return True
def get_single_peer_grading_item(location, grader_id): """ Gets peer grading for a given location and grader. Returns one submission id corresponding to the location and the grader. Input: location - problem location. grader_id - student id of the peer grader Returns: found - Boolean indicating whether or not something to grade was found sub_id - If found, the id of a submission to grade """ found = False sub_id = 0 to_be_graded = peer_grading_submissions_pending_for_location(location, grader_id) #Do some checks to ensure that there are actually items to grade if to_be_graded is not None: to_be_graded_length = to_be_graded.count() if to_be_graded_length > 0: course_id = to_be_graded[0].course_id submissions_to_grade = (to_be_graded .filter(grader__status_code=GraderStatus.success, grader__grader_type__in=["PE","BC"]) .exclude(grader__grader_id=grader_id) .annotate(num_graders=Count('grader')) .values("num_graders", "id") .order_by("num_graders")[:50]) if submissions_to_grade is not None: submission_grader_counts = [p['num_graders'] for p in submissions_to_grade] #log.debug("Submissions to grade with graders: {0} {1}".format(submission_grader_counts, submissions_to_grade)) submission_ids = [p['id'] for p in submissions_to_grade] student_profile_success, profile_dict = utilize_student_metrics.get_student_profile(grader_id, course_id) #Ensure that student hasn't graded this submission before! #Also ensures that all submissions are searched through if student has graded the minimum one fallback_sub_id = None for i in xrange(0, len(submission_ids)): #log.debug("Looping through graders, on {0}".format(i)) minimum_index = submission_grader_counts.index(min(submission_grader_counts)) grade_item = Submission.objects.get(id=int(submission_ids[minimum_index])) previous_graders = [p.grader_id for p in grade_item.get_successful_peer_graders()] if grader_id not in previous_graders: found = True sub_id = grade_item.id #Insert timing initialization code if fallback_sub_id is None: fallback_sub_id = grade_item.id if not student_profile_success: initialize_timing(sub_id) grade_item.state = SubmissionState.being_graded grade_item.save() return found, sub_id else: success, similarity_score = utilize_student_metrics.get_similarity_score(profile_dict, grade_item.student_id, course_id) log.debug(similarity_score) if similarity_score <= settings.PEER_GRADER_MIN_SIMILARITY_FOR_MATCHING: initialize_timing(sub_id) grade_item.state = SubmissionState.being_graded grade_item.save() return found, sub_id else: if len(submission_ids) > 1: submission_ids.pop(minimum_index) submission_grader_counts.pop(minimum_index) if found: initialize_timing(fallback_sub_id) grade_item = Submission.objects.get(id=fallback_sub_id) grade_item.state = SubmissionState.being_graded grade_item.save() return found, fallback_sub_id return found, sub_id
def get_single_instructor_grading_item_for_location_with_options( location, check_for_ml=True, types_to_check_for=None, submission_state_to_check_for=SubmissionState.waiting_to_be_graded): """ Returns a single instructor grading item for a given location Input: Problem location, boolean check_for_ML, which dictates whether or not problems should be returned to the instructor if there is already an ML model trained for this location or not. If True, then it does not return submissions for which an ML model has already been trained. Output: Boolean success/fail, and then either error message or submission id of a valid submission. """ if not types_to_check_for: types_to_check_for = "IN" log.debug( "Looking for location {0}, state {1}, next_grader_type {2}".format( location, submission_state_to_check_for, types_to_check_for)) subs_graded = finished_submissions_graded_by_instructor(location).count() subs_pending = submissions_pending_instructor( location, state_in=[SubmissionState.being_graded]).count() success = ml_grading_util.check_for_all_model_and_rubric_success(location) if ((subs_graded + subs_pending) < settings.MIN_TO_USE_ML or not success) or not check_for_ml: to_be_graded = Submission.objects.filter( location=location, state=submission_state_to_check_for, next_grader_type=types_to_check_for, ) #Order by confidence if we are looking for finished ML submissions finished_submission_text = submission_text_graded_by_instructor( location) if types_to_check_for == "ML" and submission_state_to_check_for == SubmissionState.finished: to_be_graded = to_be_graded.filter( grader__status_code=GraderStatus.success).order_by( 'grader__confidence') to_be_graded_count = to_be_graded.count() log.debug("Looking for location {0} and got count {1}".format( location, to_be_graded_count)) for i in xrange(0, to_be_graded_count): #In some cases, this causes a model query error without the try/except block due to the checked out state try: to_be_graded_obj = to_be_graded[i] except: return False, 0 if to_be_graded_obj is not None and to_be_graded_obj.student_response not in finished_submission_text: to_be_graded_obj.state = SubmissionState.being_graded to_be_graded_obj.next_grader_type = "IN" to_be_graded_obj.save() found = True sub_id = to_be_graded_obj.id #Insert timing initialization code initialize_timing(sub_id) return found, sub_id #If nothing is found, return false return False, 0
def handle_submission(sub): """ Handles a new submission. Decides what the next grader should be and saves it. Input: sub - A Submission object from controller.models Output: True/False status code """ try: #Run some basic sanity checks on submission. Also check to see if student is banned, and fail them if they are. sub.next_grader_type = "BC" sub.save() transaction.commit_unless_managed() timing_functions.initialize_timing(sub.id) success, check_dict = basic_check_util.simple_quality_check(sub.student_response, sub.initial_display, sub.student_id, sub.skip_basic_checks) if not success: log.exception("could not run basic checks on {0}".format(sub.student_response)) #add additional tags needed to create a grader object check_dict = grader_util.add_additional_tags_to_dict(check_dict, sub.id) if check_dict['score']==0: success, max_rubric_scores = rubric_functions.generate_targets_from_rubric(sub.rubric) log.debug(max_rubric_scores) if success: check_dict['rubric_scores_complete'] = True check_dict['rubric_scores'] = [0 for i in xrange(0,len(max_rubric_scores))] log.debug(check_dict) #Create and handle the grader, and return grader_util.create_and_handle_grader_object(check_dict) #If the checks result in a score of 0 (out of 1), then the submission fails basic sanity checks #Return to student and don't process further if check_dict['score'] == 0: return True else: sub.state = SubmissionState.waiting_to_be_graded #Assign whether grader should be ML or IN based on number of graded examples. subs_graded_by_instructor, subs_pending_instructor = staff_grading_util.count_submissions_graded_and_pending_instructor( sub.location) #TODO: abstract out logic for assigning which grader to go with. grader_settings_path = os.path.join(settings.GRADER_SETTINGS_DIRECTORY, sub.grader_settings) grader_settings = grader_util.get_grader_settings(grader_settings_path) if grader_settings['grader_type'] == "ML": success= ml_grading_util.check_for_all_model_and_rubric_success(sub.location) if(((subs_graded_by_instructor + subs_pending_instructor) >= settings.MIN_TO_USE_ML) and success): sub.next_grader_type = "ML" else: sub.next_grader_type = "IN" elif grader_settings['grader_type'] == "PE": #Ensures that there will be some calibration essays before peer grading begins! #Calibration essays can be added using command line utility, or through normal instructor grading. if((subs_graded_by_instructor + subs_pending_instructor) >= settings.MIN_TO_USE_PEER): sub.next_grader_type = "PE" else: sub.next_grader_type = "IN" elif grader_settings['grader_type'] == "IN": sub.next_grader_type = "IN" else: log.exception("Invalid grader type specified in settings file.") return False sub.preferred_grader_type=grader_settings['grader_type'] #Do duplicate checks is_duplicate, is_plagiarized, duplicate_id = grader_util.check_is_duplicate_and_plagiarized(sub.student_response, sub.location, sub.student_id, sub.preferred_grader_type) sub.is_duplicate=is_duplicate sub.is_plagiarized = is_plagiarized sub.duplicate_submission_id = duplicate_id statsd.increment("open_ended_assessment.grading_controller.controller.xqueue_interface.handle_submission.duplicates", tags=[ "duplicate:{0}".format(is_duplicate), "is_plagiarized:{0}".format(is_plagiarized) ]) sub.save() log.debug("Submission object created successfully!") except: log.exception("Submission creation failed!") return False transaction.commit_unless_managed() return True