def item_to_score(self): """ Gets an item for an instructor to score. """ subs_graded = self.graded_count() success = ml_grading_util.check_for_all_model_and_rubric_success( self.location) control = SubmissionControl(self.latest_submission()) if subs_graded < control.minimum_to_use_ai or not success: to_be_graded = self.pending() finished_submission_text = self.graded_submission_text() for tbg in to_be_graded: if tbg is not None and tbg.student_response not in finished_submission_text: tbg.state = SubmissionState.being_graded tbg.next_grader_type = "IN" tbg.save() found = True sub_id = tbg.id return found, sub_id #If nothing is found, return false return False, 0
def item_to_rescore(self): """ Gets an item for an instructor to rescore (items have already been machine scored, ML) """ success = ml_grading_util.check_for_all_model_and_rubric_success( self.location) if success: #Order by confidence if we are looking for finished ML submissions finished_submission_text = self.graded_submission_text() to_be_graded = self.pending().filter( grader__status_code=GraderStatus.success).order_by( 'grader__confidence') for tbg in to_be_graded: if tbg is not None and tbg.student_response not in finished_submission_text: tbg.state = SubmissionState.being_graded tbg.next_grader_type = "IN" tbg.save() found = True sub_id = tbg.id return found, sub_id #If nothing is found, return false return False, 0
def get_submission_ml(request): """ Gets a submission for the ML grader Input: Get request with no parameters """ unique_locations = [x["location"] for x in list(Submission.objects.values("location").distinct())] for location in unique_locations: subs_graded_by_instructor = staff_grading_util.finished_submissions_graded_by_instructor(location).count() success = ml_grading_util.check_for_all_model_and_rubric_success(location) if subs_graded_by_instructor >= settings.MIN_TO_USE_ML and success: to_be_graded = Submission.objects.filter( location=location, state=SubmissionState.waiting_to_be_graded, next_grader_type="ML" ) if to_be_graded.count() > 0: to_be_graded = to_be_graded[0] if to_be_graded is not None: to_be_graded.state = SubmissionState.being_graded to_be_graded.save() # Insert timing initialization code initialize_timing(to_be_graded) return util._success_response({"submission_id": to_be_graded.id}, _INTERFACE_VERSION) util.log_connection_data() return util._error_response("Nothing to grade.", _INTERFACE_VERSION)
def item_to_score(self): """ Gets an item for an instructor to score. """ subs_graded = self.graded_count() success = ml_grading_util.check_for_all_model_and_rubric_success(self.location) control = SubmissionControl(self.latest_submission()) if subs_graded < control.minimum_to_use_ai or not success: to_be_graded = self.pending() finished_submission_text=self.graded_submission_text() for tbg in to_be_graded: if tbg is not None: tbg.state = SubmissionState.being_graded tbg.next_grader_type="IN" tbg.save() found = True sub_id = tbg.id return found, sub_id #If nothing is found, return false return False, 0
def get_submission_ml(request): """ Gets a submission for the ML grader Input: Get request with no parameters """ unique_locations = [ x['location'] for x in list(Submission.objects.values('location').distinct()) ] for location in unique_locations: sl = staff_grading_util.StaffLocation(location) subs_graded_by_instructor = sl.graded_count() success = ml_grading_util.check_for_all_model_and_rubric_success( location) if subs_graded_by_instructor >= settings.MIN_TO_USE_ML and success: to_be_graded = Submission.objects.filter( location=location, state=SubmissionState.waiting_to_be_graded, next_grader_type="ML", ) if (to_be_graded.count() > 0): to_be_graded = to_be_graded[0] if to_be_graded is not None: to_be_graded.state = SubmissionState.being_graded to_be_graded.save() return util._success_response( {'submission_id': to_be_graded.id}, _INTERFACE_VERSION) util.log_connection_data() return util._error_response("Nothing to grade.", _INTERFACE_VERSION)
def get_single_instructor_grading_item_for_location_with_options(location,check_for_ml=True,types_to_check_for=None, submission_state_to_check_for=SubmissionState.waiting_to_be_graded): """ Returns a single instructor grading item for a given location Input: Problem location, boolean check_for_ML, which dictates whether or not problems should be returned to the instructor if there is already an ML model trained for this location or not. If True, then it does not return submissions for which an ML model has already been trained. Output: Boolean success/fail, and then either error message or submission id of a valid submission. """ if not types_to_check_for: types_to_check_for="IN" log.debug("Looking for location {0}, state {1}, next_grader_type {2}".format(location, submission_state_to_check_for, types_to_check_for)) subs_graded = finished_submissions_graded_by_instructor(location).count() subs_pending = submissions_pending_instructor(location, state_in=[SubmissionState.being_graded]).count() success= ml_grading_util.check_for_all_model_and_rubric_success(location) if ((subs_graded + subs_pending) < settings.MIN_TO_USE_ML or not success) or not check_for_ml: to_be_graded = Submission.objects.filter( location=location, state=submission_state_to_check_for, next_grader_type=types_to_check_for, ) #Order by confidence if we are looking for finished ML submissions finished_submission_text=submission_text_graded_by_instructor(location) if types_to_check_for == "ML" and submission_state_to_check_for == SubmissionState.finished: to_be_graded = to_be_graded.filter(grader__status_code=GraderStatus.success).order_by('grader__confidence') to_be_graded_count=to_be_graded.count() log.debug("Looking for location {0} and got count {1}".format(location,to_be_graded_count)) for i in xrange(0,to_be_graded_count): #In some cases, this causes a model query error without the try/except block due to the checked out state try: to_be_graded_obj = to_be_graded[i] except: return False, 0 if to_be_graded_obj is not None and to_be_graded_obj.student_response not in finished_submission_text: to_be_graded_obj.state = SubmissionState.being_graded to_be_graded_obj.next_grader_type="IN" to_be_graded_obj.save() found = True sub_id = to_be_graded_obj.id #Insert timing initialization code initialize_timing(sub_id) return found, sub_id #If nothing is found, return false return False, 0
def get_submission_ml(request): """ Gets a submission for the ML grader Input: Get request with no parameters """ unique_locations = [ x['location'] for x in list(Submission.objects.values('location').distinct()) ] for location in unique_locations: nothing_to_ml_grade_for_location_key = NOTHING_TO_ML_GRADE_LOCATION_CACHE_KEY.format( location=location) # Go to the next location if we have recently determined that a location # has no ML grading ready. if cache.get(nothing_to_ml_grade_for_location_key): continue sl = staff_grading_util.StaffLocation(location) control = SubmissionControl(sl.latest_submission()) subs_graded_by_instructor = sl.graded_count() success = ml_grading_util.check_for_all_model_and_rubric_success( location) if subs_graded_by_instructor >= control.minimum_to_use_ai and success: to_be_graded = Submission.objects.filter( location=location, state=SubmissionState.waiting_to_be_graded, next_grader_type="ML", ) if (to_be_graded.count() > 0): to_be_graded = to_be_graded[0] if to_be_graded is not None: to_be_graded.state = SubmissionState.being_graded to_be_graded.save() return util._success_response( {'submission_id': to_be_graded.id}, _INTERFACE_VERSION) # If we don't get a submission to return, then there is no ML grading for this location. # Cache this boolean to avoid an expensive loop iteration. cache.set(nothing_to_ml_grade_for_location_key, True, settings.RECHECK_EMPTY_ML_GRADE_QUEUE_DELAY) util.log_connection_data() # Set this cache key to ensure that this expensive function isn't repeatedly called when not needed. cache.set(NOTHING_TO_ML_GRADE_CACHE_KEY, True, settings.RECHECK_EMPTY_ML_GRADE_QUEUE_DELAY) return util._error_response("Nothing to grade.", _INTERFACE_VERSION)
def get_submission_ml(request): """ Gets a submission for the ML grader Input: Get request with no parameters """ unique_locations = [x['location'] for x in list(Submission.objects.values('location').distinct())] for location in unique_locations: nothing_to_ml_grade_for_location_key = NOTHING_TO_ML_GRADE_LOCATION_CACHE_KEY.format(location=location) # Go to the next location if we have recently determined that a location # has no ML grading ready. if cache.get(nothing_to_ml_grade_for_location_key): continue sl = staff_grading_util.StaffLocation(location) subs_graded_by_instructor = sl.graded_count() success = ml_grading_util.check_for_all_model_and_rubric_success(location) if subs_graded_by_instructor >= settings.MIN_TO_USE_ML and success: to_be_graded = Submission.objects.filter( location=location, state=SubmissionState.waiting_to_be_graded, next_grader_type="ML", ) if(to_be_graded.count() > 0): to_be_graded = to_be_graded[0] if to_be_graded is not None: to_be_graded.state = SubmissionState.being_graded to_be_graded.save() return util._success_response({'submission_id' : to_be_graded.id}, _INTERFACE_VERSION) # If we don't get a submission to return, then there is no ML grading for this location. # Cache this boolean to avoid an expensive loop iteration. cache.set(nothing_to_ml_grade_for_location_key, True, settings.RECHECK_EMPTY_ML_GRADE_QUEUE_DELAY) util.log_connection_data() # Set this cache key to ensure that this expensive function isn't repeatedly called when not needed. cache.set(NOTHING_TO_ML_GRADE_CACHE_KEY, True, settings.RECHECK_EMPTY_ML_GRADE_QUEUE_DELAY) return util._error_response("Nothing to grade.", _INTERFACE_VERSION)
def item_to_rescore(self): """ Gets an item for an instructor to rescore (items have already been machine scored, ML) """ success= ml_grading_util.check_for_all_model_and_rubric_success(self.location) if success: #Order by confidence if we are looking for finished ML submissions finished_submission_text=self.graded_submission_text() to_be_graded = self.pending().filter(grader__status_code=GraderStatus.success).order_by('grader__confidence') for tbg in to_be_graded: if tbg is not None and tbg.student_response not in finished_submission_text: tbg.state = SubmissionState.being_graded tbg.next_grader_type="IN" tbg.save() found = True sub_id = tbg.id return found, sub_id #If nothing is found, return false return False, 0
def reset_in_subs_to_ml(subs): count=0 in_subs=Submission.objects.filter( state=SubmissionState.waiting_to_be_graded, next_grader_type="IN", preferred_grader_type="ML" ) for sub in in_subs: #If an instructor checks out a submission after ML grading has started, # this resets it to ML if the instructor times out success= ml_grading_util.check_for_all_model_and_rubric_success(sub.location) if (sub.next_grader_type=="IN" and success): sub.next_grader_type="ML" sub.save() count+=1 if count>0: statsd.increment("open_ended_assessment.grading_controller.expire_submissions.reset_in_subs_to_ml", tags=["counter:{0}".format(count)]) log.debug("Reset {0} instructor subs to ML".format(count)) return True
def reset_in_subs_to_ml(): count=0 in_subs=Submission.objects.filter( state=SubmissionState.waiting_to_be_graded, next_grader_type="IN", preferred_grader_type="ML" ) for sub in in_subs: #If an instructor checks out a submission after ML grading has started, # this resets it to ML if the instructor times out success= ml_grading_util.check_for_all_model_and_rubric_success(sub.location) if (sub.next_grader_type=="IN" and success): sub.next_grader_type="ML" sub.save() count+=1 if count>0: statsd.increment("open_ended_assessment.grading_controller.expire_submissions.reset_in_subs_to_ml", tags=["counter:{0}".format(count)]) log.debug("Reset {0} instructor subs to ML".format(count)) return True
def item_to_score(self): """ Gets an item for an instructor to score. """ subs_graded = self.graded_count() success= ml_grading_util.check_for_all_model_and_rubric_success(self.location) if subs_graded < settings.MIN_TO_USE_ML or not success: to_be_graded = self.pending() finished_submission_text=self.graded_submission_text() for tbg in to_be_graded: if tbg is not None and tbg.student_response not in finished_submission_text: tbg.state = SubmissionState.being_graded tbg.next_grader_type="IN" tbg.save() found = True sub_id = tbg.id return found, sub_id #If nothing is found, return false return False, 0
def handle_submission(sub): """ Handles a new submission. Decides what the next grader should be and saves it. Input: sub - A Submission object from controller.models Output: True/False status code """ try: #Run some basic sanity checks on submission. Also check to see if student is banned, and fail them if they are. sub.next_grader_type = "BC" sub.save() transaction.commit_unless_managed() timing_functions.initialize_timing(sub.id) success, check_dict = basic_check_util.simple_quality_check(sub.student_response, sub.initial_display, sub.student_id, sub.skip_basic_checks) if not success: log.exception("could not run basic checks on {0}".format(sub.student_response)) #add additional tags needed to create a grader object check_dict = grader_util.add_additional_tags_to_dict(check_dict, sub.id) if check_dict['score']==0: success, max_rubric_scores = rubric_functions.generate_targets_from_rubric(sub.rubric) log.debug(max_rubric_scores) if success: check_dict['rubric_scores_complete'] = True check_dict['rubric_scores'] = [0 for i in xrange(0,len(max_rubric_scores))] log.debug(check_dict) #Create and handle the grader, and return grader_util.create_and_handle_grader_object(check_dict) #If the checks result in a score of 0 (out of 1), then the submission fails basic sanity checks #Return to student and don't process further if check_dict['score'] == 0: return True else: sub.state = SubmissionState.waiting_to_be_graded #Assign whether grader should be ML or IN based on number of graded examples. subs_graded_by_instructor, subs_pending_instructor = staff_grading_util.count_submissions_graded_and_pending_instructor( sub.location) #TODO: abstract out logic for assigning which grader to go with. grader_settings_path = os.path.join(settings.GRADER_SETTINGS_DIRECTORY, sub.grader_settings) grader_settings = grader_util.get_grader_settings(grader_settings_path) if grader_settings['grader_type'] == "ML": success= ml_grading_util.check_for_all_model_and_rubric_success(sub.location) if(((subs_graded_by_instructor + subs_pending_instructor) >= settings.MIN_TO_USE_ML) and success): sub.next_grader_type = "ML" else: sub.next_grader_type = "IN" elif grader_settings['grader_type'] == "PE": #Ensures that there will be some calibration essays before peer grading begins! #Calibration essays can be added using command line utility, or through normal instructor grading. if((subs_graded_by_instructor + subs_pending_instructor) >= settings.MIN_TO_USE_PEER): sub.next_grader_type = "PE" else: sub.next_grader_type = "IN" elif grader_settings['grader_type'] == "IN": sub.next_grader_type = "IN" else: log.exception("Invalid grader type specified in settings file.") return False sub.preferred_grader_type=grader_settings['grader_type'] #Do duplicate checks is_duplicate, is_plagiarized, duplicate_id = grader_util.check_is_duplicate_and_plagiarized(sub.student_response, sub.location, sub.student_id, sub.preferred_grader_type) sub.is_duplicate=is_duplicate sub.is_plagiarized = is_plagiarized sub.duplicate_submission_id = duplicate_id sub.has_been_duplicate_checked = True statsd.increment("open_ended_assessment.grading_controller.controller.xqueue_interface.handle_submission.duplicates", tags=[ "duplicate:{0}".format(is_duplicate), "is_plagiarized:{0}".format(is_plagiarized) ]) sub.save() log.debug("Submission object created successfully!") except: log.exception("Submission creation failed!") return False transaction.commit_unless_managed() return True
def handle_submission(sub): """ Handles a new submission. Decides what the next grader should be and saves it. Input: sub - A Submission object from controller.models Output: True/False status code """ try: #Run some basic sanity checks on submission. Also check to see if student is banned, and fail them if they are. sub.next_grader_type = "BC" sub.save() transaction.commit_unless_managed() success, check_dict = basic_check_util.simple_quality_check( sub.student_response, sub.initial_display, sub.student_id, sub.skip_basic_checks) if not success: log.exception("could not run basic checks on {0}".format( sub.student_response)) #add additional tags needed to create a grader object check_dict = grader_util.add_additional_tags_to_dict( check_dict, sub.id) if check_dict['score'] == 0: success, max_rubric_scores = rubric_functions.generate_targets_from_rubric( sub.rubric) if success: check_dict['rubric_scores_complete'] = True check_dict['rubric_scores'] = [ 0 for i in xrange(0, len(max_rubric_scores)) ] #Create and handle the grader, and return grader_util.create_and_handle_grader_object(check_dict) #If the checks result in a score of 0 (out of 1), then the submission fails basic sanity checks #Return to student and don't process further if check_dict['score'] == 0: return True else: sub.state = SubmissionState.waiting_to_be_graded sl = staff_grading_util.StaffLocation(sub.location) #Assign whether grader should be ML or IN based on number of graded examples. subs_graded_by_instructor, subs_pending_instructor = sl.graded_count( ), sl.pending_count() #TODO: abstract out logic for assigning which grader to go with. grader_settings_path = os.path.join(settings.GRADER_SETTINGS_DIRECTORY, sub.grader_settings) grader_settings = grader_util.get_grader_settings(grader_settings_path) if grader_settings['grader_type'] == "ML": success = ml_grading_util.check_for_all_model_and_rubric_success( sub.location) if (((subs_graded_by_instructor + subs_pending_instructor) >= settings.MIN_TO_USE_ML) and success): sub.next_grader_type = "ML" else: sub.next_grader_type = "IN" elif grader_settings['grader_type'] == "PE": #Ensures that there will be some calibration essays before peer grading begins! #Calibration essays can be added using command line utility, or through normal instructor grading. if ((subs_graded_by_instructor + subs_pending_instructor) >= settings.MIN_TO_USE_PEER): sub.next_grader_type = "PE" else: sub.next_grader_type = "IN" elif grader_settings['grader_type'] == "IN": sub.next_grader_type = "IN" else: log.exception("Invalid grader type specified in settings file.") return False sub.preferred_grader_type = grader_settings['grader_type'] #Do duplicate checks is_duplicate, is_plagiarized, duplicate_id = grader_util.check_is_duplicate_and_plagiarized( sub.student_response, sub.location, sub.student_id, sub.preferred_grader_type) sub.is_duplicate = is_duplicate sub.is_plagiarized = is_plagiarized sub.duplicate_submission_id = duplicate_id sub.has_been_duplicate_checked = True statsd.increment( "open_ended_assessment.grading_controller.controller.xqueue_interface.handle_submission.duplicates", tags=[ "duplicate:{0}".format(is_duplicate), "is_plagiarized:{0}".format(is_plagiarized) ]) sub.save() except Exception: log.exception("Submission creation failed!") return False transaction.commit_unless_managed() return True
def get_single_instructor_grading_item_for_location_with_options( location, check_for_ml=True, types_to_check_for=None, submission_state_to_check_for=SubmissionState.waiting_to_be_graded): """ Returns a single instructor grading item for a given location Input: Problem location, boolean check_for_ML, which dictates whether or not problems should be returned to the instructor if there is already an ML model trained for this location or not. If True, then it does not return submissions for which an ML model has already been trained. Output: Boolean success/fail, and then either error message or submission id of a valid submission. """ if not types_to_check_for: types_to_check_for = "IN" log.debug( "Looking for location {0}, state {1}, next_grader_type {2}".format( location, submission_state_to_check_for, types_to_check_for)) subs_graded = finished_submissions_graded_by_instructor(location).count() subs_pending = submissions_pending_instructor( location, state_in=[SubmissionState.being_graded]).count() success = ml_grading_util.check_for_all_model_and_rubric_success(location) if ((subs_graded + subs_pending) < settings.MIN_TO_USE_ML or not success) or not check_for_ml: to_be_graded = Submission.objects.filter( location=location, state=submission_state_to_check_for, next_grader_type=types_to_check_for, ) #Order by confidence if we are looking for finished ML submissions finished_submission_text = submission_text_graded_by_instructor( location) if types_to_check_for == "ML" and submission_state_to_check_for == SubmissionState.finished: to_be_graded = to_be_graded.filter( grader__status_code=GraderStatus.success).order_by( 'grader__confidence') to_be_graded_count = to_be_graded.count() log.debug("Looking for location {0} and got count {1}".format( location, to_be_graded_count)) for i in xrange(0, to_be_graded_count): #In some cases, this causes a model query error without the try/except block due to the checked out state try: to_be_graded_obj = to_be_graded[i] except: return False, 0 if to_be_graded_obj is not None and to_be_graded_obj.student_response not in finished_submission_text: to_be_graded_obj.state = SubmissionState.being_graded to_be_graded_obj.next_grader_type = "IN" to_be_graded_obj.save() found = True sub_id = to_be_graded_obj.id #Insert timing initialization code initialize_timing(sub_id) return found, sub_id #If nothing is found, return false return False, 0