def save_page(self, filename, request): title = request.form['title'] html = sanitize_html(request.form['content']) filename = title_to_filename(title) cursor = self.storage.cursor(request.form['revision']) cursor.add('pages/' + filename, self.to_source(html).encode('utf-8')) if 'headerimage' in request.files: header_image = request.files['headerimage'].read() extension = mimetypes.guess_extension(request.files['headerimage'].mimetype) if extension.startswith('.jpe'): extension = '.jpg' # wtf Python?! if extension in Ikwi.image_extensions: image_filename = filename + extension for other_extension in Ikwi.image_extensions: cursor.delete('images/' + filename + other_extension) cursor.add('images/' + image_filename, header_image) cursor.save('%s: %s' % (title, request.form.get('change_message', '')), Signature(self.config['editors'][request.authorization.username]['name'], self.config['editors'][request.authorization.username]['email'])) status = cursor.update('HEAD') if status.conflict: return JSONResponse({ 'status': 'conflict', 'source': status.source_revision, 'target': status.target_revision, }, 409) return JSONResponse({'status': 'ok', 'revision': status.revision})
def test_sanitize_html(self): feedback = "This is a sample feedback. <img src='abc' onerror=alert(1)>" sanitized_feedback = util.sanitize_html(feedback) # Sanitized feedback should not contain onerror attribute. self.assertFalse("onerror" in sanitized_feedback)
def POST(self, t): self.get_referer() if not self.u: raise web.seeother(self.referer) i = web.input() c = util.sanitize_html(i.content) dbm.wikis.insert_one(t=t, u=self.u, c=c) raise web.seeother('/w/%s' % t)
def receive_message(self, cymessage): message = util.yahoo_rich_to_html(cymessage.message) timestamp = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S') if not cymessage.sender: sender = self.app.me.yahoo_id else: sender = self.cybuddy.display_name if cymessage.offline: message = '(offline) {0}'.format(message) if cymessage.timestamp: timestamp = datetime.datetime.fromtimestamp(int(cymessage.timestamp)).strftime('%Y-%m-%d %H:%M:%S') self._javascript('message_in', sender, self._text_to_emotes(util.sanitize_html(message)), timestamp)
def test_removing_tags(self): """Don't trim spaces""" res = sanitize_html(' <foo>my text here!</foo>') self.assertEqual(res, " ")
def test_empty(self): """Without input return empty string""" res = sanitize_html("") self.assertEqual(res, "")
def test_None(self): """With None return empty string""" res = sanitize_html(None) self.assertEqual(res, "")
def submit_message(request): """ Submits a message to the grading controller. """ if request.method != 'POST': return util._error_response("'submit_message' must use HTTP POST", _INTERFACE_VERSION) reply_is_valid, header, body = _is_valid_reply_message(request.POST.copy()) if not reply_is_valid: log.error( "Invalid xqueue object added: request_ip: {0} request.POST: {1}". format( util.get_request_ip(request), request.POST, )) statsd.increment( "open_ended_assessment.grading_controller.controller.xqueue_interface.submit_message", tags=["success:Exception"]) return util._error_response('Incorrect format', _INTERFACE_VERSION) message = body['feedback'] message = util.sanitize_html(message) grader_id = body['grader_id'] submission_id = body['submission_id'] originator = body['student_info']['anonymous_student_id'] try: if 'score' in body: score = int(body['score']) else: score = None except Exception: error_message = "Score was not an integer, received \"{0}\" instead.".format( score) log.exception(error_message) return util._error_response(error_message, _INTERFACE_VERSION) try: grade = Grader.objects.get(id=grader_id) except Exception: error_message = "Could not find a grader object for message from xqueue" log.exception(error_message) return util._error_response(error_message, _INTERFACE_VERSION) try: submission = Submission.objects.get(id=submission_id) except Exception: error_message = "Could not find a submission object for message from xqueue" log.exception(error_message) return util._error_response(error_message, _INTERFACE_VERSION) if grade.submission.id != submission.id: error_message = "Grader id does not match submission id that was passed in" log.exception(error_message) return util._error_response(error_message, _INTERFACE_VERSION) if originator not in [submission.student_id, grade.grader_id]: error_message = "Message originator is not the grader, or the person being graded" log.exception(error_message) return util._error_response(error_message, _INTERFACE_VERSION) if grade.grader_type in ["ML", "IN"]: recipient_type = "controller" recipient = "controller" else: recipient_type = "human" if recipient_type != 'controller': if originator == submission.student_id: recipient = grade.grader_id elif originator == grade.grader_id: recipient = submission.student_id if recipient not in [submission.student_id, grade.grader_id, 'controller']: error_message = "Message recipient is not the grader, the person being graded, or the controller" log.exception(error_message) return util._error_response(error_message, _INTERFACE_VERSION) if originator == recipient: error_message = "Message recipient is the same as originator" log.exception(error_message) return util._error_response(error_message, _INTERFACE_VERSION) message_dict = { 'grader_id': grader_id, 'originator': originator, 'submission_id': submission_id, 'message': message, 'recipient': recipient, 'message_type': "feedback", 'score': score } success, error = message_util.create_message(message_dict) if not success: return util._error_response(error, _INTERFACE_VERSION) return util._success_response({'message_id': error}, _INTERFACE_VERSION)
def submit(request): ''' Xqueue pull script posts objects here. Input: request - dict with keys xqueue_header and xqueue_body xqueue_header needs submission_id,submission_key,queue_name xqueue_body needs grader_payload, student_info, student_response, max_score grader_payload needs location, course_id,problem_id,grader student_info needs anonymous_student_id, submission_time Output: Returns status code indicating success (0) or failure (1) and message ''' transaction.commit_unless_managed() if request.method != 'POST': return util._error_response("'submit' must use HTTP POST", _INTERFACE_VERSION) else: #Minimal parsing of reply reply_is_valid, header, body = _is_valid_reply(request.POST.copy()) if not reply_is_valid: log.error("Invalid xqueue object added: request_ip: {0} request.POST: {1}".format( util.get_request_ip(request), request.POST, )) statsd.increment("open_ended_assessment.grading_controller.controller.xqueue_interface.submit", tags=["success:Exception"]) return util._error_response('Incorrect format', _INTERFACE_VERSION) else: try: #Retrieve individual values from xqueue body and header. prompt = util._value_or_default(body['grader_payload']['prompt'], "") rubric = util._value_or_default(body['grader_payload']['rubric'], "") student_id = util._value_or_default(body['student_info']['anonymous_student_id']) location = util._value_or_default(body['grader_payload']['location']) course_id = util._value_or_default(body['grader_payload']['course_id']) problem_id = util._value_or_default(body['grader_payload']['problem_id'], location) grader_settings = util._value_or_default(body['grader_payload']['grader_settings'], "") student_response = util._value_or_default(body['student_response']) student_response = util.sanitize_html(student_response) xqueue_submission_id = util._value_or_default(header['submission_id']) xqueue_submission_key = util._value_or_default(header['submission_key']) state_code = SubmissionState.waiting_to_be_graded xqueue_queue_name = util._value_or_default(header["queue_name"]) max_score = util._value_or_default(body['max_score']) submission_time_string = util._value_or_default(body['student_info']['submission_time']) student_submission_time = datetime.strptime(submission_time_string, "%Y%m%d%H%M%S") skip_basic_checks = util._value_or_default(body['grader_payload']['skip_basic_checks'], False) if isinstance(skip_basic_checks, basestring): skip_basic_checks = (skip_basic_checks.lower() == "true") #TODO: find a better way to do this #Need to set rubric to whatever the first submission for this location had #as its rubric. If the rubric is changed in the course XML, it will break things. try: first_sub_for_location=Submission.objects.filter(location=location).order_by('date_created')[0] rubric= first_sub_for_location.rubric except: error_message="Could not find an existing submission in location. rubric is original." log.info(error_message) initial_display="" if 'initial_display' in body['grader_payload'].keys(): initial_display = util._value_or_default(body['grader_payload']['initial_display'], "") answer="" if 'answer' in body['grader_payload'].keys(): answer = util._value_or_default(body['grader_payload']['answer'], "") #Sleep for some time to allow other pull_from_xqueue processes to get behind/ahead time_sleep_value = random.uniform(0, .1) time.sleep(time_sleep_value) transaction.commit_unless_managed() #Without this, sometimes a race condition creates duplicate submissions sub_count = Submission.objects.filter( prompt=prompt, rubric=rubric, student_id=student_id, problem_id=problem_id, student_submission_time=student_submission_time, xqueue_submission_id=xqueue_submission_id, xqueue_submission_key=xqueue_submission_key, xqueue_queue_name=xqueue_queue_name, location=location, course_id=course_id, grader_settings=grader_settings, ).count() if sub_count>0: log.error("Exact submission already exists.") return util._error_response('Submission already exists.', _INTERFACE_VERSION) transaction.commit_unless_managed() #Create submission object sub, created = Submission.objects.get_or_create( prompt=prompt, rubric=rubric, student_id=student_id, problem_id=problem_id, state=state_code, student_response=student_response, student_submission_time=student_submission_time, xqueue_submission_id=xqueue_submission_id, xqueue_submission_key=xqueue_submission_key, xqueue_queue_name=xqueue_queue_name, location=location, course_id=course_id, max_score=max_score, grader_settings=grader_settings, initial_display=initial_display, answer=answer, skip_basic_checks = skip_basic_checks, ) transaction.commit_unless_managed() if created==False: log.error("Exact submission already exists.") return util._error_response('Submission already exists.', _INTERFACE_VERSION) except Exception as err: xqueue_submission_id = util._value_or_default(header['submission_id']) xqueue_submission_key = util._value_or_default(header['submission_key']) log.exception( "Error creating submission and adding to database: sender: {0}, submission_id: {1}, submission_key: {2}".format( util.get_request_ip(request), xqueue_submission_id, xqueue_submission_key, )) statsd.increment("open_ended_assessment.grading_controller.controller.xqueue_interface.submit", tags=["success:Exception"]) return util._error_response('Unable to create submission.', _INTERFACE_VERSION) #Handle submission and write to db success = handle_submission(sub) statsd.increment("open_ended_assessment.grading_controller.controller.xqueue_interface.submit", tags=[ "success:{0}".format(success), "location:{0}".format(sub.location), "course_id:{0}".format(course_id), ]) transaction.commit_unless_managed() if not success: return util._error_response("Failed to handle submission.", _INTERFACE_VERSION) util.log_connection_data() transaction.commit_unless_managed() return util._success_response({'message': "Saved successfully."}, _INTERFACE_VERSION)
def test_remove_script(self): """Remove javascript.""" html = """<p onClick="alert('Hello');">foo</p>""" res = sanitize_html(html) self.assertEqual(res, "<p>foo</p>")
def test_removing_element(self): """Remove html elements""" html = "playnice<something>nasty<!-- javascript --></something>" res = sanitize_html(html) self.assertEqual(res, "playnice")
def test_remove_lone_tags(self): """One-tag -elements should be removed.""" html = "<foo/>Leave me" "" res = sanitize_html(html) self.assertEqual(res, "Leave me")
def test_remove_lone_tags(self): """One-tag -elements should be removed.""" html = "<foo/>Leave me""" res = sanitize_html(html) self.assertEqual(res, "Leave me")
def create_and_handle_grader_object(grader_dict): """ Creates a Grader object and associates it with a given submission Input is grader dictionary with keys: feedback, status, grader_id, grader_type, confidence, score,submission_id, errors Feedback should be a dictionary with as many keys as needed. Errors is a string containing errors. """ for tag in ["feedback", "status", "grader_id", "grader_type", "confidence", "score", "submission_id", "errors"]: if tag not in grader_dict: return False, "{0} tag not in input dictionary.".format(tag) try: sub = Submission.objects.get(id=int(grader_dict['submission_id'])) except: return False, "Error getting submission." try: grader_dict['feedback'] = json.loads(grader_dict['feedback']) except: pass if not isinstance(grader_dict['feedback'], dict): grader_dict['feedback'] = {'feedback': grader_dict['feedback']} for k in grader_dict['feedback']: grader_dict['feedback'][k] = util.sanitize_html(grader_dict['feedback'][k]) if grader_dict['status'] == GraderStatus.failure: grader_dict['feedback'] = ' '.join(grader_dict['errors']) grader_dict['feedback'] = json.dumps(grader_dict['feedback']) grade = create_grader(grader_dict, sub) #Check to see if rubric scores were passed to the function, and handle if so. if 'rubric_scores_complete' in grader_dict and 'rubric_scores' in grader_dict: try: grader_dict['rubric_scores']=json.loads(grader_dict['rubric_scores']) except: pass if grader_dict['rubric_scores_complete'] in ['True', "TRUE", 'true', True]: grader_dict['rubric_scores']=[int(r) for r in grader_dict['rubric_scores']] try: rubric_functions.generate_rubric_object(grade,grader_dict['rubric_scores'], sub.rubric) except: log.exception("Problem with getting rubric scores from dict : {0}".format(grader_dict)) #TODO: Need some kind of logic somewhere else to handle setting next_grader sub.previous_grader_type = grade.grader_type sub.next_grader_type = grade.grader_type #check to see if submission is flagged. If so, put it in a flagged state submission_is_flagged = grader_dict.get('is_submission_flagged', False) if submission_is_flagged: sub.state = SubmissionState.flagged else: #TODO: Some kind of logic to decide when sub is finished grading. #If we are calling this after a basic check and the score is 0, that means that the submission is bad, so mark as finished if(grade.status_code == GraderStatus.success and grade.grader_type in ["BC"] and grade.score==0): sub.state = SubmissionState.finished #If submission is ML or IN graded, and was successful, state is finished elif(grade.status_code == GraderStatus.success and grade.grader_type in ["IN", "ML"]): sub.state = SubmissionState.finished elif(grade.status_code == GraderStatus.success and grade.grader_type in ["PE"]): #If grading type is Peer, and was successful, check to see how many other times peer grading has succeeded. successful_peer_grader_count = sub.get_successful_peer_graders().count() #If number of successful peer graders equals the needed count, finalize submission. if successful_peer_grader_count >= settings.PEER_GRADER_COUNT: sub.state = SubmissionState.finished else: sub.state = SubmissionState.waiting_to_be_graded #If something fails, immediately mark it for regrading #TODO: Get better logic for handling failure cases elif(grade.status_code == GraderStatus.failure and sub.state == SubmissionState.being_graded): number_of_failures = sub.get_unsuccessful_graders().count() #If it has failed too many times, just return an error if number_of_failures > settings.MAX_NUMBER_OF_TIMES_TO_RETRY_GRADING: finalize_expired_submission(sub) else: sub.state = SubmissionState.waiting_to_be_graded #Increment statsd whenever a grader object is saved. statsd.increment("open_ended_assessment.grading_controller.controller.create_grader_object", tags=["submission_state:{0}".format(sub.state), "grader_type:{0}".format(grade.grader_type), "grader_status:{0}".format(grade.status_code), "location:{0}".format(sub.location), "course_id:{0}".format(sub.course_id), "next_grader_type:{0}".format(sub.next_grader_type), "score:{0}".format(grade.score), ] ) sub.save() #Insert timing finalization code finalize_timing(sub, grade) return True, {'submission_id': sub.xqueue_submission_id, 'submission_key': sub.xqueue_submission_key}
def submit(request): ''' Xqueue pull script posts objects here. Input: request - dict with keys xqueue_header and xqueue_body xqueue_header needs submission_id,submission_key,queue_name xqueue_body needs grader_payload, student_info, student_response, max_score grader_payload needs location, course_id,problem_id,grader student_info needs anonymous_student_id, submission_time Output: Returns status code indicating success (0) or failure (1) and message ''' transaction.commit_unless_managed() if request.method != 'POST': return util._error_response("'submit' must use HTTP POST", _INTERFACE_VERSION) else: #Minimal parsing of reply reply_is_valid, header, body = _is_valid_reply(request.POST.copy()) if not reply_is_valid: log.error( "Invalid xqueue object added: request_ip: {0} request.POST: {1}" .format( util.get_request_ip(request), request.POST, )) statsd.increment( "open_ended_assessment.grading_controller.controller.xqueue_interface.submit", tags=["success:Exception"]) return util._error_response('Incorrect format', _INTERFACE_VERSION) else: try: #Retrieve individual values from xqueue body and header. prompt = util._value_or_default( body['grader_payload']['prompt'], "") rubric = util._value_or_default( body['grader_payload']['rubric'], "") student_id = util._value_or_default( body['student_info']['anonymous_student_id']) location = util._value_or_default( body['grader_payload']['location']) course_id = util._value_or_default( body['grader_payload']['course_id']) problem_id = util._value_or_default( body['grader_payload']['problem_id'], location) grader_settings = util._value_or_default( body['grader_payload']['grader_settings'], "") student_response = util._value_or_default( body['student_response']) student_response = util.sanitize_html(student_response) xqueue_submission_id = util._value_or_default( header['submission_id']) xqueue_submission_key = util._value_or_default( header['submission_key']) state_code = SubmissionState.waiting_to_be_graded xqueue_queue_name = util._value_or_default( header["queue_name"]) max_score = util._value_or_default(body['max_score']) submission_time_string = util._value_or_default( body['student_info']['submission_time']) student_submission_time = datetime.strptime( submission_time_string, "%Y%m%d%H%M%S") control_fields = body['grader_payload'].get('control', {}) try: control_fields = json.loads(control_fields) except Exception: pass skip_basic_checks = util._value_or_default( body['grader_payload']['skip_basic_checks'], False) if isinstance(skip_basic_checks, basestring): skip_basic_checks = (skip_basic_checks.lower() == "true") #TODO: find a better way to do this #Need to set rubric to whatever the first submission for this location had #as its rubric. If the rubric is changed in the course XML, it will break things. try: first_sub_for_location = Submission.objects.filter( location=location).order_by('date_created')[0] rubric = first_sub_for_location.rubric except Exception: error_message = "Could not find an existing submission in location. rubric is original." log.info(error_message) initial_display = "" if 'initial_display' in body['grader_payload'].keys(): initial_display = util._value_or_default( body['grader_payload']['initial_display'], "") answer = "" if 'answer' in body['grader_payload'].keys(): answer = util._value_or_default( body['grader_payload']['answer'], "") #Sleep for some time to allow other pull_from_xqueue processes to get behind/ahead time_sleep_value = random.uniform(0, .1) time.sleep(time_sleep_value) transaction.commit_unless_managed() #Without this, sometimes a race condition creates duplicate submissions sub_count = Submission.objects.filter( prompt=prompt, rubric=rubric, student_id=student_id, problem_id=problem_id, student_submission_time=student_submission_time, xqueue_submission_id=xqueue_submission_id, xqueue_submission_key=xqueue_submission_key, xqueue_queue_name=xqueue_queue_name, location=location, course_id=course_id, grader_settings=grader_settings, ).count() if sub_count > 0: return util._error_response('Submission already exists.', _INTERFACE_VERSION) transaction.commit_unless_managed() #Create submission object sub, created = Submission.objects.get_or_create( prompt=prompt, rubric=rubric, student_id=student_id, problem_id=problem_id, state=state_code, student_response=student_response, student_submission_time=student_submission_time, xqueue_submission_id=xqueue_submission_id, xqueue_submission_key=xqueue_submission_key, xqueue_queue_name=xqueue_queue_name, location=location, course_id=course_id, max_score=max_score, grader_settings=grader_settings, initial_display=initial_display, answer=answer, skip_basic_checks=skip_basic_checks, control_fields=json.dumps(control_fields)) transaction.commit_unless_managed() if created == False: return util._error_response('Submission already exists.', _INTERFACE_VERSION) except Exception as err: xqueue_submission_id = util._value_or_default( header['submission_id']) xqueue_submission_key = util._value_or_default( header['submission_key']) log.exception( "Error creating submission and adding to database: sender: {0}, submission_id: {1}, submission_key: {2}" .format( util.get_request_ip(request), xqueue_submission_id, xqueue_submission_key, )) statsd.increment( "open_ended_assessment.grading_controller.controller.xqueue_interface.submit", tags=["success:Exception"]) return util._error_response('Unable to create submission.', _INTERFACE_VERSION) #Handle submission and write to db success = handle_submission(sub) statsd.increment( "open_ended_assessment.grading_controller.controller.xqueue_interface.submit", tags=[ "success:{0}".format(success), "location:{0}".format(sub.location), "course_id:{0}".format(course_id), ]) transaction.commit_unless_managed() if not success: return util._error_response("Failed to handle submission.", _INTERFACE_VERSION) util.log_connection_data() transaction.commit_unless_managed() return util._success_response({'message': "Saved successfully."}, _INTERFACE_VERSION)
def submit_message(request): """ Submits a message to the grading controller. """ if request.method != 'POST': return util._error_response("'submit_message' must use HTTP POST", _INTERFACE_VERSION) reply_is_valid, header, body = _is_valid_reply_message(request.POST.copy()) if not reply_is_valid: log.error("Invalid xqueue object added: request_ip: {0} request.POST: {1}".format( util.get_request_ip(request), request.POST, )) statsd.increment("open_ended_assessment.grading_controller.controller.xqueue_interface.submit_message", tags=["success:Exception"]) return util._error_response('Incorrect format', _INTERFACE_VERSION) message = body['feedback'] message = util.sanitize_html(message) grader_id = body['grader_id'] submission_id = body['submission_id'] originator = body['student_info']['anonymous_student_id'] try: if 'score' in body: score = int(body['score']) else: score = None except: error_message = "Score was not an integer, received \"{0}\" instead.".format(score) log.exception(error_message) return util._error_response(error_message, _INTERFACE_VERSION) try: grade = Grader.objects.get(id=grader_id) except: error_message = "Could not find a grader object for message from xqueue" log.exception(error_message) return util._error_response(error_message, _INTERFACE_VERSION) try: submission = Submission.objects.get(id=submission_id) except: error_message = "Could not find a submission object for message from xqueue" log.exception(error_message) return util._error_response(error_message, _INTERFACE_VERSION) if grade.submission.id != submission.id: error_message = "Grader id does not match submission id that was passed in" log.exception(error_message) return util._error_response(error_message, _INTERFACE_VERSION) if originator not in [submission.student_id, grade.grader_id]: error_message = "Message originator is not the grader, or the person being graded" log.exception(error_message) return util._error_response(error_message, _INTERFACE_VERSION) if grade.grader_type in ["ML", "IN"]: recipient_type = "controller" recipient = "controller" else: recipient_type = "human" if recipient_type != 'controller': if originator == submission.student_id: recipient = grade.grader_id elif originator == grade.grader_id: recipient = submission.student_id if recipient not in [submission.student_id, grade.grader_id, 'controller']: error_message = "Message recipient is not the grader, the person being graded, or the controller" log.exception(error_message) return util._error_response(error_message, _INTERFACE_VERSION) if originator == recipient: error_message = "Message recipient is the same as originator" log.exception(error_message) return util._error_response(error_message, _INTERFACE_VERSION) message_dict = { 'grader_id': grader_id, 'originator': originator, 'submission_id': submission_id, 'message': message, 'recipient': recipient, 'message_type': "feedback", 'score': score } success, error = message_util.create_message(message_dict) if not success: return util._error_response(error, _INTERFACE_VERSION) return util._success_response({'message_id': error}, _INTERFACE_VERSION)
def pre_save(cls, sender, document, **kwargs): """Hash the plain password""" if isinstance(document.password, str): binary = document.password.encode('utf-8') document.password = bcrypt.hashpw(binary, bcrypt.gensalt(12)) document.email = sanitize_html(document.email)