def test_correct(self): result = get_auto_feedback(0.999999) self.assertIn("is correct", result) self.assertNotIn("bonus", result) result = get_auto_feedback(1) self.assertIn("is correct", result) self.assertNotIn("bonus", result) result = get_auto_feedback(1.000001) self.assertIn("is correct", result) self.assertNotIn("bonus", result)
def test_correct_with_bonus(self): result = get_auto_feedback(1.01) self.assertIn("is correct", result) self.assertIn("bonus", result) result = get_auto_feedback(2) self.assertIn("is correct", result) self.assertIn("bonus", result) result = get_auto_feedback(9.99999) self.assertIn("is correct", result) self.assertIn("bonus", result)
def test_validate_point_count_called(self): import random with mock.patch("course.page.base.validate_point_count") \ as mock_validate_point_count: mock_validate_point_count.side_effect = lambda x: x for i in range(10): correctness = random.uniform(0, 15) get_auto_feedback(correctness) mock_validate_point_count.assert_called_once_with(correctness) mock_validate_point_count.reset_mock() get_auto_feedback(correctness=None) mock_validate_point_count.assert_called_once_with(None)
def test_correctness_negative(self): correctness = -0.1 with self.assertRaises(InvalidFeedbackPointsError): get_auto_feedback(correctness)
def test_with_mostly_correct(self): self.assertIn("mostly correct", get_auto_feedback(0.51)) self.assertIn("mostly correct", get_auto_feedback(0.999))
def test_with_somewhat_correct(self): self.assertIn("somewhat correct", get_auto_feedback(0.5)) self.assertIn("somewhat correct", get_auto_feedback(0.5000001)) self.assertIn("somewhat correct", get_auto_feedback(0.001)) self.assertIn("somewhat correct", get_auto_feedback(0.2))
def test_not_correct(self): self.assertIn("not correct", get_auto_feedback(0.000001)) self.assertIn("not correct", get_auto_feedback(-0.000001))
def test_none(self): self.assertIn("No information", get_auto_feedback(None))
def grade(self, page_context, page_data, answer_data, grade_data): if answer_data is None: return AnswerFeedback(correctness=0, feedback=_("No answer provided.")) user_code = answer_data["answer"] # {{{ request run run_req = {"compile_only": False, "user_code": user_code} def transfer_attr(name): if hasattr(self.page_desc, name): run_req[name] = getattr(self.page_desc, name) transfer_attr("setup_code") transfer_attr("names_for_user") transfer_attr("names_from_user") if hasattr(self.page_desc, "test_code"): run_req["test_code"] = self.get_test_code() if hasattr(self.page_desc, "data_files"): run_req["data_files"] = {} from course.content import get_repo_blob for data_file in self.page_desc.data_files: from base64 import b64encode run_req["data_files"][data_file] = \ b64encode( get_repo_blob( page_context.repo, data_file, page_context.commit_sha).data).decode() try: response_dict = request_python_run_with_retries( run_req, run_timeout=self.page_desc.timeout) except: from traceback import format_exc response_dict = { "result": "uncaught_error", "message": "Error connecting to container", "traceback": "".join(format_exc()), } # }}} feedback_bits = [] # {{{ send email if the grading code broke if response_dict["result"] in [ "uncaught_error", "setup_compile_error", "setup_error", "test_compile_error", "test_error" ]: error_msg_parts = ["RESULT: %s" % response_dict["result"]] for key, val in sorted(response_dict.items()): if (key not in ["result", "figures"] and val and isinstance(val, six.string_types)): error_msg_parts.append( "-------------------------------------") error_msg_parts.append(key) error_msg_parts.append( "-------------------------------------") error_msg_parts.append(val) error_msg_parts.append("-------------------------------------") error_msg_parts.append("user code") error_msg_parts.append("-------------------------------------") error_msg_parts.append(user_code) error_msg_parts.append("-------------------------------------") error_msg = "\n".join(error_msg_parts) with translation.override(settings.RELATE_ADMIN_EMAIL_LOCALE): from django.template.loader import render_to_string message = render_to_string( "course/broken-code-question-email.txt", { "page_id": self.page_desc.id, "course": page_context.course, "error_message": error_msg, }) if (not page_context.in_sandbox and not is_nuisance_failure(response_dict)): try: from django.core.mail import send_mail send_mail( "".join( ["[%s] ", _("code question execution failed")]) % page_context.course.identifier, message, settings.ROBOT_EMAIL_FROM, recipient_list=[page_context.course.notify_email]) except Exception: from traceback import format_exc feedback_bits.append( six.text_type( string_concat( "<p>", _("Both the grading code and the attempt to " "notify course staff about the issue failed. " "Please contact the course or site staff and " "inform them of this issue, mentioning this " "entire error message:"), "</p>", "<p>", _("Sending an email to the course staff about the " "following failure failed with " "the following error message:"), "<pre>", "".join(format_exc()), "</pre>", _("The original failure message follows:"), "</p>"))) # }}} from relate.utils import dict_to_struct response = dict_to_struct(response_dict) bulk_feedback_bits = [] if hasattr(response, "points"): correctness = response.points feedback_bits.append("<p><b>%s</b></p>" % get_auto_feedback(correctness)) else: correctness = None if response.result == "success": pass elif response.result in [ "uncaught_error", "setup_compile_error", "setup_error", "test_compile_error", "test_error" ]: feedback_bits.append("".join([ "<p>", _("The grading code failed. Sorry about that. " "The staff has been informed, and if this problem is " "due to an issue with the grading code, " "it will be fixed as soon as possible. " "In the meantime, you'll see a traceback " "below that may help you figure out what went wrong."), "</p>" ])) elif response.result == "timeout": feedback_bits.append("".join([ "<p>", _("Your code took too long to execute. The problem " "specifies that your code may take at most %s seconds " "to run. " "It took longer than that and was aborted."), "</p>" ]) % self.page_desc.timeout) correctness = 0 elif response.result == "user_compile_error": feedback_bits.append("".join([ "<p>", _("Your code failed to compile. An error message is " "below."), "</p>" ])) correctness = 0 elif response.result == "user_error": feedback_bits.append("".join([ "<p>", _("Your code failed with an exception. " "A traceback is below."), "</p>" ])) correctness = 0 else: raise RuntimeError("invalid runpy result: %s" % response.result) if hasattr(response, "feedback") and response.feedback: feedback_bits.append("".join([ "<p>", _("Here is some feedback on your code"), ":" "<ul>%s</ul></p>" ]) % "".join("<li>%s</li>" % escape(fb_item) for fb_item in response.feedback)) if hasattr(response, "traceback") and response.traceback: feedback_bits.append("".join([ "<p>", _("This is the exception traceback"), ":" "<pre>%s</pre></p>" ]) % escape(response.traceback)) if hasattr(response, "exec_host") and response.exec_host != "localhost": import socket try: exec_host_name, dummy, dummy = socket.gethostbyaddr( response.exec_host) except socket.error: exec_host_name = response.exec_host feedback_bits.append("".join( ["<p>", _("Your code ran on %s.") % exec_host_name, "</p>"])) if hasattr(response, "stdout") and response.stdout: bulk_feedback_bits.append("".join([ "<p>", _("Your code printed the following output"), ":" "<pre>%s</pre></p>" ]) % escape(response.stdout)) if hasattr(response, "stderr") and response.stderr: bulk_feedback_bits.append("".join([ "<p>", _("Your code printed the following error messages"), ":" "<pre>%s</pre></p>" ]) % escape(response.stderr)) if hasattr(response, "figures") and response.figures: fig_lines = [ "".join([ "<p>", _("Your code produced the following plots"), ":</p>" ]), '<dl class="result-figure-list">', ] for nr, mime_type, b64data in response.figures: fig_lines.extend([ "".join(["<dt>", _("Figure"), "%d<dt>"]) % nr, '<dd><img alt="Figure %d" src="data:%s;base64,%s"></dd>' % (nr, mime_type, b64data) ]) fig_lines.append("</dl>") bulk_feedback_bits.extend(fig_lines) return AnswerFeedback(correctness=correctness, feedback="\n".join(feedback_bits), bulk_feedback="\n".join(bulk_feedback_bits))
def grade(self, page_context, page_data, answer_data, grade_data): if answer_data is None: return AnswerFeedback(correctness=0, feedback=_("No answer provided.")) user_code = answer_data["answer"] # {{{ request run run_req = {"compile_only": False, "user_code": user_code} def transfer_attr(name): if hasattr(self.page_desc, name): run_req[name] = getattr(self.page_desc, name) transfer_attr("setup_code") transfer_attr("names_for_user") transfer_attr("names_from_user") run_req["test_code"] = self.get_test_code() if hasattr(self.page_desc, "data_files"): run_req["data_files"] = {} from course.content import get_repo_blob for data_file in self.page_desc.data_files: from base64 import b64encode run_req["data_files"][data_file] = \ b64encode( get_repo_blob( page_context.repo, data_file, page_context.commit_sha).data).decode() try: response_dict = request_run_with_retries( run_req, run_timeout=self.page_desc.timeout, image=self.container_image) except Exception: from traceback import format_exc response_dict = { "result": "uncaught_error", "message": "Error connecting to container", "traceback": "".join(format_exc()), } # }}} feedback_bits = [] correctness = None if "points" in response_dict: correctness = response_dict["points"] try: feedback_bits.append("<p><b>%s</b></p>" % _(get_auto_feedback(correctness))) except Exception as e: correctness = None response_dict["result"] = "setup_error" response_dict["message"] = ("%s: %s" % (type(e).__name__, str(e))) # {{{ send email if the grading code broke if response_dict["result"] in [ "uncaught_error", "setup_compile_error", "setup_error", "test_compile_error", "test_error" ]: error_msg_parts = ["RESULT: %s" % response_dict["result"]] for key, val in sorted(response_dict.items()): if (key not in ["result", "figures"] and val and isinstance(val, str)): error_msg_parts.append( "-------------------------------------") error_msg_parts.append(key) error_msg_parts.append( "-------------------------------------") error_msg_parts.append(val) error_msg_parts.append("-------------------------------------") error_msg_parts.append("user code") error_msg_parts.append("-------------------------------------") error_msg_parts.append(user_code) error_msg_parts.append("-------------------------------------") error_msg = "\n".join(error_msg_parts) from relate.utils import local_now, format_datetime_local from course.utils import LanguageOverride with LanguageOverride(page_context.course): from relate.utils import render_email_template message = render_email_template( "course/broken-code-question-email.txt", { "site": getattr(settings, "RELATE_BASE_URL"), "page_id": self.page_desc.id, "course": page_context.course, "error_message": error_msg, "review_uri": page_context.page_uri, "time": format_datetime_local(local_now()) }) if (not page_context.in_sandbox and not is_nuisance_failure(response_dict)): try: from django.core.mail import EmailMessage msg = EmailMessage( "".join([ "[%s:%s] ", _("code question execution failed") ]) % (page_context.course.identifier, page_context.flow_session.flow_id if page_context.flow_session is not None else _("<unknown flow>")), message, settings.ROBOT_EMAIL_FROM, [page_context.course.notify_email]) from relate.utils import get_outbound_mail_connection msg.connection = get_outbound_mail_connection("robot") msg.send() except Exception: from traceback import format_exc feedback_bits.append( str( string_concat( "<p>", _("Both the grading code and the attempt to " "notify course staff about the issue failed. " "Please contact the course or site staff and " "inform them of this issue, mentioning this " "entire error message:"), "</p>", "<p>", _("Sending an email to the course staff about the " "following failure failed with " "the following error message:"), "<pre>", "".join(format_exc()), "</pre>", _("The original failure message follows:"), "</p>"))) # }}} if hasattr(self.page_desc, "correct_code"): def normalize_code(s): return (s.replace(" ", "").replace("\r", "").replace( "\n", "").replace("\t", "")) if (normalize_code(user_code) == normalize_code( self.page_desc.correct_code)): feedback_bits.append( "<p><b>%s</b></p>" % _("It looks like you submitted code that is identical to " "the reference solution. This is not allowed.")) from relate.utils import dict_to_struct response = dict_to_struct(response_dict) bulk_feedback_bits = [] if response.result == "success": pass elif response.result in [ "uncaught_error", "setup_compile_error", "setup_error", "test_compile_error", "test_error" ]: feedback_bits.append("".join([ "<p>", _("The grading code failed. Sorry about that. " "The staff has been informed, and if this problem is " "due to an issue with the grading code, " "it will be fixed as soon as possible. " "In the meantime, you'll see a traceback " "below that may help you figure out what went wrong."), "</p>" ])) elif response.result == "timeout": feedback_bits.append("".join([ "<p>", _("Your code took too long to execute. The problem " "specifies that your code may take at most %s seconds " "to run. " "It took longer than that and was aborted."), "</p>" ]) % self.page_desc.timeout) correctness = 0 elif response.result == "user_compile_error": feedback_bits.append("".join([ "<p>", _("Your code failed to compile. An error message is " "below."), "</p>" ])) correctness = 0 elif response.result == "user_error": feedback_bits.append("".join([ "<p>", _("Your code failed with an exception. " "A traceback is below."), "</p>" ])) correctness = 0 else: raise RuntimeError("invalid run result: %s" % response.result) if hasattr(response, "feedback") and response.feedback: def sanitize(s): import bleach return bleach.clean(s, tags=["p", "pre"]) feedback_bits.append("".join([ "<p>", _("Here is some feedback on your code"), ":" "<ul>%s</ul></p>" ]) % "".join("<li>%s</li>" % sanitize(fb_item) for fb_item in response.feedback)) if hasattr(response, "traceback") and response.traceback: feedback_bits.append("".join([ "<p>", _("This is the exception traceback"), ":" "<pre>%s</pre></p>" ]) % escape(response.traceback)) if hasattr(response, "exec_host") and response.exec_host != "localhost": import socket try: exec_host_name, dummy, dummy = socket.gethostbyaddr( response.exec_host) except socket.error: exec_host_name = response.exec_host feedback_bits.append("".join( ["<p>", _("Your code ran on %s.") % exec_host_name, "</p>"])) if hasattr(response, "stdout") and response.stdout: bulk_feedback_bits.append("".join([ "<p>", _("Your code printed the following output"), ":" "<pre>%s</pre></p>" ]) % escape(response.stdout)) if hasattr(response, "stderr") and response.stderr: bulk_feedback_bits.append("".join([ "<p>", _("Your code printed the following error messages"), ":" "<pre>%s</pre></p>" ]) % escape(response.stderr)) if hasattr(response, "figures") and response.figures: fig_lines = [ "".join([ "<p>", _("Your code produced the following plots"), ":</p>" ]), '<dl class="result-figure-list">', ] for nr, mime_type, b64data in response.figures: if mime_type in ["image/jpeg", "image/png"]: fig_lines.extend([ "".join(["<dt>", _("Figure"), "%d<dt>"]) % nr, '<dd><img alt="Figure %d" src="data:%s;base64,%s"></dd>' % (nr, mime_type, b64data) ]) fig_lines.append("</dl>") bulk_feedback_bits.extend(fig_lines) # {{{ html output / santization if hasattr(response, "html") and response.html: def is_allowed_data_uri(allowed_mimetypes, uri): import re m = re.match(r"^data:([-a-z0-9]+/[-a-z0-9]+);base64,", uri) if not m: return False mimetype = m.group(1) return mimetype in allowed_mimetypes def sanitize(s): import bleach def filter_audio_attributes(tag, name, value): if name in ["controls"]: return True else: return False def filter_source_attributes(tag, name, value): if name in ["type"]: return True elif name == "src": if is_allowed_data_uri([ "audio/wav", ], value): return bleach.sanitizer.VALUE_SAFE else: return False else: return False def filter_img_attributes(tag, name, value): if name in ["alt", "title"]: return True elif name == "src": return is_allowed_data_uri([ "image/png", "image/jpeg", ], value) else: return False if not isinstance(s, str): return _("(Non-string in 'HTML' output filtered out)") return bleach.clean(s, tags=bleach.ALLOWED_TAGS + ["audio", "video", "source"], attributes={ "audio": filter_audio_attributes, "source": filter_source_attributes, "img": filter_img_attributes, }) bulk_feedback_bits.extend( sanitize(snippet) for snippet in response.html) # }}} return AnswerFeedback(correctness=correctness, feedback="\n".join(feedback_bits), bulk_feedback="\n".join(bulk_feedback_bits))
def test_correctness_exceed_max_extra_credit_factor(self): correctness = MAX_EXTRA_CREDIT_FACTOR + 0.1 with self.assertRaises(InvalidFeedbackPointsError): get_auto_feedback(correctness)
def grade(self, page_context, page_data, answer_data, grade_data): if answer_data is None: return AnswerFeedback(correctness=0, feedback=_("No answer provided.")) user_code = answer_data["answer"] # {{{ request run run_req = {"compile_only": False, "user_code": user_code} def transfer_attr(name): if hasattr(self.page_desc, name): run_req[name] = getattr(self.page_desc, name) transfer_attr("setup_code") transfer_attr("names_for_user") transfer_attr("names_from_user") if hasattr(self.page_desc, "test_code"): run_req["test_code"] = self.get_test_code() if hasattr(self.page_desc, "data_files"): run_req["data_files"] = {} from course.content import get_repo_blob for data_file in self.page_desc.data_files: from base64 import b64encode run_req["data_files"][data_file] = b64encode( get_repo_blob(page_context.repo, data_file, page_context.commit_sha).data ).decode() try: response_dict = request_python_run_with_retries(run_req, run_timeout=self.page_desc.timeout) except: from traceback import format_exc response_dict = { "result": "uncaught_error", "message": "Error connecting to container", "traceback": "".join(format_exc()), } # }}} feedback_bits = [] # {{{ send email if the grading code broke if response_dict["result"] in [ "uncaught_error", "setup_compile_error", "setup_error", "test_compile_error", "test_error", ]: error_msg_parts = ["RESULT: %s" % response_dict["result"]] for key, val in sorted(response_dict.items()): if key not in ["result", "figures"] and val and isinstance(val, six.string_types): error_msg_parts.append("-------------------------------------") error_msg_parts.append(key) error_msg_parts.append("-------------------------------------") error_msg_parts.append(val) error_msg_parts.append("-------------------------------------") error_msg_parts.append("user code") error_msg_parts.append("-------------------------------------") error_msg_parts.append(user_code) error_msg_parts.append("-------------------------------------") error_msg = "\n".join(error_msg_parts) with translation.override(settings.RELATE_ADMIN_EMAIL_LOCALE): from django.template.loader import render_to_string message = render_to_string( "course/broken-code-question-email.txt", {"page_id": self.page_desc.id, "course": page_context.course, "error_message": error_msg}, ) if not page_context.in_sandbox and not is_nuisance_failure(response_dict): try: from django.core.mail import send_mail send_mail( "".join(["[%s] ", _("code question execution failed")]) % page_context.course.identifier, message, settings.ROBOT_EMAIL_FROM, recipient_list=[page_context.course.notify_email], ) except Exception: from traceback import format_exc feedback_bits.append( six.text_type( string_concat( "<p>", _( "Both the grading code and the attempt to " "notify course staff about the issue failed. " "Please contact the course or site staff and " "inform them of this issue, mentioning this " "entire error message:" ), "</p>", "<p>", _( "Sending an email to the course staff about the " "following failure failed with " "the following error message:" ), "<pre>", "".join(format_exc()), "</pre>", _("The original failure message follows:"), "</p>", ) ) ) # }}} from relate.utils import dict_to_struct response = dict_to_struct(response_dict) bulk_feedback_bits = [] if hasattr(response, "points"): correctness = response.points feedback_bits.append("<p><b>%s</b></p>" % get_auto_feedback(correctness)) else: correctness = None if response.result == "success": pass elif response.result in [ "uncaught_error", "setup_compile_error", "setup_error", "test_compile_error", "test_error", ]: feedback_bits.append( "".join( [ "<p>", _( "The grading code failed. Sorry about that. " "The staff has been informed, and if this problem is " "due to an issue with the grading code, " "it will be fixed as soon as possible. " "In the meantime, you'll see a traceback " "below that may help you figure out what went wrong." ), "</p>", ] ) ) elif response.result == "timeout": feedback_bits.append( "".join( [ "<p>", _( "Your code took too long to execute. The problem " "specifies that your code may take at most %s seconds " "to run. " "It took longer than that and was aborted." ), "</p>", ] ) % self.page_desc.timeout ) correctness = 0 elif response.result == "user_compile_error": feedback_bits.append( "".join(["<p>", _("Your code failed to compile. An error message is " "below."), "</p>"]) ) correctness = 0 elif response.result == "user_error": feedback_bits.append( "".join(["<p>", _("Your code failed with an exception. " "A traceback is below."), "</p>"]) ) correctness = 0 else: raise RuntimeError("invalid runpy result: %s" % response.result) if hasattr(response, "feedback") and response.feedback: feedback_bits.append( "".join(["<p>", _("Here is some feedback on your code"), ":" "<ul>%s</ul></p>"]) % "".join("<li>%s</li>" % escape(fb_item) for fb_item in response.feedback) ) if hasattr(response, "traceback") and response.traceback: feedback_bits.append( "".join(["<p>", _("This is the exception traceback"), ":" "<pre>%s</pre></p>"]) % escape(response.traceback) ) if hasattr(response, "exec_host") and response.exec_host != "localhost": import socket try: exec_host_name, dummy, dummy = socket.gethostbyaddr(response.exec_host) except socket.error: exec_host_name = response.exec_host feedback_bits.append("".join(["<p>", _("Your code ran on %s.") % exec_host_name, "</p>"])) if hasattr(response, "stdout") and response.stdout: bulk_feedback_bits.append( "".join(["<p>", _("Your code printed the following output"), ":" "<pre>%s</pre></p>"]) % escape(response.stdout) ) if hasattr(response, "stderr") and response.stderr: bulk_feedback_bits.append( "".join(["<p>", _("Your code printed the following error messages"), ":" "<pre>%s</pre></p>"]) % escape(response.stderr) ) if hasattr(response, "figures") and response.figures: fig_lines = [ "".join(["<p>", _("Your code produced the following plots"), ":</p>"]), '<dl class="result-figure-list">', ] for nr, mime_type, b64data in response.figures: fig_lines.extend( [ "".join(["<dt>", _("Figure"), "%d<dt>"]) % nr, '<dd><img alt="Figure %d" src="data:%s;base64,%s"></dd>' % (nr, mime_type, b64data), ] ) fig_lines.append("</dl>") bulk_feedback_bits.extend(fig_lines) return AnswerFeedback( correctness=correctness, feedback="\n".join(feedback_bits), bulk_feedback="\n".join(bulk_feedback_bits) )
def grade(self, page_context, page_data, answer_data, grade_data): if answer_data is None: return AnswerFeedback(correctness=0, feedback=_("No answer provided.")) user_code = answer_data["answer"] # {{{ request run run_req = {"compile_only": False, "user_code": user_code} def transfer_attr(name): if hasattr(self.page_desc, name): run_req[name] = getattr(self.page_desc, name) transfer_attr("setup_code") transfer_attr("names_for_user") transfer_attr("names_from_user") run_req["test_code"] = self.get_test_code() if hasattr(self.page_desc, "data_files"): run_req["data_files"] = {} from course.content import get_repo_blob for data_file in self.page_desc.data_files: from base64 import b64encode run_req["data_files"][data_file] = \ b64encode( get_repo_blob( page_context.repo, data_file, page_context.commit_sha).data).decode() try: response_dict = request_python_run_with_retries(run_req, run_timeout=self.page_desc.timeout) except Exception: from traceback import format_exc response_dict = { "result": "uncaught_error", "message": "Error connecting to container", "traceback": "".join(format_exc()), } # }}} feedback_bits = [] correctness = None if "points" in response_dict: correctness = response_dict["points"] try: feedback_bits.append( "<p><b>%s</b></p>" % _(get_auto_feedback(correctness))) except Exception as e: correctness = None response_dict["result"] = "setup_error" response_dict["message"] = ( "%s: %s" % (type(e).__name__, str(e)) ) # {{{ send email if the grading code broke if response_dict["result"] in [ "uncaught_error", "setup_compile_error", "setup_error", "test_compile_error", "test_error"]: error_msg_parts = ["RESULT: %s" % response_dict["result"]] for key, val in sorted(response_dict.items()): if (key not in ["result", "figures"] and val and isinstance(val, six.string_types)): error_msg_parts.append("-------------------------------------") error_msg_parts.append(key) error_msg_parts.append("-------------------------------------") error_msg_parts.append(val) error_msg_parts.append("-------------------------------------") error_msg_parts.append("user code") error_msg_parts.append("-------------------------------------") error_msg_parts.append(user_code) error_msg_parts.append("-------------------------------------") error_msg = "\n".join(error_msg_parts) from relate.utils import local_now, format_datetime_local from course.utils import LanguageOverride with LanguageOverride(page_context.course): from relate.utils import render_email_template message = render_email_template( "course/broken-code-question-email.txt", { "site": getattr(settings, "RELATE_BASE_URL"), "page_id": self.page_desc.id, "course": page_context.course, "error_message": error_msg, "review_uri": page_context.page_uri, "time": format_datetime_local(local_now()) }) if ( not page_context.in_sandbox and not is_nuisance_failure(response_dict)): try: from django.core.mail import EmailMessage msg = EmailMessage("".join(["[%s:%s] ", _("code question execution failed")]) % ( page_context.course.identifier, page_context.flow_session.flow_id if page_context.flow_session is not None else _("<unknown flow>")), message, settings.ROBOT_EMAIL_FROM, [page_context.course.notify_email]) from relate.utils import get_outbound_mail_connection msg.connection = get_outbound_mail_connection("robot") msg.send() except Exception: from traceback import format_exc feedback_bits.append( six.text_type(string_concat( "<p>", _( "Both the grading code and the attempt to " "notify course staff about the issue failed. " "Please contact the course or site staff and " "inform them of this issue, mentioning this " "entire error message:"), "</p>", "<p>", _( "Sending an email to the course staff about the " "following failure failed with " "the following error message:"), "<pre>", "".join(format_exc()), "</pre>", _("The original failure message follows:"), "</p>"))) # }}} if hasattr(self.page_desc, "correct_code"): def normalize_code(s): return (s .replace(" ", "") .replace("\r", "") .replace("\n", "") .replace("\t", "")) if (normalize_code(user_code) == normalize_code(self.page_desc.correct_code)): feedback_bits.append( "<p><b>%s</b></p>" % _("It looks like you submitted code that is identical to " "the reference solution. This is not allowed.")) from relate.utils import dict_to_struct response = dict_to_struct(response_dict) bulk_feedback_bits = [] if response.result == "success": pass elif response.result in [ "uncaught_error", "setup_compile_error", "setup_error", "test_compile_error", "test_error"]: feedback_bits.append("".join([ "<p>", _( "The grading code failed. Sorry about that. " "The staff has been informed, and if this problem is " "due to an issue with the grading code, " "it will be fixed as soon as possible. " "In the meantime, you'll see a traceback " "below that may help you figure out what went wrong." ), "</p>"])) elif response.result == "timeout": feedback_bits.append("".join([ "<p>", _( "Your code took too long to execute. The problem " "specifies that your code may take at most %s seconds " "to run. " "It took longer than that and was aborted." ), "</p>"]) % self.page_desc.timeout) correctness = 0 elif response.result == "user_compile_error": feedback_bits.append("".join([ "<p>", _("Your code failed to compile. An error message is " "below."), "</p>"])) correctness = 0 elif response.result == "user_error": feedback_bits.append("".join([ "<p>", _("Your code failed with an exception. " "A traceback is below."), "</p>"])) correctness = 0 else: raise RuntimeError("invalid runpy result: %s" % response.result) if hasattr(response, "feedback") and response.feedback: def sanitize(s): import bleach return bleach.clean(s, tags=["p", "pre"]) feedback_bits.append("".join([ "<p>", _("Here is some feedback on your code"), ":" "<ul>%s</ul></p>"]) % "".join( "<li>%s</li>" % sanitize(fb_item) for fb_item in response.feedback)) if hasattr(response, "traceback") and response.traceback: feedback_bits.append("".join([ "<p>", _("This is the exception traceback"), ":" "<pre>%s</pre></p>"]) % escape(response.traceback)) if hasattr(response, "exec_host") and response.exec_host != "localhost": import socket try: exec_host_name, dummy, dummy = socket.gethostbyaddr( response.exec_host) except socket.error: exec_host_name = response.exec_host feedback_bits.append("".join([ "<p>", _("Your code ran on %s.") % exec_host_name, "</p>"])) if hasattr(response, "stdout") and response.stdout: bulk_feedback_bits.append("".join([ "<p>", _("Your code printed the following output"), ":" "<pre>%s</pre></p>"]) % escape(response.stdout)) if hasattr(response, "stderr") and response.stderr: bulk_feedback_bits.append("".join([ "<p>", _("Your code printed the following error messages"), ":" "<pre>%s</pre></p>"]) % escape(response.stderr)) if hasattr(response, "figures") and response.figures: fig_lines = ["".join([ "<p>", _("Your code produced the following plots"), ":</p>"]), '<dl class="result-figure-list">', ] for nr, mime_type, b64data in response.figures: if mime_type in ["image/jpeg", "image/png"]: fig_lines.extend([ "".join([ "<dt>", _("Figure"), "%d<dt>"]) % nr, '<dd><img alt="Figure %d" src="data:%s;base64,%s"></dd>' % (nr, mime_type, b64data)]) fig_lines.append("</dl>") bulk_feedback_bits.extend(fig_lines) # {{{ html output / santization if hasattr(response, "html") and response.html: def is_allowed_data_uri(allowed_mimetypes, uri): import re m = re.match(r"^data:([-a-z0-9]+/[-a-z0-9]+);base64,", uri) if not m: return False mimetype = m.group(1) return mimetype in allowed_mimetypes def sanitize(s): import bleach def filter_audio_attributes(tag, name, value): if name in ["controls"]: return True else: return False def filter_source_attributes(tag, name, value): if name in ["type"]: return True elif name == "src": if is_allowed_data_uri([ "audio/wav", ], value): return bleach.sanitizer.VALUE_SAFE else: return False else: return False def filter_img_attributes(tag, name, value): if name in ["alt", "title"]: return True elif name == "src": return is_allowed_data_uri([ "image/png", "image/jpeg", ], value) else: return False if not isinstance(s, six.text_type): return _("(Non-string in 'HTML' output filtered out)") return bleach.clean(s, tags=bleach.ALLOWED_TAGS + ["audio", "video", "source"], attributes={ "audio": filter_audio_attributes, "source": filter_source_attributes, "img": filter_img_attributes, }) bulk_feedback_bits.extend( sanitize(snippet) for snippet in response.html) # }}} return AnswerFeedback( correctness=correctness, feedback="\n".join(feedback_bits), bulk_feedback="\n".join(bulk_feedback_bits))