def test_correct(self):
result = get_auto_feedback(0.999999)
self.assertIn("is correct", result)
self.assertNotIn("bonus", result)
result = get_auto_feedback(1)
self.assertIn("is correct", result)
self.assertNotIn("bonus", result)
result = get_auto_feedback(1.000001)
self.assertIn("is correct", result)
self.assertNotIn("bonus", result)
def test_correct(self):
result = get_auto_feedback(0.999999)
self.assertIn("is correct", result)
self.assertNotIn("bonus", result)
result = get_auto_feedback(1)
self.assertIn("is correct", result)
self.assertNotIn("bonus", result)
result = get_auto_feedback(1.000001)
self.assertIn("is correct", result)
self.assertNotIn("bonus", result)
def test_correct_with_bonus(self):
result = get_auto_feedback(1.01)
self.assertIn("is correct", result)
self.assertIn("bonus", result)
result = get_auto_feedback(2)
self.assertIn("is correct", result)
self.assertIn("bonus", result)
result = get_auto_feedback(9.99999)
self.assertIn("is correct", result)
self.assertIn("bonus", result)
def test_correct_with_bonus(self):
result = get_auto_feedback(1.01)
self.assertIn("is correct", result)
self.assertIn("bonus", result)
result = get_auto_feedback(2)
self.assertIn("is correct", result)
self.assertIn("bonus", result)
result = get_auto_feedback(9.99999)
self.assertIn("is correct", result)
self.assertIn("bonus", result)
def test_validate_point_count_called(self):
import random
with mock.patch("course.page.base.validate_point_count") \
as mock_validate_point_count:
mock_validate_point_count.side_effect = lambda x: x
for i in range(10):
correctness = random.uniform(0, 15)
get_auto_feedback(correctness)
mock_validate_point_count.assert_called_once_with(correctness)
mock_validate_point_count.reset_mock()
get_auto_feedback(correctness=None)
mock_validate_point_count.assert_called_once_with(None)
def test_validate_point_count_called(self):
import random
with mock.patch("course.page.base.validate_point_count") \
as mock_validate_point_count:
mock_validate_point_count.side_effect = lambda x: x
for i in range(10):
correctness = random.uniform(0, 15)
get_auto_feedback(correctness)
mock_validate_point_count.assert_called_once_with(correctness)
mock_validate_point_count.reset_mock()
get_auto_feedback(correctness=None)
mock_validate_point_count.assert_called_once_with(None)
def test_correctness_negative(self):
correctness = -0.1
with self.assertRaises(InvalidFeedbackPointsError):
get_auto_feedback(correctness)
def test_with_mostly_correct(self):
self.assertIn("mostly correct", get_auto_feedback(0.51))
self.assertIn("mostly correct", get_auto_feedback(0.999))
def test_with_somewhat_correct(self):
self.assertIn("somewhat correct", get_auto_feedback(0.5))
self.assertIn("somewhat correct", get_auto_feedback(0.5000001))
self.assertIn("somewhat correct", get_auto_feedback(0.001))
self.assertIn("somewhat correct", get_auto_feedback(0.2))
def test_not_correct(self):
self.assertIn("not correct", get_auto_feedback(0.000001))
self.assertIn("not correct", get_auto_feedback(-0.000001))
def test_none(self):
self.assertIn("No information", get_auto_feedback(None))
def test_none(self):
self.assertIn("No information", get_auto_feedback(None))
def test_not_correct(self):
self.assertIn("not correct", get_auto_feedback(0.000001))
self.assertIn("not correct", get_auto_feedback(-0.000001))
def grade(self, page_context, page_data, answer_data, grade_data):
if answer_data is None:
return AnswerFeedback(correctness=0,
feedback=_("No answer provided."))
user_code = answer_data["answer"]
# {{{ request run
run_req = {"compile_only": False, "user_code": user_code}
def transfer_attr(name):
if hasattr(self.page_desc, name):
run_req[name] = getattr(self.page_desc, name)
transfer_attr("setup_code")
transfer_attr("names_for_user")
transfer_attr("names_from_user")
if hasattr(self.page_desc, "test_code"):
run_req["test_code"] = self.get_test_code()
if hasattr(self.page_desc, "data_files"):
run_req["data_files"] = {}
from course.content import get_repo_blob
for data_file in self.page_desc.data_files:
from base64 import b64encode
run_req["data_files"][data_file] = \
b64encode(
get_repo_blob(
page_context.repo, data_file,
page_context.commit_sha).data).decode()
try:
response_dict = request_python_run_with_retries(
run_req, run_timeout=self.page_desc.timeout)
except:
from traceback import format_exc
response_dict = {
"result": "uncaught_error",
"message": "Error connecting to container",
"traceback": "".join(format_exc()),
}
# }}}
feedback_bits = []
# {{{ send email if the grading code broke
if response_dict["result"] in [
"uncaught_error", "setup_compile_error", "setup_error",
"test_compile_error", "test_error"
]:
error_msg_parts = ["RESULT: %s" % response_dict["result"]]
for key, val in sorted(response_dict.items()):
if (key not in ["result", "figures"] and val
and isinstance(val, six.string_types)):
error_msg_parts.append(
"-------------------------------------")
error_msg_parts.append(key)
error_msg_parts.append(
"-------------------------------------")
error_msg_parts.append(val)
error_msg_parts.append("-------------------------------------")
error_msg_parts.append("user code")
error_msg_parts.append("-------------------------------------")
error_msg_parts.append(user_code)
error_msg_parts.append("-------------------------------------")
error_msg = "\n".join(error_msg_parts)
with translation.override(settings.RELATE_ADMIN_EMAIL_LOCALE):
from django.template.loader import render_to_string
message = render_to_string(
"course/broken-code-question-email.txt", {
"page_id": self.page_desc.id,
"course": page_context.course,
"error_message": error_msg,
})
if (not page_context.in_sandbox
and not is_nuisance_failure(response_dict)):
try:
from django.core.mail import send_mail
send_mail(
"".join(
["[%s] ",
_("code question execution failed")]) %
page_context.course.identifier,
message,
settings.ROBOT_EMAIL_FROM,
recipient_list=[page_context.course.notify_email])
except Exception:
from traceback import format_exc
feedback_bits.append(
six.text_type(
string_concat(
"<p>",
_("Both the grading code and the attempt to "
"notify course staff about the issue failed. "
"Please contact the course or site staff and "
"inform them of this issue, mentioning this "
"entire error message:"), "</p>", "<p>",
_("Sending an email to the course staff about the "
"following failure failed with "
"the following error message:"), "<pre>",
"".join(format_exc()), "</pre>",
_("The original failure message follows:"),
"</p>")))
# }}}
from relate.utils import dict_to_struct
response = dict_to_struct(response_dict)
bulk_feedback_bits = []
if hasattr(response, "points"):
correctness = response.points
feedback_bits.append("<p><b>%s</b></p>" %
get_auto_feedback(correctness))
else:
correctness = None
if response.result == "success":
pass
elif response.result in [
"uncaught_error", "setup_compile_error", "setup_error",
"test_compile_error", "test_error"
]:
feedback_bits.append("".join([
"<p>",
_("The grading code failed. Sorry about that. "
"The staff has been informed, and if this problem is "
"due to an issue with the grading code, "
"it will be fixed as soon as possible. "
"In the meantime, you'll see a traceback "
"below that may help you figure out what went wrong."),
"</p>"
]))
elif response.result == "timeout":
feedback_bits.append("".join([
"<p>",
_("Your code took too long to execute. The problem "
"specifies that your code may take at most %s seconds "
"to run. "
"It took longer than that and was aborted."), "</p>"
]) % self.page_desc.timeout)
correctness = 0
elif response.result == "user_compile_error":
feedback_bits.append("".join([
"<p>",
_("Your code failed to compile. An error message is "
"below."), "</p>"
]))
correctness = 0
elif response.result == "user_error":
feedback_bits.append("".join([
"<p>",
_("Your code failed with an exception. "
"A traceback is below."), "</p>"
]))
correctness = 0
else:
raise RuntimeError("invalid runpy result: %s" % response.result)
if hasattr(response, "feedback") and response.feedback:
feedback_bits.append("".join([
"<p>",
_("Here is some feedback on your code"), ":"
"<ul>%s</ul></p>"
]) % "".join("<li>%s</li>" % escape(fb_item)
for fb_item in response.feedback))
if hasattr(response, "traceback") and response.traceback:
feedback_bits.append("".join([
"<p>",
_("This is the exception traceback"), ":"
"<pre>%s</pre></p>"
]) % escape(response.traceback))
if hasattr(response,
"exec_host") and response.exec_host != "localhost":
import socket
try:
exec_host_name, dummy, dummy = socket.gethostbyaddr(
response.exec_host)
except socket.error:
exec_host_name = response.exec_host
feedback_bits.append("".join(
["<p>",
_("Your code ran on %s.") % exec_host_name, "</p>"]))
if hasattr(response, "stdout") and response.stdout:
bulk_feedback_bits.append("".join([
"<p>",
_("Your code printed the following output"), ":"
"<pre>%s</pre></p>"
]) % escape(response.stdout))
if hasattr(response, "stderr") and response.stderr:
bulk_feedback_bits.append("".join([
"<p>",
_("Your code printed the following error messages"), ":"
"<pre>%s</pre></p>"
]) % escape(response.stderr))
if hasattr(response, "figures") and response.figures:
fig_lines = [
"".join([
"<p>",
_("Your code produced the following plots"), ":</p>"
]),
'<dl class="result-figure-list">',
]
for nr, mime_type, b64data in response.figures:
fig_lines.extend([
"".join(["<dt>", _("Figure"), "%d<dt>"]) % nr,
'<dd><img alt="Figure %d" src="data:%s;base64,%s"></dd>' %
(nr, mime_type, b64data)
])
fig_lines.append("</dl>")
bulk_feedback_bits.extend(fig_lines)
return AnswerFeedback(correctness=correctness,
feedback="\n".join(feedback_bits),
bulk_feedback="\n".join(bulk_feedback_bits))
def grade(self, page_context, page_data, answer_data, grade_data):
if answer_data is None:
return AnswerFeedback(correctness=0,
feedback=_("No answer provided."))
user_code = answer_data["answer"]
# {{{ request run
run_req = {"compile_only": False, "user_code": user_code}
def transfer_attr(name):
if hasattr(self.page_desc, name):
run_req[name] = getattr(self.page_desc, name)
transfer_attr("setup_code")
transfer_attr("names_for_user")
transfer_attr("names_from_user")
run_req["test_code"] = self.get_test_code()
if hasattr(self.page_desc, "data_files"):
run_req["data_files"] = {}
from course.content import get_repo_blob
for data_file in self.page_desc.data_files:
from base64 import b64encode
run_req["data_files"][data_file] = \
b64encode(
get_repo_blob(
page_context.repo, data_file,
page_context.commit_sha).data).decode()
try:
response_dict = request_run_with_retries(
run_req,
run_timeout=self.page_desc.timeout,
image=self.container_image)
except Exception:
from traceback import format_exc
response_dict = {
"result": "uncaught_error",
"message": "Error connecting to container",
"traceback": "".join(format_exc()),
}
# }}}
feedback_bits = []
correctness = None
if "points" in response_dict:
correctness = response_dict["points"]
try:
feedback_bits.append("<p><b>%s</b></p>" %
_(get_auto_feedback(correctness)))
except Exception as e:
correctness = None
response_dict["result"] = "setup_error"
response_dict["message"] = ("%s: %s" %
(type(e).__name__, str(e)))
# {{{ send email if the grading code broke
if response_dict["result"] in [
"uncaught_error", "setup_compile_error", "setup_error",
"test_compile_error", "test_error"
]:
error_msg_parts = ["RESULT: %s" % response_dict["result"]]
for key, val in sorted(response_dict.items()):
if (key not in ["result", "figures"] and val
and isinstance(val, str)):
error_msg_parts.append(
"-------------------------------------")
error_msg_parts.append(key)
error_msg_parts.append(
"-------------------------------------")
error_msg_parts.append(val)
error_msg_parts.append("-------------------------------------")
error_msg_parts.append("user code")
error_msg_parts.append("-------------------------------------")
error_msg_parts.append(user_code)
error_msg_parts.append("-------------------------------------")
error_msg = "\n".join(error_msg_parts)
from relate.utils import local_now, format_datetime_local
from course.utils import LanguageOverride
with LanguageOverride(page_context.course):
from relate.utils import render_email_template
message = render_email_template(
"course/broken-code-question-email.txt", {
"site": getattr(settings, "RELATE_BASE_URL"),
"page_id": self.page_desc.id,
"course": page_context.course,
"error_message": error_msg,
"review_uri": page_context.page_uri,
"time": format_datetime_local(local_now())
})
if (not page_context.in_sandbox
and not is_nuisance_failure(response_dict)):
try:
from django.core.mail import EmailMessage
msg = EmailMessage(
"".join([
"[%s:%s] ",
_("code question execution failed")
]) % (page_context.course.identifier,
page_context.flow_session.flow_id
if page_context.flow_session is not None else
_("<unknown flow>")), message,
settings.ROBOT_EMAIL_FROM,
[page_context.course.notify_email])
from relate.utils import get_outbound_mail_connection
msg.connection = get_outbound_mail_connection("robot")
msg.send()
except Exception:
from traceback import format_exc
feedback_bits.append(
str(
string_concat(
"<p>",
_("Both the grading code and the attempt to "
"notify course staff about the issue failed. "
"Please contact the course or site staff and "
"inform them of this issue, mentioning this "
"entire error message:"), "</p>", "<p>",
_("Sending an email to the course staff about the "
"following failure failed with "
"the following error message:"), "<pre>",
"".join(format_exc()), "</pre>",
_("The original failure message follows:"),
"</p>")))
# }}}
if hasattr(self.page_desc, "correct_code"):
def normalize_code(s):
return (s.replace(" ", "").replace("\r", "").replace(
"\n", "").replace("\t", ""))
if (normalize_code(user_code) == normalize_code(
self.page_desc.correct_code)):
feedback_bits.append(
"<p><b>%s</b></p>" %
_("It looks like you submitted code that is identical to "
"the reference solution. This is not allowed."))
from relate.utils import dict_to_struct
response = dict_to_struct(response_dict)
bulk_feedback_bits = []
if response.result == "success":
pass
elif response.result in [
"uncaught_error", "setup_compile_error", "setup_error",
"test_compile_error", "test_error"
]:
feedback_bits.append("".join([
"<p>",
_("The grading code failed. Sorry about that. "
"The staff has been informed, and if this problem is "
"due to an issue with the grading code, "
"it will be fixed as soon as possible. "
"In the meantime, you'll see a traceback "
"below that may help you figure out what went wrong."),
"</p>"
]))
elif response.result == "timeout":
feedback_bits.append("".join([
"<p>",
_("Your code took too long to execute. The problem "
"specifies that your code may take at most %s seconds "
"to run. "
"It took longer than that and was aborted."), "</p>"
]) % self.page_desc.timeout)
correctness = 0
elif response.result == "user_compile_error":
feedback_bits.append("".join([
"<p>",
_("Your code failed to compile. An error message is "
"below."), "</p>"
]))
correctness = 0
elif response.result == "user_error":
feedback_bits.append("".join([
"<p>",
_("Your code failed with an exception. "
"A traceback is below."), "</p>"
]))
correctness = 0
else:
raise RuntimeError("invalid run result: %s" % response.result)
if hasattr(response, "feedback") and response.feedback:
def sanitize(s):
import bleach
return bleach.clean(s, tags=["p", "pre"])
feedback_bits.append("".join([
"<p>",
_("Here is some feedback on your code"), ":"
"<ul>%s</ul></p>"
]) % "".join("<li>%s</li>" % sanitize(fb_item)
for fb_item in response.feedback))
if hasattr(response, "traceback") and response.traceback:
feedback_bits.append("".join([
"<p>",
_("This is the exception traceback"), ":"
"<pre>%s</pre></p>"
]) % escape(response.traceback))
if hasattr(response,
"exec_host") and response.exec_host != "localhost":
import socket
try:
exec_host_name, dummy, dummy = socket.gethostbyaddr(
response.exec_host)
except socket.error:
exec_host_name = response.exec_host
feedback_bits.append("".join(
["<p>",
_("Your code ran on %s.") % exec_host_name, "</p>"]))
if hasattr(response, "stdout") and response.stdout:
bulk_feedback_bits.append("".join([
"<p>",
_("Your code printed the following output"), ":"
"<pre>%s</pre></p>"
]) % escape(response.stdout))
if hasattr(response, "stderr") and response.stderr:
bulk_feedback_bits.append("".join([
"<p>",
_("Your code printed the following error messages"), ":"
"<pre>%s</pre></p>"
]) % escape(response.stderr))
if hasattr(response, "figures") and response.figures:
fig_lines = [
"".join([
"<p>",
_("Your code produced the following plots"), ":</p>"
]),
'<dl class="result-figure-list">',
]
for nr, mime_type, b64data in response.figures:
if mime_type in ["image/jpeg", "image/png"]:
fig_lines.extend([
"".join(["<dt>", _("Figure"), "%d<dt>"]) % nr,
'<dd><img alt="Figure %d" src="data:%s;base64,%s"></dd>'
% (nr, mime_type, b64data)
])
fig_lines.append("</dl>")
bulk_feedback_bits.extend(fig_lines)
# {{{ html output / santization
if hasattr(response, "html") and response.html:
def is_allowed_data_uri(allowed_mimetypes, uri):
import re
m = re.match(r"^data:([-a-z0-9]+/[-a-z0-9]+);base64,", uri)
if not m:
return False
mimetype = m.group(1)
return mimetype in allowed_mimetypes
def sanitize(s):
import bleach
def filter_audio_attributes(tag, name, value):
if name in ["controls"]:
return True
else:
return False
def filter_source_attributes(tag, name, value):
if name in ["type"]:
return True
elif name == "src":
if is_allowed_data_uri([
"audio/wav",
], value):
return bleach.sanitizer.VALUE_SAFE
else:
return False
else:
return False
def filter_img_attributes(tag, name, value):
if name in ["alt", "title"]:
return True
elif name == "src":
return is_allowed_data_uri([
"image/png",
"image/jpeg",
], value)
else:
return False
if not isinstance(s, str):
return _("(Non-string in 'HTML' output filtered out)")
return bleach.clean(s,
tags=bleach.ALLOWED_TAGS +
["audio", "video", "source"],
attributes={
"audio": filter_audio_attributes,
"source": filter_source_attributes,
"img": filter_img_attributes,
})
bulk_feedback_bits.extend(
sanitize(snippet) for snippet in response.html)
# }}}
return AnswerFeedback(correctness=correctness,
feedback="\n".join(feedback_bits),
bulk_feedback="\n".join(bulk_feedback_bits))
def test_correctness_exceed_max_extra_credit_factor(self):
correctness = MAX_EXTRA_CREDIT_FACTOR + 0.1
with self.assertRaises(InvalidFeedbackPointsError):
get_auto_feedback(correctness)
def test_correctness_exceed_max_extra_credit_factor(self):
correctness = MAX_EXTRA_CREDIT_FACTOR + 0.1
with self.assertRaises(InvalidFeedbackPointsError):
get_auto_feedback(correctness)
def test_correctness_negative(self):
correctness = -0.1
with self.assertRaises(InvalidFeedbackPointsError):
get_auto_feedback(correctness)
def test_with_somewhat_correct(self):
self.assertIn("somewhat correct", get_auto_feedback(0.5))
self.assertIn("somewhat correct", get_auto_feedback(0.5000001))
self.assertIn("somewhat correct", get_auto_feedback(0.001))
self.assertIn("somewhat correct", get_auto_feedback(0.2))
def test_with_mostly_correct(self):
self.assertIn("mostly correct", get_auto_feedback(0.51))
self.assertIn("mostly correct", get_auto_feedback(0.999))
def grade(self, page_context, page_data, answer_data, grade_data):
if answer_data is None:
return AnswerFeedback(correctness=0, feedback=_("No answer provided."))
user_code = answer_data["answer"]
# {{{ request run
run_req = {"compile_only": False, "user_code": user_code}
def transfer_attr(name):
if hasattr(self.page_desc, name):
run_req[name] = getattr(self.page_desc, name)
transfer_attr("setup_code")
transfer_attr("names_for_user")
transfer_attr("names_from_user")
if hasattr(self.page_desc, "test_code"):
run_req["test_code"] = self.get_test_code()
if hasattr(self.page_desc, "data_files"):
run_req["data_files"] = {}
from course.content import get_repo_blob
for data_file in self.page_desc.data_files:
from base64 import b64encode
run_req["data_files"][data_file] = b64encode(
get_repo_blob(page_context.repo, data_file, page_context.commit_sha).data
).decode()
try:
response_dict = request_python_run_with_retries(run_req, run_timeout=self.page_desc.timeout)
except:
from traceback import format_exc
response_dict = {
"result": "uncaught_error",
"message": "Error connecting to container",
"traceback": "".join(format_exc()),
}
# }}}
feedback_bits = []
# {{{ send email if the grading code broke
if response_dict["result"] in [
"uncaught_error",
"setup_compile_error",
"setup_error",
"test_compile_error",
"test_error",
]:
error_msg_parts = ["RESULT: %s" % response_dict["result"]]
for key, val in sorted(response_dict.items()):
if key not in ["result", "figures"] and val and isinstance(val, six.string_types):
error_msg_parts.append("-------------------------------------")
error_msg_parts.append(key)
error_msg_parts.append("-------------------------------------")
error_msg_parts.append(val)
error_msg_parts.append("-------------------------------------")
error_msg_parts.append("user code")
error_msg_parts.append("-------------------------------------")
error_msg_parts.append(user_code)
error_msg_parts.append("-------------------------------------")
error_msg = "\n".join(error_msg_parts)
with translation.override(settings.RELATE_ADMIN_EMAIL_LOCALE):
from django.template.loader import render_to_string
message = render_to_string(
"course/broken-code-question-email.txt",
{"page_id": self.page_desc.id, "course": page_context.course, "error_message": error_msg},
)
if not page_context.in_sandbox and not is_nuisance_failure(response_dict):
try:
from django.core.mail import send_mail
send_mail(
"".join(["[%s] ", _("code question execution failed")]) % page_context.course.identifier,
message,
settings.ROBOT_EMAIL_FROM,
recipient_list=[page_context.course.notify_email],
)
except Exception:
from traceback import format_exc
feedback_bits.append(
six.text_type(
string_concat(
"<p>",
_(
"Both the grading code and the attempt to "
"notify course staff about the issue failed. "
"Please contact the course or site staff and "
"inform them of this issue, mentioning this "
"entire error message:"
),
"</p>",
"<p>",
_(
"Sending an email to the course staff about the "
"following failure failed with "
"the following error message:"
),
"<pre>",
"".join(format_exc()),
"</pre>",
_("The original failure message follows:"),
"</p>",
)
)
)
# }}}
from relate.utils import dict_to_struct
response = dict_to_struct(response_dict)
bulk_feedback_bits = []
if hasattr(response, "points"):
correctness = response.points
feedback_bits.append("<p><b>%s</b></p>" % get_auto_feedback(correctness))
else:
correctness = None
if response.result == "success":
pass
elif response.result in [
"uncaught_error",
"setup_compile_error",
"setup_error",
"test_compile_error",
"test_error",
]:
feedback_bits.append(
"".join(
[
"<p>",
_(
"The grading code failed. Sorry about that. "
"The staff has been informed, and if this problem is "
"due to an issue with the grading code, "
"it will be fixed as soon as possible. "
"In the meantime, you'll see a traceback "
"below that may help you figure out what went wrong."
),
"</p>",
]
)
)
elif response.result == "timeout":
feedback_bits.append(
"".join(
[
"<p>",
_(
"Your code took too long to execute. The problem "
"specifies that your code may take at most %s seconds "
"to run. "
"It took longer than that and was aborted."
),
"</p>",
]
)
% self.page_desc.timeout
)
correctness = 0
elif response.result == "user_compile_error":
feedback_bits.append(
"".join(["<p>", _("Your code failed to compile. An error message is " "below."), "</p>"])
)
correctness = 0
elif response.result == "user_error":
feedback_bits.append(
"".join(["<p>", _("Your code failed with an exception. " "A traceback is below."), "</p>"])
)
correctness = 0
else:
raise RuntimeError("invalid runpy result: %s" % response.result)
if hasattr(response, "feedback") and response.feedback:
feedback_bits.append(
"".join(["<p>", _("Here is some feedback on your code"), ":" "<ul>%s</ul></p>"])
% "".join("<li>%s</li>" % escape(fb_item) for fb_item in response.feedback)
)
if hasattr(response, "traceback") and response.traceback:
feedback_bits.append(
"".join(["<p>", _("This is the exception traceback"), ":" "<pre>%s</pre></p>"])
% escape(response.traceback)
)
if hasattr(response, "exec_host") and response.exec_host != "localhost":
import socket
try:
exec_host_name, dummy, dummy = socket.gethostbyaddr(response.exec_host)
except socket.error:
exec_host_name = response.exec_host
feedback_bits.append("".join(["<p>", _("Your code ran on %s.") % exec_host_name, "</p>"]))
if hasattr(response, "stdout") and response.stdout:
bulk_feedback_bits.append(
"".join(["<p>", _("Your code printed the following output"), ":" "<pre>%s</pre></p>"])
% escape(response.stdout)
)
if hasattr(response, "stderr") and response.stderr:
bulk_feedback_bits.append(
"".join(["<p>", _("Your code printed the following error messages"), ":" "<pre>%s</pre></p>"])
% escape(response.stderr)
)
if hasattr(response, "figures") and response.figures:
fig_lines = [
"".join(["<p>", _("Your code produced the following plots"), ":</p>"]),
'<dl class="result-figure-list">',
]
for nr, mime_type, b64data in response.figures:
fig_lines.extend(
[
"".join(["<dt>", _("Figure"), "%d<dt>"]) % nr,
'<dd><img alt="Figure %d" src="data:%s;base64,%s"></dd>' % (nr, mime_type, b64data),
]
)
fig_lines.append("</dl>")
bulk_feedback_bits.extend(fig_lines)
return AnswerFeedback(
correctness=correctness, feedback="\n".join(feedback_bits), bulk_feedback="\n".join(bulk_feedback_bits)
)
def grade(self, page_context, page_data, answer_data, grade_data):
if answer_data is None:
return AnswerFeedback(correctness=0,
feedback=_("No answer provided."))
user_code = answer_data["answer"]
# {{{ request run
run_req = {"compile_only": False, "user_code": user_code}
def transfer_attr(name):
if hasattr(self.page_desc, name):
run_req[name] = getattr(self.page_desc, name)
transfer_attr("setup_code")
transfer_attr("names_for_user")
transfer_attr("names_from_user")
run_req["test_code"] = self.get_test_code()
if hasattr(self.page_desc, "data_files"):
run_req["data_files"] = {}
from course.content import get_repo_blob
for data_file in self.page_desc.data_files:
from base64 import b64encode
run_req["data_files"][data_file] = \
b64encode(
get_repo_blob(
page_context.repo, data_file,
page_context.commit_sha).data).decode()
try:
response_dict = request_python_run_with_retries(run_req,
run_timeout=self.page_desc.timeout)
except Exception:
from traceback import format_exc
response_dict = {
"result": "uncaught_error",
"message": "Error connecting to container",
"traceback": "".join(format_exc()),
}
# }}}
feedback_bits = []
correctness = None
if "points" in response_dict:
correctness = response_dict["points"]
try:
feedback_bits.append(
"<p><b>%s</b></p>"
% _(get_auto_feedback(correctness)))
except Exception as e:
correctness = None
response_dict["result"] = "setup_error"
response_dict["message"] = (
"%s: %s" % (type(e).__name__, str(e))
)
# {{{ send email if the grading code broke
if response_dict["result"] in [
"uncaught_error",
"setup_compile_error",
"setup_error",
"test_compile_error",
"test_error"]:
error_msg_parts = ["RESULT: %s" % response_dict["result"]]
for key, val in sorted(response_dict.items()):
if (key not in ["result", "figures"]
and val
and isinstance(val, six.string_types)):
error_msg_parts.append("-------------------------------------")
error_msg_parts.append(key)
error_msg_parts.append("-------------------------------------")
error_msg_parts.append(val)
error_msg_parts.append("-------------------------------------")
error_msg_parts.append("user code")
error_msg_parts.append("-------------------------------------")
error_msg_parts.append(user_code)
error_msg_parts.append("-------------------------------------")
error_msg = "\n".join(error_msg_parts)
from relate.utils import local_now, format_datetime_local
from course.utils import LanguageOverride
with LanguageOverride(page_context.course):
from relate.utils import render_email_template
message = render_email_template(
"course/broken-code-question-email.txt", {
"site": getattr(settings, "RELATE_BASE_URL"),
"page_id": self.page_desc.id,
"course": page_context.course,
"error_message": error_msg,
"review_uri": page_context.page_uri,
"time": format_datetime_local(local_now())
})
if (
not page_context.in_sandbox
and not is_nuisance_failure(response_dict)):
try:
from django.core.mail import EmailMessage
msg = EmailMessage("".join(["[%s:%s] ",
_("code question execution failed")])
% (
page_context.course.identifier,
page_context.flow_session.flow_id
if page_context.flow_session is not None
else _("<unknown flow>")),
message,
settings.ROBOT_EMAIL_FROM,
[page_context.course.notify_email])
from relate.utils import get_outbound_mail_connection
msg.connection = get_outbound_mail_connection("robot")
msg.send()
except Exception:
from traceback import format_exc
feedback_bits.append(
six.text_type(string_concat(
"<p>",
_(
"Both the grading code and the attempt to "
"notify course staff about the issue failed. "
"Please contact the course or site staff and "
"inform them of this issue, mentioning this "
"entire error message:"),
"</p>",
"<p>",
_(
"Sending an email to the course staff about the "
"following failure failed with "
"the following error message:"),
"<pre>",
"".join(format_exc()),
"</pre>",
_("The original failure message follows:"),
"</p>")))
# }}}
if hasattr(self.page_desc, "correct_code"):
def normalize_code(s):
return (s
.replace(" ", "")
.replace("\r", "")
.replace("\n", "")
.replace("\t", ""))
if (normalize_code(user_code)
== normalize_code(self.page_desc.correct_code)):
feedback_bits.append(
"<p><b>%s</b></p>"
% _("It looks like you submitted code that is identical to "
"the reference solution. This is not allowed."))
from relate.utils import dict_to_struct
response = dict_to_struct(response_dict)
bulk_feedback_bits = []
if response.result == "success":
pass
elif response.result in [
"uncaught_error",
"setup_compile_error",
"setup_error",
"test_compile_error",
"test_error"]:
feedback_bits.append("".join([
"<p>",
_(
"The grading code failed. Sorry about that. "
"The staff has been informed, and if this problem is "
"due to an issue with the grading code, "
"it will be fixed as soon as possible. "
"In the meantime, you'll see a traceback "
"below that may help you figure out what went wrong."
),
"</p>"]))
elif response.result == "timeout":
feedback_bits.append("".join([
"<p>",
_(
"Your code took too long to execute. The problem "
"specifies that your code may take at most %s seconds "
"to run. "
"It took longer than that and was aborted."
),
"</p>"])
% self.page_desc.timeout)
correctness = 0
elif response.result == "user_compile_error":
feedback_bits.append("".join([
"<p>",
_("Your code failed to compile. An error message is "
"below."),
"</p>"]))
correctness = 0
elif response.result == "user_error":
feedback_bits.append("".join([
"<p>",
_("Your code failed with an exception. "
"A traceback is below."),
"</p>"]))
correctness = 0
else:
raise RuntimeError("invalid runpy result: %s" % response.result)
if hasattr(response, "feedback") and response.feedback:
def sanitize(s):
import bleach
return bleach.clean(s, tags=["p", "pre"])
feedback_bits.append("".join([
"<p>",
_("Here is some feedback on your code"),
":"
"<ul>%s</ul></p>"]) %
"".join(
"<li>%s</li>" % sanitize(fb_item)
for fb_item in response.feedback))
if hasattr(response, "traceback") and response.traceback:
feedback_bits.append("".join([
"<p>",
_("This is the exception traceback"),
":"
"<pre>%s</pre></p>"]) % escape(response.traceback))
if hasattr(response, "exec_host") and response.exec_host != "localhost":
import socket
try:
exec_host_name, dummy, dummy = socket.gethostbyaddr(
response.exec_host)
except socket.error:
exec_host_name = response.exec_host
feedback_bits.append("".join([
"<p>",
_("Your code ran on %s.") % exec_host_name,
"</p>"]))
if hasattr(response, "stdout") and response.stdout:
bulk_feedback_bits.append("".join([
"<p>",
_("Your code printed the following output"),
":"
"<pre>%s</pre></p>"])
% escape(response.stdout))
if hasattr(response, "stderr") and response.stderr:
bulk_feedback_bits.append("".join([
"<p>",
_("Your code printed the following error messages"),
":"
"<pre>%s</pre></p>"]) % escape(response.stderr))
if hasattr(response, "figures") and response.figures:
fig_lines = ["".join([
"<p>",
_("Your code produced the following plots"),
":</p>"]),
'<dl class="result-figure-list">',
]
for nr, mime_type, b64data in response.figures:
if mime_type in ["image/jpeg", "image/png"]:
fig_lines.extend([
"".join([
"<dt>",
_("Figure"), "%d<dt>"]) % nr,
'<dd><img alt="Figure %d" src="data:%s;base64,%s"></dd>'
% (nr, mime_type, b64data)])
fig_lines.append("</dl>")
bulk_feedback_bits.extend(fig_lines)
# {{{ html output / santization
if hasattr(response, "html") and response.html:
def is_allowed_data_uri(allowed_mimetypes, uri):
import re
m = re.match(r"^data:([-a-z0-9]+/[-a-z0-9]+);base64,", uri)
if not m:
return False
mimetype = m.group(1)
return mimetype in allowed_mimetypes
def sanitize(s):
import bleach
def filter_audio_attributes(tag, name, value):
if name in ["controls"]:
return True
else:
return False
def filter_source_attributes(tag, name, value):
if name in ["type"]:
return True
elif name == "src":
if is_allowed_data_uri([
"audio/wav",
], value):
return bleach.sanitizer.VALUE_SAFE
else:
return False
else:
return False
def filter_img_attributes(tag, name, value):
if name in ["alt", "title"]:
return True
elif name == "src":
return is_allowed_data_uri([
"image/png",
"image/jpeg",
], value)
else:
return False
if not isinstance(s, six.text_type):
return _("(Non-string in 'HTML' output filtered out)")
return bleach.clean(s,
tags=bleach.ALLOWED_TAGS + ["audio", "video", "source"],
attributes={
"audio": filter_audio_attributes,
"source": filter_source_attributes,
"img": filter_img_attributes,
})
bulk_feedback_bits.extend(
sanitize(snippet) for snippet in response.html)
# }}}
return AnswerFeedback(
correctness=correctness,
feedback="\n".join(feedback_bits),
bulk_feedback="\n".join(bulk_feedback_bits))
