def test_grade(self, gradebook):
"""Can files be graded?"""
self._copy_file("files/submitted-unchanged.ipynb", "source/ps1/p1.ipynb")
run_command('nbgrader assign ps1 --db="{}" '.format(gradebook))
self._copy_file("files/submitted-unchanged.ipynb", "submitted/foo/ps1/p1.ipynb")
self._copy_file("files/submitted-changed.ipynb", "submitted/bar/ps1/p1.ipynb")
run_command('nbgrader autograde ps1 --db="{}"'.format(gradebook))
assert os.path.isfile("autograded/foo/ps1/p1.ipynb")
assert not os.path.isfile("autograded/foo/ps1/timestamp.txt")
assert os.path.isfile("autograded/bar/ps1/p1.ipynb")
assert not os.path.isfile("autograded/bar/ps1/timestamp.txt")
gb = Gradebook(gradebook)
notebook = gb.find_submission_notebook("p1", "ps1", "foo")
assert notebook.score == 1
assert notebook.max_score == 4
assert notebook.needs_manual_grade == False
comment1 = gb.find_comment(0, "p1", "ps1", "foo")
comment2 = gb.find_comment(1, "p1", "ps1", "foo")
assert comment1.comment == "No response."
assert comment2.comment == "No response."
notebook = gb.find_submission_notebook("p1", "ps1", "bar")
assert notebook.score == 2
assert notebook.max_score == 4
assert notebook.needs_manual_grade == True
comment1 = gb.find_comment(0, "p1", "ps1", "bar")
comment2 = gb.find_comment(1, "p1", "ps1", "bar")
assert comment1.comment == None
assert comment2.comment == None
def test_find_comment(assignmentWithSubmissionWithMarks: Gradebook) -> None:
s = assignmentWithSubmissionWithMarks.find_submission('foo', 'hacker123')
for n in s.notebooks:
comments = n.comments
for c1 in comments:
c2 = assignmentWithSubmissionWithMarks.find_comment(c1.name, n.name, 'foo', 'hacker123')
assert c1 == c2
with pytest.raises(MissingEntry):
assignmentWithSubmissionWithMarks.find_comment('asdf', n.name, 'foo', 'hacker123')
class GetGrades(NbGraderPreprocessor):
"""Preprocessor for saving grades from the database to the notebook"""
def preprocess(self, nb, resources):
# pull information from the resources
self.notebook_id = resources['nbgrader']['notebook']
self.assignment_id = resources['nbgrader']['assignment']
self.student_id = resources['nbgrader']['student']
self.db_url = resources['nbgrader']['db_url']
# connect to the database
self.gradebook = Gradebook(self.db_url)
# process the cells
nb, resources = super(GetGrades, self).preprocess(nb, resources)
submission = self.gradebook.find_submission_notebook(
self.notebook_id, self.assignment_id, self.student_id)
resources['nbgrader']['score'] = submission.score
resources['nbgrader']['max_score'] = submission.max_score
return nb, resources
def _get_comment(self, cell, resources):
"""Graders can optionally add comments to the student's solutions, so
add the comment information into the database if it doesn't
already exist. It should NOT overwrite existing comments that
might have been added by a grader already.
"""
# retrieve or create the comment object from the database
comment = self.gradebook.find_comment(
cell.metadata['nbgrader']['grade_id'], self.notebook_id,
self.assignment_id, self.student_id)
# save it in the notebook
cell.metadata.nbgrader['comment'] = comment.comment
def _get_score(self, cell, resources):
grade = self.gradebook.find_grade(
cell.metadata['nbgrader']['grade_id'], self.notebook_id,
self.assignment_id, self.student_id)
cell.metadata.nbgrader['score'] = grade.score
cell.metadata.nbgrader['points'] = grade.max_score
def preprocess_cell(self, cell, resources, cell_index):
# if it's a solution cell, then add a comment
if utils.is_solution(cell):
self._get_comment(cell, resources)
# if it's a grade cell, the add a grade
if utils.is_grade(cell):
self._get_score(cell, resources)
return cell, resources
def test_grade(self):
"""Can files be graded?"""
with self._temp_cwd(["files/submitted-unchanged.ipynb", "files/submitted-changed.ipynb"]):
dbpath = self._setup_db()
os.makedirs('source/ps1')
shutil.copy('submitted-unchanged.ipynb', 'source/ps1/p1.ipynb')
self._run_command('nbgrader assign ps1 --db="{}" '.format(dbpath))
os.makedirs('submitted/foo/ps1')
shutil.move('submitted-unchanged.ipynb', 'submitted/foo/ps1/p1.ipynb')
os.makedirs('submitted/bar/ps1')
shutil.move('submitted-changed.ipynb', 'submitted/bar/ps1/p1.ipynb')
self._run_command('nbgrader autograde ps1 --db="{}"'.format(dbpath))
assert os.path.isfile("autograded/foo/ps1/p1.ipynb")
assert not os.path.isfile("autograded/foo/ps1/timestamp.txt")
assert os.path.isfile("autograded/bar/ps1/p1.ipynb")
assert not os.path.isfile("autograded/bar/ps1/timestamp.txt")
gb = Gradebook(dbpath)
notebook = gb.find_submission_notebook("p1", "ps1", "foo")
assert_equal(notebook.score, 1)
assert_equal(notebook.max_score, 4)
assert_equal(notebook.needs_manual_grade, False)
comment1 = gb.find_comment(0, "p1", "ps1", "foo")
comment2 = gb.find_comment(1, "p1", "ps1", "foo")
assert_equal(comment1.comment, "No response.")
assert_equal(comment2.comment, "No response.")
notebook = gb.find_submission_notebook("p1", "ps1", "bar")
assert_equal(notebook.score, 2)
assert_equal(notebook.max_score, 4)
assert_equal(notebook.needs_manual_grade, True)
comment1 = gb.find_comment(0, "p1", "ps1", "bar")
comment2 = gb.find_comment(1, "p1", "ps1", "bar")
assert_equal(comment1.comment, None)
assert_equal(comment2.comment, None)
def test_grade(self, gradebook):
"""Can files be graded?"""
self._copy_file("files/submitted-unchanged.ipynb",
"source/ps1/p1.ipynb")
run_command(["nbgrader", "assign", "ps1", "--db", gradebook])
self._copy_file("files/submitted-unchanged.ipynb",
"submitted/foo/ps1/p1.ipynb")
self._copy_file("files/submitted-changed.ipynb",
"submitted/bar/ps1/p1.ipynb")
run_command(["nbgrader", "autograde", "ps1", "--db", gradebook])
assert os.path.isfile("autograded/foo/ps1/p1.ipynb")
assert not os.path.isfile("autograded/foo/ps1/timestamp.txt")
assert os.path.isfile("autograded/bar/ps1/p1.ipynb")
assert not os.path.isfile("autograded/bar/ps1/timestamp.txt")
gb = Gradebook(gradebook)
notebook = gb.find_submission_notebook("p1", "ps1", "foo")
assert notebook.score == 1
assert notebook.max_score == 7
assert notebook.needs_manual_grade == False
comment1 = gb.find_comment("set_a", "p1", "ps1", "foo")
comment2 = gb.find_comment("baz", "p1", "ps1", "foo")
comment3 = gb.find_comment("quux", "p1", "ps1", "foo")
assert comment1.comment == "No response."
assert comment2.comment == "No response."
assert comment3.comment == "No response."
notebook = gb.find_submission_notebook("p1", "ps1", "bar")
assert notebook.score == 2
assert notebook.max_score == 7
assert notebook.needs_manual_grade == True
comment1 = gb.find_comment("set_a", "p1", "ps1", "bar")
comment2 = gb.find_comment("baz", "p1", "ps1", "bar")
comment2 = gb.find_comment("quux", "p1", "ps1", "bar")
assert comment1.comment == None
assert comment2.comment == None
def test_grade(self, gradebook):
"""Can files be graded?"""
self._copy_file("files/submitted-unchanged.ipynb", "source/ps1/p1.ipynb")
run_command(["nbgrader", "assign", "ps1", "--db", gradebook])
self._copy_file("files/submitted-unchanged.ipynb", "submitted/foo/ps1/p1.ipynb")
self._copy_file("files/submitted-changed.ipynb", "submitted/bar/ps1/p1.ipynb")
run_command(["nbgrader", "autograde", "ps1", "--db", gradebook])
assert os.path.isfile("autograded/foo/ps1/p1.ipynb")
assert not os.path.isfile("autograded/foo/ps1/timestamp.txt")
assert os.path.isfile("autograded/bar/ps1/p1.ipynb")
assert not os.path.isfile("autograded/bar/ps1/timestamp.txt")
gb = Gradebook(gradebook)
notebook = gb.find_submission_notebook("p1", "ps1", "foo")
assert notebook.score == 1
assert notebook.max_score == 7
assert notebook.needs_manual_grade == False
comment1 = gb.find_comment("set_a", "p1", "ps1", "foo")
comment2 = gb.find_comment("baz", "p1", "ps1", "foo")
comment3 = gb.find_comment("quux", "p1", "ps1", "foo")
assert comment1.comment == "No response."
assert comment2.comment == "No response."
assert comment3.comment == "No response."
notebook = gb.find_submission_notebook("p1", "ps1", "bar")
assert notebook.score == 2
assert notebook.max_score == 7
assert notebook.needs_manual_grade == True
comment1 = gb.find_comment("set_a", "p1", "ps1", "bar")
comment2 = gb.find_comment("baz", "p1", "ps1", "bar")
comment2 = gb.find_comment("quux", "p1", "ps1", "bar")
assert comment1.comment == None
assert comment2.comment == None
class SaveAutoGrades(NbGraderPreprocessor):
"""Preprocessor for saving out the autograder grades into a database"""
def preprocess(self, nb, resources):
# pull information from the resources
self.notebook_id = resources['nbgrader']['notebook']
self.assignment_id = resources['nbgrader']['assignment']
self.student_id = resources['nbgrader']['student']
self.db_url = resources['nbgrader']['db_url']
# connect to the database
self.gradebook = Gradebook(self.db_url)
# process the cells
nb, resources = super(SaveAutoGrades, self).preprocess(nb, resources)
return nb, resources
def _add_score(self, cell, resources):
"""Graders can override the autograder grades, and may need to
manually grade written solutions anyway. This function adds
score information to the database if it doesn't exist. It does
NOT override the 'score' field, as this is the manual score
that might have been provided by a grader.
"""
# these are the fields by which we will identify the score
# information
grade = self.gradebook.find_grade(
cell.metadata['nbgrader']['grade_id'], self.notebook_id,
self.assignment_id, self.student_id)
# determine what the grade is
auto_score, _ = utils.determine_grade(cell)
grade.auto_score = auto_score
# if there was previously a manual grade, or if there is no autograder
# score, then we should mark this as needing review
if (grade.manual_score is not None) or (grade.auto_score is None):
grade.needs_manual_grade = True
else:
grade.needs_manual_grade = False
self.gradebook.db.commit()
self.log.debug(grade)
def _add_comment(self, cell, resources):
comment = self.gradebook.find_comment(
cell.metadata['nbgrader']['grade_id'], self.notebook_id,
self.assignment_id, self.student_id)
if cell.metadata.nbgrader.get("checksum",
None) == utils.compute_checksum(cell):
comment.auto_comment = "No response."
else:
comment.auto_comment = None
self.gradebook.db.commit()
self.log.debug(comment)
def preprocess_cell(self, cell, resources, cell_index):
# if it's a grade cell, the add a grade
if utils.is_grade(cell):
self._add_score(cell, resources)
if utils.is_solution(cell):
self._add_comment(cell, resources)
return cell, resources
class TestSaveAutoGrades(TestBase):
def setup(self):
super(TestSaveAutoGrades, self).setup()
db_url = self._init_db()
self.gb = Gradebook(db_url)
self.gb.add_assignment("ps0")
self.gb.add_student("bar")
self.preprocessor1 = SaveCells()
self.preprocessor2 = SaveAutoGrades()
self.resources = {
"nbgrader": {
"db_url": db_url,
"assignment": "ps0",
"notebook": "test",
"student": "bar"
}
}
def test_grade_correct_code(self):
"""Is a passing code cell correctly graded?"""
cell = self._create_grade_cell("hello", "code", "foo", 1)
nb = new_notebook()
nb.cells.append(cell)
self.preprocessor1.preprocess(nb, self.resources)
self.gb.add_submission("ps0", "bar")
self.preprocessor2.preprocess(nb, self.resources)
grade_cell = self.gb.find_grade("foo", "test", "ps0", "bar")
assert_equal(grade_cell.score, 1)
assert_equal(grade_cell.max_score, 1)
assert_equal(grade_cell.auto_score, 1)
assert_equal(grade_cell.manual_score, None)
assert not grade_cell.needs_manual_grade
def test_grade_incorrect_code(self):
"""Is a failing code cell correctly graded?"""
cell = self._create_grade_cell("hello", "code", "foo", 1)
cell.outputs = [new_output('error', ename="NotImplementedError", evalue="", traceback=["error"])]
nb = new_notebook()
nb.cells.append(cell)
self.preprocessor1.preprocess(nb, self.resources)
self.gb.add_submission("ps0", "bar")
self.preprocessor2.preprocess(nb, self.resources)
grade_cell = self.gb.find_grade("foo", "test", "ps0", "bar")
assert_equal(grade_cell.score, 0)
assert_equal(grade_cell.max_score, 1)
assert_equal(grade_cell.auto_score, 0)
assert_equal(grade_cell.manual_score, None)
assert not grade_cell.needs_manual_grade
def test_grade_unchanged_markdown(self):
"""Is an unchanged markdown cell correctly graded?"""
cell = self._create_grade_and_solution_cell("hello", "markdown", "foo", 1)
nb = new_notebook()
nb.cells.append(cell)
self.preprocessor1.preprocess(nb, self.resources)
self.gb.add_submission("ps0", "bar")
self.preprocessor2.preprocess(nb, self.resources)
grade_cell = self.gb.find_grade("foo", "test", "ps0", "bar")
assert_equal(grade_cell.score, 0)
assert_equal(grade_cell.max_score, 1)
assert_equal(grade_cell.auto_score, 0)
assert_equal(grade_cell.manual_score, None)
assert not grade_cell.needs_manual_grade
def test_grade_changed_markdown(self):
"""Is a changed markdown cell correctly graded?"""
cell = self._create_grade_and_solution_cell("hello", "markdown", "foo", 1)
nb = new_notebook()
nb.cells.append(cell)
self.preprocessor1.preprocess(nb, self.resources)
self.gb.add_submission("ps0", "bar")
cell.source = "hello!"
self.preprocessor2.preprocess(nb, self.resources)
grade_cell = self.gb.find_grade("foo", "test", "ps0", "bar")
assert_equal(grade_cell.score, 0)
assert_equal(grade_cell.max_score, 1)
assert_equal(grade_cell.auto_score, None)
assert_equal(grade_cell.manual_score, None)
assert grade_cell.needs_manual_grade
def test_comment_unchanged_code(self):
"""Is an unchanged code cell given the correct comment?"""
cell = self._create_solution_cell("hello", "code")
nb = new_notebook()
nb.cells.append(cell)
self.preprocessor1.preprocess(nb, self.resources)
self.gb.add_submission("ps0", "bar")
self.preprocessor2.preprocess(nb, self.resources)
comment = self.gb.find_comment(0, "test", "ps0", "bar")
assert_equal(comment.comment, "No response.")
def test_comment_changed_code(self):
"""Is a changed code cell given the correct comment?"""
cell = self._create_solution_cell("hello", "code")
nb = new_notebook()
nb.cells.append(cell)
self.preprocessor1.preprocess(nb, self.resources)
self.gb.add_submission("ps0", "bar")
cell.source = "hello!"
self.preprocessor2.preprocess(nb, self.resources)
comment = self.gb.find_comment(0, "test", "ps0", "bar")
assert_equal(comment.comment, None)
def test_comment_unchanged_markdown(self):
"""Is an unchanged markdown cell given the correct comment?"""
cell = self._create_grade_and_solution_cell("hello", "markdown", "foo", 1)
nb = new_notebook()
nb.cells.append(cell)
self.preprocessor1.preprocess(nb, self.resources)
self.gb.add_submission("ps0", "bar")
self.preprocessor2.preprocess(nb, self.resources)
comment = self.gb.find_comment(0, "test", "ps0", "bar")
assert_equal(comment.comment, "No response.")
def test_comment_changed_markdown(self):
"""Is a changed markdown cell given the correct comment?"""
cell = self._create_grade_and_solution_cell("hello", "markdown", "foo", 1)
nb = new_notebook()
nb.cells.append(cell)
self.preprocessor1.preprocess(nb, self.resources)
self.gb.add_submission("ps0", "bar")
cell.source = "hello!"
self.preprocessor2.preprocess(nb, self.resources)
comment = self.gb.find_comment(0, "test", "ps0", "bar")
assert_equal(comment.comment, None)
class GetGrades(NbGraderPreprocessor):
"""Preprocessor for saving grades from the database to the notebook"""
def preprocess(self, nb, resources):
# pull information from the resources
self.notebook_id = resources['nbgrader']['notebook']
self.assignment_id = resources['nbgrader']['assignment']
self.student_id = resources['nbgrader']['student']
self.db_url = resources['nbgrader']['db_url']
# connect to the database
self.gradebook = Gradebook(self.db_url)
self.comment_index = 0
# process the cells
nb, resources = super(GetGrades, self).preprocess(nb, resources)
submission = self.gradebook.find_submission_notebook(
self.notebook_id, self.assignment_id, self.student_id)
resources['nbgrader']['score'] = submission.score
resources['nbgrader']['max_score'] = submission.max_score
return nb, resources
def _get_comment(self, cell, resources):
"""Graders can optionally add comments to the student's solutions, so
add the comment information into the database if it doesn't
already exist. It should NOT overwrite existing comments that
might have been added by a grader already.
"""
# retrieve or create the comment object from the database
comment = self.gradebook.find_comment(
self.comment_index,
self.notebook_id,
self.assignment_id,
self.student_id)
# save it in the notebook
cell.metadata.nbgrader['comment'] = comment.to_dict()
# update the number of comments we have inserted
self.comment_index += 1
def _get_score(self, cell, resources):
grade = self.gradebook.find_grade(
cell.metadata['nbgrader']['grade_id'],
self.notebook_id,
self.assignment_id,
self.student_id)
cell.metadata.nbgrader['score'] = grade.score
cell.metadata.nbgrader['points'] = grade.max_score
def preprocess_cell(self, cell, resources, cell_index):
# if it's a solution cell, then add a comment
if utils.is_solution(cell):
self._get_comment(cell, resources)
# if it's a grade cell, the add a grade
if utils.is_grade(cell):
self._get_score(cell, resources)
return cell, resources
class SaveAutoGrades(NbGraderPreprocessor):
"""Preprocessor for saving out the autograder grades into a database"""
def preprocess(self, nb, resources):
# pull information from the resources
self.notebook_id = resources['nbgrader']['notebook']
self.assignment_id = resources['nbgrader']['assignment']
self.student_id = resources['nbgrader']['student']
self.db_url = resources['nbgrader']['db_url']
# connect to the database
self.gradebook = Gradebook(self.db_url)
self.comment_index = 0
# process the cells
nb, resources = super(SaveAutoGrades, self).preprocess(nb, resources)
return nb, resources
def _add_score(self, cell, resources):
"""Graders can override the autograder grades, and may need to
manually grade written solutions anyway. This function adds
score information to the database if it doesn't exist. It does
NOT override the 'score' field, as this is the manual score
that might have been provided by a grader.
"""
# these are the fields by which we will identify the score
# information
grade = self.gradebook.find_grade(
cell.metadata['nbgrader']['grade_id'],
self.notebook_id,
self.assignment_id,
self.student_id)
# determine what the grade is
auto_score, max_score = utils.determine_grade(cell)
grade.auto_score = auto_score
self.gradebook.db.commit()
self.log.debug(grade)
def _add_comment(self, cell, resources):
comment = self.gradebook.find_comment(
self.comment_index,
self.notebook_id,
self.assignment_id,
self.student_id)
if comment.comment:
return
elif cell.metadata.nbgrader.get("checksum", None) == utils.compute_checksum(cell):
comment.comment = "No response."
self.gradebook.db.commit()
self.log.debug(comment)
def preprocess_cell(self, cell, resources, cell_index):
# if it's a grade cell, the add a grade
if utils.is_grade(cell):
self._add_score(cell, resources)
if utils.is_solution(cell):
self._add_comment(cell, resources)
self.comment_index += 1
return cell, resources
