def setUp(self):
super(ComparisonAPITests, self).setUp()
self.data = ComparisonTestData()
self.course = self.data.get_course()
self.assignment = self.data.get_assignments()[0]
self.base_url = self._build_url(self.course.uuid, self.assignment.uuid)
self.lti_data = LTITestData()
secondary_criterion = self.data.create_criterion(
self.data.authorized_instructor)
AssignmentCriterionFactory(criterion=secondary_criterion,
assignment=self.assignment)
db.session.commit()
def setUp(self):
# remove existing sqlite db if exists
self._delete_sqlite_db()
# get a new seed (needed for ConcurrentTestSuite so they don't produce the same results)
random.seed()
numpy.random.seed()
super(AlgorithmValidityTests, self).setUp()
self.data = ComparisonTestData()
self.MAX_COMPARSIONS = (NUMBER_OF_ANSWERS - 1) * (NUMBER_OF_ANSWERS -
2) / 2
self.TOTAL_MAX_ROUNDS = 50
self.COMPARISONS_IN_ROUND = math.ceil(NUMBER_OF_ANSWERS / 2)
# stop after lowest of total comparisons possible or 100 rounds worth of comparisons are complete
self.TOTAL_MAX_COMPARISONS = min(
self.COMPARISONS_IN_ROUND * self.TOTAL_MAX_ROUNDS,
NUMBER_OF_ANSWERS * self.MAX_COMPARSIONS)
self.course = self.data.create_course()
self.instructor = self.data.create_instructor()
self.data.enrol_instructor(self.instructor, self.course)
self.assignment = self.data.create_assignment_in_comparison_period(
self.course,
self.instructor,
number_of_comparisons=self.MAX_COMPARSIONS,
scoring_algorithm=AlgorithmValidityTests.SCORING_ALGORITHM,
pairing_algorithm=AlgorithmValidityTests.PAIRING_ALGORITHM)
self.students = []
self.answers = []
self.grade_by_answer_uuid = {}
actual_grades = ACTUAL_GRADES
if not actual_grades:
actual_grades = numpy.random.normal(0.78, 0.1,
self.NUMBER_OF_ANSWERS)
for grade in actual_grades:
student = self.data.create_normal_user()
self.data.enrol_student(student, self.course)
self.students.append(student)
answer = self.data.create_answer(self.assignment,
student,
with_score=False)
self.answers.append(answer)
self.grade_by_answer_uuid[answer.uuid] = grade
self.base_url = self._build_url(self.course.uuid, self.assignment.uuid)
db.session.commit()
def setUp(self):
if SKIP_VALIDITY_TEST:
self.skipTest("scipy and numpy not installed. run `make deps`")
# remove existing sqlite db if exists
# self._delete_sqlite_db()
# TODO: Modify conditions to be more fuzzy (closely_matched_errors with 0.05 correct rate)
# Depends on results of research
super(AlgorithmValidityTests, self).setUp()
self.data = ComparisonTestData()
self.ACCEPTABLE_CORRELATION = 0.8
self.NUMBER_OF_ANSWERS = 40
self.WINNER_SELECTOR = WinnerSelector.always_correct
self.CORRECT_RATE = 1.0
self.MAX_COMPARSIONS = (self.NUMBER_OF_ANSWERS - 1) * (self.NUMBER_OF_ANSWERS - 2) / 2
self.TOTAL_MAX_ROUNDS = 6 # 3 comparisons per student
self.COMPARISONS_IN_ROUND = math.ceil(self.NUMBER_OF_ANSWERS / 2)
# stop after lowest of total comparisons possible or 100 rounds worth of comparisons are complete
self.TOTAL_MAX_COMPARISONS = min(
self.COMPARISONS_IN_ROUND * self.TOTAL_MAX_ROUNDS,
self.NUMBER_OF_ANSWERS * self.MAX_COMPARSIONS
)
self.course = self.data.create_course()
self.instructor = self.data.create_instructor()
self.data.enrol_instructor(self.instructor, self.course)
self.assignment = self.data.create_assignment_in_comparison_period(
self.course, self.instructor,
number_of_comparisons=self.MAX_COMPARSIONS,
scoring_algorithm=ScoringAlgorithm.elo,
pairing_algorithm=PairingAlgorithm.adaptive_min_delta
)
self.students = []
self.answers = []
self.grade_by_answer_uuid = {}
actual_grades = numpy.random.normal(0.78, 0.1, self.NUMBER_OF_ANSWERS)
for grade in actual_grades:
student = self.data.create_normal_user()
self.data.enrol_student(student, self.course)
self.students.append(student)
answer = self.data.create_answer(self.assignment, student, with_score=False)
self.answers.append(answer)
self.grade_by_answer_uuid[answer.uuid] = grade
self.base_url = self._build_url(self.course.uuid, self.assignment.uuid)
db.session.commit()
def setUp(self):
super(ComparisonAPITests, self).setUp()
self.data = ComparisonTestData()
self.course = self.data.get_course()
self.assignment = self.data.get_assignments()[0]
self.base_url = self._build_url(self.course.uuid, self.assignment.uuid)
self.lti_data = LTITestData()
secondary_criterion = self.data.create_criterion(self.data.authorized_instructor)
AssignmentCriterionFactory(criterion=secondary_criterion, assignment=self.assignment)
db.session.commit()
def setUp(self):
super(ComPAIRXAPITestCase, self).setUp()
self.data = ComparisonTestData()
self.user = self.data.authorized_student
self.course = self.data.main_course
self.assignment = self.data.assignments[0]
self.answer1 = self.data.answers[0]
self.answer2 = self.data.answers[1]
self.comparison_example = self.data.comparisons_examples[0]
self.example_comparison = ComparisonFactory(
assignment=self.assignment,
user=self.user,
answer1_id=self.comparison_example.answer1.id,
answer2_id=self.comparison_example.answer2.id,
winner=None,
completed=False)
self.example_comparison_criteria = []
self.comparison = ComparisonFactory(assignment=self.assignment,
user=self.user,
answer1_id=self.answer1.id,
answer2_id=self.answer2.id,
winner=None,
completed=False)
self.comparison_criteria = []
for criterion in self.assignment.criteria:
self.example_comparison_criteria.append(
ComparisonCriterionFactory(
comparison=self.example_comparison,
criterion=criterion,
winner=WinningAnswer.answer1,
))
self.comparison_criteria.append(
ComparisonCriterionFactory(
comparison=self.comparison,
criterion=criterion,
winner=WinningAnswer.answer1,
))
db.session.commit()
self.maxDiff = None
def setUp(self):
# remove existing sqlite db if exists
self._delete_sqlite_db()
# get a new seed (needed for ConcurrentTestSuite so they don't produce the same results)
random.seed()
numpy.random.seed()
super(AlgorithmValidityTests, self).setUp()
self.data = ComparisonTestData()
self.MAX_COMPARSIONS = (NUMBER_OF_ANSWERS - 1) * (NUMBER_OF_ANSWERS - 2) / 2
self.TOTAL_MAX_ROUNDS = 50
self.COMPARISONS_IN_ROUND = math.ceil(NUMBER_OF_ANSWERS / 2)
# stop after lowest of total comparisons possible or 100 rounds worth of comparisons are complete
self.TOTAL_MAX_COMPARISONS = min(
self.COMPARISONS_IN_ROUND * self.TOTAL_MAX_ROUNDS,
NUMBER_OF_ANSWERS * self.MAX_COMPARSIONS
)
self.course = self.data.create_course()
self.instructor = self.data.create_instructor()
self.data.enrol_instructor(self.instructor, self.course)
self.assignment = self.data.create_assignment_in_comparison_period(
self.course, self.instructor,
number_of_comparisons=self.MAX_COMPARSIONS,
scoring_algorithm=AlgorithmValidityTests.SCORING_ALGORITHM,
pairing_algorithm=AlgorithmValidityTests.PAIRING_ALGORITHM)
self.students = []
self.answers = []
self.grade_by_answer_uuid = {}
actual_grades = ACTUAL_GRADES
if not actual_grades:
actual_grades = numpy.random.normal(0.78, 0.1, self.NUMBER_OF_ANSWERS)
for grade in actual_grades:
student = self.data.create_normal_user()
self.data.enrol_student(student, self.course)
self.students.append(student)
answer = self.data.create_answer(self.assignment, student, with_score=False)
self.answers.append(answer)
self.grade_by_answer_uuid[answer.uuid] = grade
self.base_url = self._build_url(self.course.uuid, self.assignment.uuid)
db.session.commit()
def setUp(self):
super(CoursesDuplicateComplexAPITests, self).setUp()
self.data = ComparisonTestData()
self.course = self.data.get_course()
# add start_date
self.course.start_date = datetime.datetime.utcnow()
self.course.end_date = datetime.datetime.utcnow() + datetime.timedelta(days=90)
db.session.commit()
self.url = '/api/courses/' + self.course.uuid + '/duplicate'
self.date_delta = datetime.timedelta(days=180)
self.expected = {
'name': 'duplicate course',
'year': 2015,
'term': 'Winter',
'start_date': (self.course.start_date + self.date_delta).isoformat() + 'Z',
'end_date': (self.course.end_date + self.date_delta).isoformat() + 'Z',
'assignments': []
}
for assignment in self.course.assignments:
if not assignment.active:
continue
assignment_data = {
'id': assignment.uuid,
'answer_start': (assignment.answer_start + self.date_delta).isoformat() + 'Z',
'answer_end': (assignment.answer_end + self.date_delta).isoformat() + 'Z'
}
if assignment.compare_start != None:
assignment_data['compare_start'] = (assignment.compare_start + self.date_delta).isoformat() + 'Z'
if assignment.compare_end != None:
assignment_data['compare_end'] = (assignment.compare_end + self.date_delta).isoformat() + 'Z'
self.expected['assignments'].append(assignment_data)
class ComparisonAPITests(ComPAIRAPITestCase):
def setUp(self):
super(ComparisonAPITests, self).setUp()
self.data = ComparisonTestData()
self.course = self.data.get_course()
self.assignment = self.data.get_assignments()[0]
self.base_url = self._build_url(self.course.uuid, self.assignment.uuid)
self.lti_data = LTITestData()
secondary_criterion = self.data.create_criterion(self.data.authorized_instructor)
AssignmentCriterionFactory(criterion=secondary_criterion, assignment=self.assignment)
db.session.commit()
def _build_url(self, course_uuid, assignment_uuid, tail=""):
url = '/api/courses/' + course_uuid + '/assignments/' + assignment_uuid + '/comparisons' + tail
return url
def _build_comparison_submit(self, winner_uuid, draft=False):
submit = {
'comparisons': [
{
'criterion_id': self.assignment.criteria[0].uuid,
'winner_id': winner_uuid,
'draft': draft
},
{
'criterion_id': self.assignment.criteria[1].uuid,
'winner_id': winner_uuid,
'draft': draft
}
]
}
return submit
def test_get_answer_pair_access_control(self):
# test login required
rv = self.client.get(self.base_url)
self.assert401(rv)
# test deny access to unenroled users
with self.login(self.data.get_unauthorized_student().username):
rv = self.client.get(self.base_url)
self.assert403(rv)
with self.login(self.data.get_unauthorized_instructor().username):
rv = self.client.get(self.base_url)
self.assert403(rv)
# enroled user from this point on
with self.login(self.data.get_authorized_student().username):
# test non-existent course
rv = self.client.get(self._build_url("9993929", self.assignment.uuid))
self.assert404(rv)
# test non-existent assignment
rv = self.client.get(self._build_url(self.course.uuid, "23902390"))
self.assert404(rv)
# no comparisons has been entered yet, assignment is not in comparing period
rv = self.client.get(self._build_url(
self.course.uuid, self.data.get_assignment_in_answer_period().uuid))
self.assert403(rv)
def test_submit_comparison_access_control(self):
# test login required
rv = self.client.post(
self.base_url,
data=json.dumps({}),
content_type='application/json')
self.assert401(rv)
# establish expected data by first getting an answer pair
with self.login(self.data.get_authorized_student().username):
rv = self.client.get(self.base_url)
self.assert200(rv)
# expected_comparisons = rv.json
comparison_submit = self._build_comparison_submit(rv.json['objects'][0]['answer1_id'])
# test deny access to unenroled users
with self.login(self.data.get_unauthorized_student().username):
rv = self.client.post(
self.base_url,
data=json.dumps(comparison_submit),
content_type='application/json')
self.assert403(rv)
with self.login(self.data.get_unauthorized_instructor().username):
rv = self.client.post(
self.base_url,
data=json.dumps(comparison_submit),
content_type='application/json')
self.assert403(rv)
# authorized user from this point
with self.login(self.data.get_authorized_student().username):
# test non-existent course
rv = self.client.post(
self._build_url("9999999", self.assignment.uuid),
data=json.dumps(comparison_submit),
content_type='application/json')
self.assert404(rv)
# test non-existent assignment
rv = self.client.post(
self._build_url(self.course.uuid, "9999999"),
data=json.dumps(comparison_submit),
content_type='application/json')
self.assert404(rv)
# test reject missing criteria
faulty_comparisons = copy.deepcopy(comparison_submit)
faulty_comparisons['comparisons'] = []
rv = self.client.post(
self.base_url,
data=json.dumps(faulty_comparisons),
content_type='application/json')
self.assert400(rv)
# test reject missing course criteria id
faulty_comparisons = copy.deepcopy(comparison_submit)
del faulty_comparisons['comparisons'][0]['criterion_id']
rv = self.client.post(
self.base_url,
data=json.dumps(faulty_comparisons),
content_type='application/json')
self.assert400(rv)
# test invalid criterion id
faulty_comparisons = copy.deepcopy(comparison_submit)
faulty_comparisons['comparisons'][0]['criterion_id'] = 3930230
rv = self.client.post(
self.base_url,
data=json.dumps(faulty_comparisons),
content_type='application/json')
self.assert400(rv)
# test invalid winner id
faulty_comparisons = copy.deepcopy(comparison_submit)
faulty_comparisons['comparisons'][0]['winner_id'] = 2382301
rv = self.client.post(
self.base_url,
data=json.dumps(faulty_comparisons),
content_type='application/json')
self.assert400(rv)
# test past grace period
self.assignment.compare_start = datetime.datetime.utcnow() - datetime.timedelta(days=7)
self.assignment.compare_end = datetime.datetime.utcnow() - datetime.timedelta(minutes=2)
db.session.add(self.assignment)
db.session.commit()
ok_comparisons = copy.deepcopy(comparison_submit)
rv = self.client.post(
self.base_url,
data=json.dumps(ok_comparisons),
content_type='application/json')
self.assert403(rv)
self.assertEqual("Assignment comparison deadline has passed.", rv.json['error'])
# test within grace period
self.assignment.compare_start = datetime.datetime.utcnow() - datetime.timedelta(days=7)
self.assignment.compare_end = datetime.datetime.utcnow() - datetime.timedelta(seconds=15)
db.session.add(self.assignment)
db.session.commit()
ok_comparisons = copy.deepcopy(comparison_submit)
rv = self.client.post(
self.base_url,
data=json.dumps(ok_comparisons),
content_type='application/json')
self.assert200(rv)
self.assignment.educators_can_compare = False
db.session.commit()
# instructors can access
with self.login(self.data.get_authorized_instructor().username):
rv = self.client.get(self.base_url)
self.assert403(rv)
self.assignment.educators_can_compare = True
db.session.commit()
rv = self.client.get(self.base_url)
self.assert200(rv)
# expected_comparisons = rv.json
comparison_submit = self._build_comparison_submit(rv.json['objects'][0]['answer1_id'])
ok_comparisons = copy.deepcopy(comparison_submit)
rv = self.client.post(
self.base_url,
data=json.dumps(ok_comparisons),
content_type='application/json')
self.assert200(rv)
self.assignment.educators_can_compare = False
db.session.commit()
# ta can access
with self.login(self.data.get_authorized_ta().username):
rv = self.client.get(self.base_url)
self.assert403(rv)
self.assignment.educators_can_compare = True
db.session.commit()
rv = self.client.get(self.base_url)
self.assert200(rv)
# expected_comparisons = rv.json
comparison_submit = self._build_comparison_submit(rv.json['objects'][0]['answer1_id'])
ok_comparisons = copy.deepcopy(comparison_submit)
rv = self.client.post(
self.base_url,
data=json.dumps(ok_comparisons),
content_type='application/json')
self.assert200(rv)
@mock.patch('compair.tasks.lti_outcomes.update_lti_course_grades.run')
@mock.patch('compair.tasks.lti_outcomes.update_lti_assignment_grades.run')
def test_get_and_submit_comparison(self, mocked_update_assignment_grades_run, mocked_update_course_grades_run):
lti_consumer = self.lti_data.lti_consumer
(lti_user_resource_link1, lti_user_resource_link2) = self.lti_data.setup_student_user_resource_links(
self.data.get_authorized_student(), self.course, self.assignment)
users = [self.data.get_authorized_student(), self.data.get_authorized_instructor(), self.data.get_authorized_ta()]
for user in users:
compared_answer_uuids = set()
valid_answer_uuids = set()
for answer in self.data.get_student_answers():
if answer.assignment.id == self.assignment.id and answer.user_id != user.id:
valid_answer_uuids.add(answer.uuid)
if user.id == self.data.get_authorized_student().id:
for comparison_example in self.data.comparisons_examples:
if comparison_example.assignment_id == self.assignment.id:
valid_answer_uuids.add(comparison_example.answer1_uuid)
valid_answer_uuids.add(comparison_example.answer2_uuid)
with self.login(user.username):
if user.id in [self.data.get_authorized_instructor().id, self.data.get_authorized_ta().id]:
self.assignment.educators_can_compare = False
db.session.commit()
# cannot compare answers unless educators_can_compare is set for assignment
rv = self.client.get(self.base_url)
self.assert403(rv)
self.assignment.educators_can_compare = True
db.session.commit()
current = 0
while len(valid_answer_uuids - compared_answer_uuids) > 0:
current += 1
if user.id == self.data.get_authorized_student().id:
course_grade = CourseGrade.get_user_course_grade(self.course, user).grade
assignment_grade = AssignmentGrade.get_user_assignment_grade(self.assignment, user).grade
# establish expected data by first getting an answer pair
rv = self.client.get(self.base_url)
self.assert200(rv)
actual_answer1_uuid = rv.json['objects'][0]['answer1_id']
actual_answer2_uuid = rv.json['objects'][0]['answer2_id']
self.assertIn(actual_answer1_uuid, valid_answer_uuids)
self.assertIn(actual_answer2_uuid, valid_answer_uuids)
self.assertNotEqual(actual_answer1_uuid, actual_answer2_uuid)
self.assertTrue(rv.json['new_pair'])
self.assertEqual(rv.json['current'], current)
# fetch again
rv = self.client.get(self.base_url)
self.assert200(rv)
expected_comparisons = rv.json
self.assertEqual(actual_answer1_uuid, rv.json['objects'][0]['answer1_id'])
self.assertEqual(actual_answer2_uuid, rv.json['objects'][0]['answer2_id'])
self.assertFalse(rv.json['new_pair'])
self.assertEqual(rv.json['current'], current)
# test draft post
comparison_submit = self._build_comparison_submit(rv.json['objects'][0]['answer1_id'], True)
rv = self.client.post(
self.base_url,
data=json.dumps(comparison_submit),
content_type='application/json')
self.assert200(rv)
actual_comparisons = rv.json['objects']
self._validate_comparison_submit(comparison_submit, actual_comparisons, expected_comparisons)
# test draft post (no answer id)
comparison_submit = self._build_comparison_submit(None)
rv = self.client.post(
self.base_url,
data=json.dumps(comparison_submit),
content_type='application/json')
self.assert200(rv)
actual_comparisons = rv.json['objects']
self._validate_comparison_submit(comparison_submit, actual_comparisons, expected_comparisons)
# test normal post
comparison_submit = self._build_comparison_submit(rv.json['objects'][0]['answer1_id'])
rv = self.client.post(
self.base_url,
data=json.dumps(comparison_submit),
content_type='application/json')
self.assert200(rv)
actual_comparisons = rv.json['objects']
compared_answer_uuids.add(actual_comparisons[0]['answer1_id'])
compared_answer_uuids.add(actual_comparisons[0]['answer2_id'])
self._validate_comparison_submit(comparison_submit, actual_comparisons, expected_comparisons)
# grades should increase for every comparison
if user.id == self.data.get_authorized_student().id:
new_course_grade = CourseGrade.get_user_course_grade(self.course, user)
new_assignment_grade = AssignmentGrade.get_user_assignment_grade(self.assignment, user)
self.assertGreater(new_course_grade.grade, course_grade)
self.assertGreater(new_assignment_grade.grade, assignment_grade)
mocked_update_course_grades_run.assert_called_once_with(
lti_consumer.id,
[(lti_user_resource_link1.lis_result_sourcedid, new_course_grade.id)]
)
mocked_update_course_grades_run.reset_mock()
mocked_update_assignment_grades_run.assert_called_once_with(
lti_consumer.id,
[(lti_user_resource_link2.lis_result_sourcedid, new_assignment_grade.id)]
)
mocked_update_assignment_grades_run.reset_mock()
else:
new_course_grade = CourseGrade.get_user_course_grade(self.course, user)
new_assignment_grade = AssignmentGrade.get_user_assignment_grade(self.assignment, user)
self.assertIsNone(new_course_grade)
self.assertIsNone(new_assignment_grade)
mocked_update_assignment_grades_run.assert_not_called()
mocked_update_course_grades_run.assert_not_called()
# Resubmit of same comparison should fail
rv = self.client.post(
self.base_url,
data=json.dumps(comparison_submit),
content_type='application/json')
self.assert400(rv)
# all answers has been compared by the user, errors out when trying to get another pair
rv = self.client.get(self.base_url)
self.assert400(rv)
def _validate_comparison_submit(self, comparison_submit, actual_comparisons, expected_comparisons):
self.assertEqual(
len(actual_comparisons), len(comparison_submit['comparisons']),
"The number of comparisons saved does not match the number sent")
for actual_comparison in actual_comparisons:
self.assertEqual(
expected_comparisons['objects'][0]['answer1_id'],
actual_comparison['answer1_id'],
"Expected and actual comparison answer1 id did not match")
self.assertEqual(
expected_comparisons['objects'][0]['answer2_id'],
actual_comparison['answer2_id'],
"Expected and actual comparison answer2 id did not match")
found_comparison = False
for expected_comparison in comparison_submit['comparisons']:
if expected_comparison['criterion_id'] != \
actual_comparison['criterion_id']:
continue
self.assertEqual(
expected_comparison['winner_id'],
actual_comparison['winner_id'],
"Expected and actual winner answer id did not match.")
found_comparison = True
self.assertTrue(
found_comparison,
"Actual comparison received contains a comparison that was not sent.")
def _submit_all_possible_comparisons_for_user(self, user_id):
example_winner_ids = []
example_loser_ids = []
for comparison_example in self.data.comparisons_examples:
if comparison_example.assignment_id == self.assignment.id:
comparisons = Comparison.create_new_comparison_set(self.assignment.id, user_id, False)
self.assertEqual(comparisons[0].answer1_id, comparison_example.answer1_id)
self.assertEqual(comparisons[0].answer2_id, comparison_example.answer2_id)
min_id = min([comparisons[0].answer1_id, comparisons[0].answer2_id])
max_id = max([comparisons[0].answer1_id, comparisons[0].answer2_id])
example_winner_ids.append(min_id)
example_loser_ids.append(max_id)
for comparison in comparisons:
comparison.completed = True
comparison.winner_id = min_id
db.session.add(comparison)
db.session.commit()
# self.login(username)
# calculate number of comparisons to do before user has compared all the pairs it can
num_eligible_answers = 0 # need to minus one to exclude the logged in user's own answer
for answer in self.data.get_student_answers():
if answer.assignment_id == self.assignment.id and answer.user_id != user_id:
num_eligible_answers += 1
# n - 1 possible pairs before all answers have been compared
num_possible_comparisons = num_eligible_answers - 1
winner_ids = []
loser_ids = []
for i in range(num_possible_comparisons):
comparisons = Comparison.create_new_comparison_set(self.assignment.id, user_id, False)
answer1_id = comparisons[0].answer1_id
answer2_id = comparisons[0].answer2_id
min_id = min([answer1_id, answer2_id])
max_id = max([answer1_id, answer2_id])
winner_ids.append(min_id)
loser_ids.append(max_id)
for comparison in comparisons:
comparison.completed = True
comparison.winner_id = min_id
db.session.add(comparison)
db.session.commit()
Comparison.calculate_scores(self.assignment.id)
return {
'comparisons': {
'winners': winner_ids, 'losers': loser_ids
},
'comparison_examples': {
'winners': example_winner_ids, 'losers': example_loser_ids
}
}
@mock.patch('random.shuffle')
def test_score_calculation(self, mock_shuffle):
"""
This is just a rough check on whether score calculations are correct. Answers
that has more wins should have the highest scores.
"""
# Make sure all answers are compared first
comparisons_auth = self._submit_all_possible_comparisons_for_user(
self.data.get_authorized_student().id)
comparisons_secondary = self._submit_all_possible_comparisons_for_user(
self.data.get_secondary_authorized_student().id)
loser_ids = comparisons_auth['comparisons']['losers'] + comparisons_secondary['comparisons']['losers']
winner_ids = comparisons_auth['comparisons']['winners'] + comparisons_secondary['comparisons']['winners']
# Count the number of wins each answer has had
num_wins_by_id = {}
for loser_id in loser_ids:
num_wins_by_id[loser_id] = num_wins_by_id.setdefault(loser_id, 0)
for winner_id in winner_ids:
num_wins = num_wins_by_id.setdefault(winner_id, 0)
num_wins_by_id[winner_id] = num_wins + 1
# Get the actual score calculated for each answer
answers = self.data.get_student_answers()
answer_scores = {}
for answer in answers:
if answer.assignment.id == self.assignment.id:
answer_scores[answer.id] = answer.scores[0].score
# Check that ranking by score and by wins match, this only works for low number of
# comparisons
sorted_expect_ranking = sorted(num_wins_by_id.items(), key=operator.itemgetter(0), reverse=True)
sorted_expect_ranking = sorted(sorted_expect_ranking, key=operator.itemgetter(1))
expected_ranking_by_wins = [answer_id for (answer_id, wins) in sorted_expect_ranking]
sorted_actual_ranking = sorted(answer_scores.items(), key=operator.itemgetter(1))
actual_ranking_by_scores = [answer_id for (answer_id, score) in sorted_actual_ranking]
self.assertSequenceEqual(actual_ranking_by_scores, expected_ranking_by_wins)
def test_comparison_count_matched_pairing(self):
# Make sure all answers are compared first
answer_ids = self._submit_all_possible_comparisons_for_user(
self.data.get_authorized_student().id)
answer_ids2 = self._submit_all_possible_comparisons_for_user(
self.data.get_secondary_authorized_student().id)
compared_ids = \
answer_ids['comparisons']['winners'] + answer_ids2['comparisons']['winners'] + \
answer_ids['comparisons']['losers'] + answer_ids2['comparisons']['losers'] + \
answer_ids['comparison_examples']['winners'] + answer_ids2['comparison_examples']['winners'] + \
answer_ids['comparison_examples']['losers'] + answer_ids2['comparison_examples']['losers']
# Just a simple test for now, make sure that answers with the smaller number of
# comparisons are matched up with each other
# Count number of comparisons done for each answer
num_comp_by_id = {}
for answer_id in compared_ids:
num_comp = num_comp_by_id.setdefault(answer_id, 0)
num_comp_by_id[answer_id] = num_comp + 1
comp_groups = {}
for answerId in num_comp_by_id:
count = num_comp_by_id[answerId]
comp_groups.setdefault(count, [])
comp_groups[count].append(answerId)
counts = sorted(comp_groups)
# get the answerIds with the lowest count of comparisons
possible_answer_ids = comp_groups[counts[0]]
if len(possible_answer_ids) < 2:
# if the lowest count group does not have enough to create a pair - add the next group
possible_answer_ids += comp_groups[counts[1]]
# Check that the 2 answers with 1 win gets returned
with self.login(self.data.get_authorized_student_with_no_answers().username):
rv = self.client.get(self.base_url)
self.assert200(rv)
answer1 = Answer.query.filter_by(uuid=rv.json['objects'][0]['answer1_id']).first()
answer2 = Answer.query.filter_by(uuid=rv.json['objects'][0]['answer2_id']).first()
self.assertIsNotNone(answer1)
self.assertIsNotNone(answer2)
self.assertIn(answer1.id, possible_answer_ids)
self.assertIn(answer2.id, possible_answer_ids)
def setUp(self):
super(ComPAIRLearningRecordTestCase, self).setUp()
self.data = ComparisonTestData()
self.lti_data = LTITestData()
self.user = self.data.authorized_student
self.setup_session_data(self.user)
self.course = self.data.main_course
self.lti_context = self.lti_data.create_context(
self.lti_data.lti_consumer,
compair_course_id=self.course.id,
lis_course_offering_sourcedid="sis_course_id",
lis_course_section_sourcedid="sis_section_id",
)
self.assignment = self.data.assignments[0]
self.criterion = self.assignment.criteria[0]
self.answer1 = self.data.answers[0]
self.answer2 = self.data.answers[1]
self.example_comparison = ComparisonFactory(assignment=self.assignment,
user=self.user,
answer1_id=self.answer1.id,
answer2_id=self.answer2.id,
winner=None,
completed=False)
self.example_comparison_criterion = ComparisonCriterionFactory(
comparison=self.example_comparison,
criterion=self.criterion,
winner=WinningAnswer.answer1,
)
self.comparison = ComparisonFactory(assignment=self.assignment,
user=self.user,
answer1_id=self.answer1.id,
answer2_id=self.answer2.id,
winner=None,
completed=False)
self.comparison_criterion = ComparisonCriterionFactory(
comparison=self.comparison,
criterion=self.criterion,
winner=WinningAnswer.answer1,
)
db.session.commit()
self.expected_caliper_course = {
'academicSession':
self.course.term,
'dateCreated':
self.course.created.replace(tzinfo=pytz.utc).isoformat(),
'dateModified':
self.course.modified.replace(tzinfo=pytz.utc).isoformat(),
'id':
"https://localhost:8888/app/course/" + self.course.uuid,
'name':
self.course.name,
'type':
'CourseOffering',
'extensions': {
'ltiContexts': [{
'context_id':
self.lti_context.context_id,
'oauth_consumer_key':
self.lti_data.lti_consumer.oauth_consumer_key,
'lis_course_offering_sourcedid':
"sis_course_id",
'lis_course_section_sourcedid':
"sis_section_id",
}]
}
}
self.expected_caliper_assignment = {
'name':
self.assignment.name,
'type':
'Assessment',
'dateCreated':
self.assignment.created.replace(tzinfo=pytz.utc).isoformat(),
'dateModified':
self.assignment.modified.replace(tzinfo=pytz.utc).isoformat(),
'dateToStartOn':
self.assignment.answer_start.replace(tzinfo=pytz.utc).isoformat(),
'description':
self.assignment.description,
'id':
"https://localhost:8888/app/course/" + self.course.uuid +
"/assignment/" + self.assignment.uuid,
'isPartOf':
self.expected_caliper_course,
'items': [{
'id':
"https://localhost:8888/app/course/" + self.course.uuid +
"/assignment/" + self.assignment.uuid + "/question",
'type':
'AssessmentItem'
}, {
'id':
"https://localhost:8888/app/course/" + self.course.uuid +
"/assignment/" + self.assignment.uuid +
"/comparison/question/1",
'type':
'AssessmentItem'
}, {
'id':
"https://localhost:8888/app/course/" + self.course.uuid +
"/assignment/" + self.assignment.uuid +
"/evaluation/question/1",
'type':
'AssessmentItem'
}, {
'id':
"https://localhost:8888/app/course/" + self.course.uuid +
"/assignment/" + self.assignment.uuid +
"/evaluation/question/2",
'type':
'AssessmentItem'
}, {
'id':
"https://localhost:8888/app/course/" + self.course.uuid +
"/assignment/" + self.assignment.uuid +
"/comparison/question/2",
'type':
'AssessmentItem'
}, {
'id':
"https://localhost:8888/app/course/" + self.course.uuid +
"/assignment/" + self.assignment.uuid +
"/evaluation/question/3",
'type':
'AssessmentItem'
}, {
'id':
"https://localhost:8888/app/course/" + self.course.uuid +
"/assignment/" + self.assignment.uuid +
"/evaluation/question/4",
'type':
'AssessmentItem'
}, {
'id':
"https://localhost:8888/app/course/" + self.course.uuid +
"/assignment/" + self.assignment.uuid +
"/comparison/question/3",
'type':
'AssessmentItem'
}, {
'id':
"https://localhost:8888/app/course/" + self.course.uuid +
"/assignment/" + self.assignment.uuid +
"/evaluation/question/5",
'type':
'AssessmentItem'
}, {
'id':
"https://localhost:8888/app/course/" + self.course.uuid +
"/assignment/" + self.assignment.uuid +
"/evaluation/question/6",
'type':
'AssessmentItem'
}],
}
self.expected_caliper_assignment_question = {
'name':
self.assignment.name,
'type':
'AssessmentItem',
'dateCreated':
self.assignment.created.replace(tzinfo=pytz.utc).isoformat(),
'dateModified':
self.assignment.modified.replace(tzinfo=pytz.utc).isoformat(),
'dateToStartOn':
self.assignment.answer_start.replace(tzinfo=pytz.utc).isoformat(),
'dateToSubmit':
self.assignment.answer_end.replace(tzinfo=pytz.utc).isoformat(),
'description':
self.assignment.description,
'id':
"https://localhost:8888/app/course/" + self.course.uuid +
"/assignment/" + self.assignment.uuid + "/question",
'isPartOf':
self.expected_caliper_assignment,
}
self.expected_caliper_answer1_attempt = {
'assignable':
self.expected_caliper_assignment_question,
'assignee':
self.get_compair_caliper_actor(self.answer1.user),
'id':
"https://localhost:8888/app/course/" + self.course.uuid +
"/assignment/" + self.assignment.uuid + "/question/attempt/" +
self.answer1.attempt_uuid,
'duration':
"PT05M00S",
'startedAtTime':
self.answer1.attempt_started.replace(tzinfo=pytz.utc).isoformat(),
'endedAtTime':
self.answer1.attempt_ended.replace(tzinfo=pytz.utc).isoformat(),
'type':
'Attempt'
}
self.expected_caliper_answer1 = {
'attempt':
self.expected_caliper_answer1_attempt,
'id':
"https://localhost:8888/app/course/" + self.course.uuid +
"/assignment/" + self.assignment.uuid + "/answer/" +
self.answer1.uuid,
'type':
'Response',
'dateCreated':
self.answer1.created.replace(tzinfo=pytz.utc).isoformat(),
'dateModified':
self.answer1.modified.replace(tzinfo=pytz.utc).isoformat(),
'extensions': {
'characterCount': len(self.answer1.content),
'content': self.answer1.content,
'isDraft': False,
'wordCount': 8,
'scoreDetails': {
'algorithm': self.assignment.scoring_algorithm.value,
'loses': 0,
'opponents': 0,
'rounds': 0,
'score': 5,
'wins': 0,
'criteria': {
"https://localhost:8888/app/criterion/" + self.criterion.uuid:
{
'loses': 0,
'opponents': 0,
'rounds': 0,
'score': 5,
'wins': 0
},
}
},
}
}
self.expected_caliper_answer2_attempt = {
'assignable':
self.expected_caliper_assignment_question,
'assignee':
self.get_compair_caliper_actor(self.answer2.user),
'id':
"https://localhost:8888/app/course/" + self.course.uuid +
"/assignment/" + self.assignment.uuid + "/question/attempt/" +
self.answer2.attempt_uuid,
'duration':
"PT05M00S",
'startedAtTime':
self.answer2.attempt_started.replace(tzinfo=pytz.utc).isoformat(),
'endedAtTime':
self.answer2.attempt_ended.replace(tzinfo=pytz.utc).isoformat(),
'type':
'Attempt'
}
self.expected_caliper_answer2 = {
'attempt':
self.expected_caliper_answer2_attempt,
'id':
"https://localhost:8888/app/course/" + self.course.uuid +
"/assignment/" + self.assignment.uuid + "/answer/" +
self.answer2.uuid,
'type':
'Response',
'dateCreated':
self.answer2.created.replace(tzinfo=pytz.utc).isoformat(),
'dateModified':
self.answer2.modified.replace(tzinfo=pytz.utc).isoformat(),
'extensions': {
'characterCount': len(self.answer2.content),
'content': self.answer2.content,
'isDraft': False,
'wordCount': 8,
'scoreDetails': {
'algorithm': self.assignment.scoring_algorithm.value,
'loses': 0,
'opponents': 0,
'rounds': 0,
'score': 5,
'wins': 0,
'criteria': {
"https://localhost:8888/app/criterion/" + self.criterion.uuid:
{
'loses': 0,
'opponents': 0,
'rounds': 0,
'score': 5,
'wins': 0
},
}
},
}
}
self.expected_caliper_comparison_question = {
'name':
"Assignment comparison #1",
'type':
'AssessmentItem',
'dateCreated':
self.assignment.created.replace(tzinfo=pytz.utc).isoformat(),
'dateModified':
self.assignment.modified.replace(tzinfo=pytz.utc).isoformat(),
'dateToStartOn':
self.assignment.answer_end.replace(tzinfo=pytz.utc).isoformat(),
'id':
"https://localhost:8888/app/course/" + self.course.uuid +
"/assignment/" + self.assignment.uuid + "/comparison/question/1",
'isPartOf':
self.expected_caliper_assignment,
}
self.expected_xapi_course = {
'id': "https://localhost:8888/app/course/" + self.course.uuid,
'definition': {
'type': 'http://adlnet.gov/expapi/activities/course',
'name': {
'en-US': self.course.name
}
},
'objectType': 'Activity'
}
self.expected_xapi_sis_course = {
'id': 'https://localhost:8888/course/' +
self.lti_context.lis_course_offering_sourcedid,
'objectType': 'Activity'
}
self.expected_xapi_sis_section = {
'id':
'https://localhost:8888/course/' +
self.lti_context.lis_course_offering_sourcedid + '/section/' +
self.lti_context.lis_course_section_sourcedid,
'objectType':
'Activity'
}
self.expected_xapi_assignment = {
'id':
"https://localhost:8888/app/course/" + self.course.uuid +
"/assignment/" + self.assignment.uuid,
'definition': {
'type': 'http://adlnet.gov/expapi/activities/assessment',
'name': {
'en-US': self.assignment.name
},
'description': {
'en-US': self.assignment.description
},
},
'objectType':
'Activity'
}
self.expected_xapi_assignment_question = {
'id':
"https://localhost:8888/app/course/" + self.course.uuid +
"/assignment/" + self.assignment.uuid + "/question",
'definition': {
'type': 'http://adlnet.gov/expapi/activities/question',
'name': {
'en-US': self.assignment.name
},
'description': {
'en-US': self.assignment.description
},
},
'objectType':
'Activity'
}
self.expected_xapi_answer1_attempt = {
'id':
"https://localhost:8888/app/course/" + self.course.uuid +
"/assignment/" + self.assignment.uuid + "/question/attempt/" +
self.answer1.attempt_uuid,
'definition': {
'type': 'http://adlnet.gov/expapi/activities/attempt',
'extensions': {
'http://id.tincanapi.com/extension/attempt': {
'duration':
"PT05M00S",
'startedAtTime':
self.answer1.attempt_started.replace(
tzinfo=pytz.utc).isoformat(),
'endedAtTime':
self.answer1.attempt_ended.replace(
tzinfo=pytz.utc).isoformat(),
}
}
},
'objectType':
'Activity'
}
self.expected_xapi_answer1 = {
'id':
"https://localhost:8888/app/course/" + self.course.uuid +
"/assignment/" + self.assignment.uuid + "/answer/" +
self.answer1.uuid,
'definition': {
'type': 'http://id.tincanapi.com/activitytype/solution',
'extensions': {
'http://id.tincanapi.com/extension/isDraft': False
}
},
'objectType':
'Activity'
}
self.expected_xapi_answer2_attempt = {
'id':
"https://localhost:8888/app/course/" + self.course.uuid +
"/assignment/" + self.assignment.uuid + "/question/attempt/" +
self.answer2.attempt_uuid,
'definition': {
'type': 'http://adlnet.gov/expapi/activities/attempt',
'extensions': {
'http://id.tincanapi.com/extension/attempt': {
'duration':
"PT05M00S",
'startedAtTime':
self.answer2.attempt_started.replace(
tzinfo=pytz.utc).isoformat(),
'endedAtTime':
self.answer2.attempt_ended.replace(
tzinfo=pytz.utc).isoformat(),
}
}
},
'objectType':
'Activity'
}
self.expected_xapi_answer2 = {
'id':
"https://localhost:8888/app/course/" + self.course.uuid +
"/assignment/" + self.assignment.uuid + "/answer/" +
self.answer2.uuid,
'definition': {
'type': 'http://id.tincanapi.com/activitytype/solution',
'extensions': {
'http://id.tincanapi.com/extension/isDraft': False
}
},
'objectType':
'Activity'
}
self.expected_xapi_comparison_question = {
'id':
"https://localhost:8888/app/course/" + self.course.uuid +
"/assignment/" + self.assignment.uuid + "/comparison/question/1",
'definition': {
'type': 'http://adlnet.gov/expapi/activities/question',
'name': {
'en-US': "Assignment comparison #1"
}
},
'objectType':
'Activity'
}
class CoursesDuplicateComplexAPITests(ComPAIRAPITestCase):
def setUp(self):
super(CoursesDuplicateComplexAPITests, self).setUp()
self.data = ComparisonTestData()
self.course = self.data.get_course()
# add start_date
self.course.start_date = datetime.datetime.utcnow()
self.course.end_date = datetime.datetime.utcnow() + datetime.timedelta(
days=90)
index = 0
for assignment in self.course.assignments:
assignment.answer_start = self.course.start_date + datetime.timedelta(
days=index)
assignment.answer_end = self.course.start_date + datetime.timedelta(
days=index + 7)
if assignment.compare_start != None:
assignment.compare_start = self.course.start_date + datetime.timedelta(
days=index + 14)
if assignment.compare_end != None:
assignment.compare_end = self.course.start_date + datetime.timedelta(
days=index + 21)
index += 1
db.session.commit()
self.url = '/api/courses/' + self.course.uuid + '/duplicate'
self.date_delta = datetime.timedelta(days=180)
self.expected = {
'name': 'duplicate course',
'year': 2015,
'term': 'Winter',
'sandbox': False,
'start_date':
(self.course.start_date + self.date_delta).isoformat() + 'Z',
'end_date':
(self.course.end_date + self.date_delta).isoformat() + 'Z',
'assignments': []
}
for assignment in self.course.assignments:
if not assignment.active:
continue
assignment_data = {
'id':
assignment.uuid,
'name':
assignment.name,
'answer_start':
(assignment.answer_start + self.date_delta).isoformat() + 'Z',
'answer_end':
(assignment.answer_end + self.date_delta).isoformat() + 'Z'
}
if assignment.compare_start != None:
assignment_data['compare_start'] = (
assignment.compare_start +
self.date_delta).isoformat() + 'Z'
if assignment.compare_end != None:
assignment_data['compare_end'] = (
assignment.compare_end + self.date_delta).isoformat() + 'Z'
self.expected['assignments'].append(assignment_data)
now = datetime.datetime.utcnow()
self.valid_course = self.data.create_course()
self.data.enrol_instructor(self.data.get_authorized_instructor(),
self.valid_course)
self.valid_assignment = self.data.create_assignment_in_answer_period(
self.valid_course, self.data.get_authorized_instructor())
self.valid_start_date = now.isoformat() + 'Z'
self.valid_end_date = (now +
datetime.timedelta(days=90)).isoformat() + 'Z'
self.invalid_end_date = (now -
datetime.timedelta(days=1)).isoformat() + 'Z'
self.valid_answer_start = now.isoformat() + 'Z'
self.valid_answer_end = (now +
datetime.timedelta(days=90)).isoformat() + 'Z'
self.invalid_answer_start = (
now - datetime.timedelta(days=1)).isoformat() + 'Z'
self.invalid_answer_end = (
now - datetime.timedelta(days=1)).isoformat() + 'Z'
self.invalid_answer_end2 = (
now + datetime.timedelta(days=91)).isoformat() + 'Z'
self.valid_compare_start = now.isoformat() + 'Z'
self.valid_compare_end = (
now + datetime.timedelta(days=90)).isoformat() + 'Z'
self.invalid_compare_start = (
now - datetime.timedelta(days=1)).isoformat() + 'Z'
self.invalid_compare_end = (
now - datetime.timedelta(days=1)).isoformat() + 'Z'
self.invalid_compare_end2 = (
now + datetime.timedelta(days=91)).isoformat() + 'Z'
self.validation_url = '/api/courses/' + self.valid_course.uuid + '/duplicate'
self.validate_expected_assignment = {
'id': self.valid_assignment.uuid,
'name': self.valid_assignment.name,
'answer_start': self.valid_answer_start,
'answer_end': self.valid_answer_end,
'compare_start': self.valid_compare_start,
'compare_end': self.valid_compare_end
}
self.validate_expected_course = {
'name': 'duplicate validation course',
'year': 2015,
'term': 'Winter',
'sandbox': False,
'start_date': self.valid_start_date,
'end_date': self.valid_end_date,
'assignments': [self.validate_expected_assignment]
}
def test_duplicate_course_complex(self):
original_course = self.data.get_course()
# test authorized user
with self.login(self.data.get_authorized_instructor().username):
# test valid expected
rv = self.client.post(self.validation_url,
data=json.dumps(
self.validate_expected_course),
content_type='application/json')
self.assert200(rv)
# test invalid course start/end dates
invalid_expected = self.validate_expected_course.copy()
invalid_expected['end_date'] = self.invalid_end_date
rv = self.client.post(self.validation_url,
data=json.dumps(invalid_expected),
content_type='application/json')
self.assert400(rv)
self.assertEqual(rv.json["title"], "Course Not Saved")
self.assertEqual(
rv.json["message"],
"Course end time must be after course start time.")
# test invalid assignment answer start
invalid_expected = self.validate_expected_course.copy()
invalid_expected['assignments'][
0] = self.validate_expected_assignment.copy()
invalid_expected['assignments'][0]['answer_start'] = None
rv = self.client.post(self.validation_url,
data=json.dumps(invalid_expected),
content_type='application/json')
self.assert400(rv)
self.assertEqual(rv.json["title"], "Course Not Saved")
self.assertEqual(
rv.json["message"],
"No answer period start time provided for assignment " +
self.valid_assignment.name + ".")
# test invalid assignment answer end
invalid_expected = self.validate_expected_course.copy()
invalid_expected['assignments'][
0] = self.validate_expected_assignment.copy()
invalid_expected['assignments'][0]['answer_end'] = None
rv = self.client.post(self.validation_url,
data=json.dumps(invalid_expected),
content_type='application/json')
self.assert400(rv)
self.assertEqual(rv.json["title"], "Course Not Saved")
self.assertEqual(
rv.json["message"],
"No answer period end time provided for assignment " +
self.valid_assignment.name + ".")
invalid_expected = self.validate_expected_course.copy()
invalid_expected['assignments'][
0] = self.validate_expected_assignment.copy()
invalid_expected['assignments'][0][
'answer_end'] = self.invalid_answer_end
rv = self.client.post(self.validation_url,
data=json.dumps(invalid_expected),
content_type='application/json')
self.assert400(rv)
self.assertEqual(rv.json["title"], "Course Not Saved")
self.assertEqual(
rv.json["message"],
"Answer period end time must be after the answer start time for assignment "
+ self.valid_assignment.name + ".")
invalid_expected = self.validate_expected_course.copy()
invalid_expected['assignments'][
0] = self.validate_expected_assignment.copy()
invalid_expected['assignments'][0][
'answer_end'] = self.invalid_answer_end2
rv = self.client.post(self.validation_url,
data=json.dumps(invalid_expected),
content_type='application/json')
self.assert400(rv)
self.assertEqual(rv.json["title"], "Course Not Saved")
self.assertEqual(
rv.json["message"],
"Answer period end time must be before the course end time for assignment "
+ self.valid_assignment.name + ".")
# test invalid assignment compare start
invalid_expected = self.validate_expected_course.copy()
invalid_expected['assignments'][
0] = self.validate_expected_assignment.copy()
invalid_expected['assignments'][0]['compare_start'] = None
rv = self.client.post(self.validation_url,
data=json.dumps(invalid_expected),
content_type='application/json')
self.assert400(rv)
self.assertEqual(rv.json["title"], "Course Not Saved")
self.assertEqual(
rv.json["message"],
"No compare period start time provided for assignment " +
self.valid_assignment.name + ".")
invalid_expected = self.validate_expected_course.copy()
invalid_expected['assignments'][
0] = self.validate_expected_assignment.copy()
invalid_expected['assignments'][0][
'compare_start'] = self.invalid_compare_start
rv = self.client.post(self.validation_url,
data=json.dumps(invalid_expected),
content_type='application/json')
self.assert400(rv)
self.assertEqual(rv.json["title"], "Course Not Saved")
self.assertEqual(
rv.json["message"],
"Compare period start time must be after the answer start time for assignment "
+ self.valid_assignment.name + ".")
# test invalid assignment compare end
invalid_expected = self.validate_expected_course.copy()
invalid_expected['assignments'][
0] = self.validate_expected_assignment.copy()
invalid_expected['assignments'][0]['compare_end'] = None
rv = self.client.post(self.validation_url,
data=json.dumps(invalid_expected),
content_type='application/json')
self.assert400(rv)
self.assertEqual(rv.json["title"], "Course Not Saved")
self.assertEqual(
rv.json["message"],
"No compare period end time provided for assignment " +
self.valid_assignment.name + ".")
invalid_expected = self.validate_expected_course.copy()
invalid_expected['assignments'][
0] = self.validate_expected_assignment.copy()
invalid_expected['assignments'][0][
'compare_end'] = self.invalid_compare_end
rv = self.client.post(self.validation_url,
data=json.dumps(invalid_expected),
content_type='application/json')
self.assert400(rv)
self.assertEqual(rv.json["title"], "Course Not Saved")
self.assertEqual(
rv.json["message"],
"Compare period end time must be after the compare start time for assignment "
+ self.valid_assignment.name + ".")
invalid_expected = self.validate_expected_course.copy()
invalid_expected['assignments'][
0] = self.validate_expected_assignment.copy()
invalid_expected['assignments'][0][
'compare_end'] = self.invalid_compare_end2
rv = self.client.post(self.validation_url,
data=json.dumps(invalid_expected),
content_type='application/json')
self.assert400(rv)
self.assertEqual(rv.json["title"], "Course Not Saved")
self.assertEqual(
rv.json["message"],
"Compare period end time must be before the course end time for assignment "
+ self.valid_assignment.name + ".")
# test deep copy assignments
rv = self.client.post(self.url,
data=json.dumps(self.expected),
content_type='application/json')
self.assert200(rv)
duplicate_course = Course.query.filter_by(
uuid=rv.json['id']).first()
self.assertIsNotNone(duplicate_course)
# verify course duplicated correctly
self.assertNotEqual(original_course.id, duplicate_course.id)
self.assertNotEqual(original_course.uuid, duplicate_course.uuid)
self.assertEqual(self.expected['name'], duplicate_course.name)
self.assertEqual(self.expected['year'], duplicate_course.year)
self.assertEqual(self.expected['term'], duplicate_course.term)
self.assertEqual(self.expected['sandbox'],
duplicate_course.sandbox)
self.assertEqual(self.expected['start_date'].replace('Z', ''),
rv.json['start_date'].replace('+00:00', ''))
self.assertEqual(self.expected['end_date'].replace('Z', ''),
rv.json['end_date'].replace('+00:00', ''))
# verify instructor added to duplicate course
user_course = UserCourse.query \
.filter_by(
user_id=self.data.get_authorized_instructor().id,
course_uuid=rv.json['id']
) \
.one_or_none()
self.assertIsNotNone(user_course)
original_assignments = original_course.assignments.all()
duplicate_assignments = duplicate_course.assignments.all()
self.assertEqual(len(original_assignments), 3)
self.assertEqual(len(original_assignments),
len(duplicate_assignments))
for index, original_assignment in enumerate(original_assignments):
duplicate_assignment = duplicate_assignments[index]
self.assertNotEqual(original_assignment.id,
duplicate_assignment.id)
self.assertNotEqual(original_assignment.uuid,
duplicate_assignment.uuid)
self.assertEqual(duplicate_course.id,
duplicate_assignment.course_id)
self.assertEqual(original_assignment.name,
duplicate_assignment.name)
self.assertEqual(original_assignment.description,
duplicate_assignment.description)
self.assertEqual(original_assignment.number_of_comparisons,
duplicate_assignment.number_of_comparisons)
self.assertEqual(original_assignment.students_can_reply,
duplicate_assignment.students_can_reply)
self.assertEqual(original_assignment.enable_self_evaluation,
duplicate_assignment.enable_self_evaluation)
self.assertEqual(original_assignment.pairing_algorithm,
duplicate_assignment.pairing_algorithm)
self.assertEqual(
original_assignment.answer_start,
(duplicate_assignment.answer_start - self.date_delta))
self.assertEqual(
original_assignment.answer_end,
(duplicate_assignment.answer_end - self.date_delta))
if original_assignment.compare_start != None:
self.assertEqual(
original_assignment.compare_start,
(duplicate_assignment.compare_start - self.date_delta))
else:
self.assertIsNone(duplicate_assignment.compare_start)
if original_assignment.compare_end != None:
self.assertEqual(
original_assignment.compare_end,
(duplicate_assignment.compare_end - self.date_delta))
else:
self.assertIsNone(duplicate_assignment.compare_end)
self.assertEqual(len(original_assignment.criteria), 1)
self.assertEqual(len(original_assignment.criteria),
len(duplicate_assignment.criteria))
for index, original_criteria in enumerate(
original_assignment.criteria):
duplicate_criteria = duplicate_assignment.criteria[index]
self.assertEqual(original_criteria.id,
duplicate_criteria.id)
original_comparison_examples = original_assignment.comparison_examples.all(
)
duplicate_comparison_examples = duplicate_assignment.comparison_examples.all(
)
if original_assignment.id in [1, 2]:
self.assertEqual(len(original_comparison_examples), 1)
else:
self.assertEqual(len(original_comparison_examples), 0)
self.assertEqual(len(original_comparison_examples),
len(duplicate_comparison_examples))
for index, original_comparison_example in enumerate(
original_comparison_examples):
duplicate_comparison_example = duplicate_comparison_examples[
index]
self.assertNotEqual(original_comparison_example.id,
duplicate_comparison_example.id)
self.assertNotEqual(
original_comparison_example.answer1_id,
duplicate_comparison_example.answer1_id)
self.assertNotEqual(
original_comparison_example.answer2_id,
duplicate_comparison_example.answer2_id)
self.assertEqual(
duplicate_assignment.id,
duplicate_comparison_example.assignment_id)
original_answer1 = original_comparison_example.answer1
duplicate_answer1 = duplicate_comparison_example.answer1
self.assertNotEqual(original_answer1.id,
duplicate_answer1.id)
self.assertEqual(duplicate_assignment.id,
duplicate_answer1.assignment_id)
self.assertEqual(original_answer1.content,
duplicate_answer1.content)
self.assertEqual(original_answer1.practice,
duplicate_answer1.practice)
self.assertEqual(original_answer1.active,
duplicate_answer1.active)
self.assertEqual(original_answer1.draft,
duplicate_answer1.draft)
original_answer2 = original_comparison_example.answer2
duplicate_answer2 = duplicate_comparison_example.answer2
self.assertNotEqual(original_answer2.id,
duplicate_answer2.id)
self.assertEqual(duplicate_assignment.id,
duplicate_answer2.assignment_id)
self.assertEqual(original_answer2.content,
duplicate_answer2.content)
self.assertEqual(original_answer2.practice,
original_answer2.practice)
self.assertEqual(original_answer2.active,
original_answer2.active)
self.assertEqual(original_answer2.draft,
original_answer2.draft)
def setUp(self):
super(CoursesDuplicateComplexAPITests, self).setUp()
self.data = ComparisonTestData()
self.course = self.data.get_course()
# add start_date
self.course.start_date = datetime.datetime.utcnow()
self.course.end_date = datetime.datetime.utcnow() + datetime.timedelta(
days=90)
index = 0
for assignment in self.course.assignments:
assignment.answer_start = self.course.start_date + datetime.timedelta(
days=index)
assignment.answer_end = self.course.start_date + datetime.timedelta(
days=index + 7)
if assignment.compare_start != None:
assignment.compare_start = self.course.start_date + datetime.timedelta(
days=index + 14)
if assignment.compare_end != None:
assignment.compare_end = self.course.start_date + datetime.timedelta(
days=index + 21)
index += 1
db.session.commit()
self.url = '/api/courses/' + self.course.uuid + '/duplicate'
self.date_delta = datetime.timedelta(days=180)
self.expected = {
'name': 'duplicate course',
'year': 2015,
'term': 'Winter',
'sandbox': False,
'start_date':
(self.course.start_date + self.date_delta).isoformat() + 'Z',
'end_date':
(self.course.end_date + self.date_delta).isoformat() + 'Z',
'assignments': []
}
for assignment in self.course.assignments:
if not assignment.active:
continue
assignment_data = {
'id':
assignment.uuid,
'name':
assignment.name,
'answer_start':
(assignment.answer_start + self.date_delta).isoformat() + 'Z',
'answer_end':
(assignment.answer_end + self.date_delta).isoformat() + 'Z'
}
if assignment.compare_start != None:
assignment_data['compare_start'] = (
assignment.compare_start +
self.date_delta).isoformat() + 'Z'
if assignment.compare_end != None:
assignment_data['compare_end'] = (
assignment.compare_end + self.date_delta).isoformat() + 'Z'
self.expected['assignments'].append(assignment_data)
now = datetime.datetime.utcnow()
self.valid_course = self.data.create_course()
self.data.enrol_instructor(self.data.get_authorized_instructor(),
self.valid_course)
self.valid_assignment = self.data.create_assignment_in_answer_period(
self.valid_course, self.data.get_authorized_instructor())
self.valid_start_date = now.isoformat() + 'Z'
self.valid_end_date = (now +
datetime.timedelta(days=90)).isoformat() + 'Z'
self.invalid_end_date = (now -
datetime.timedelta(days=1)).isoformat() + 'Z'
self.valid_answer_start = now.isoformat() + 'Z'
self.valid_answer_end = (now +
datetime.timedelta(days=90)).isoformat() + 'Z'
self.invalid_answer_start = (
now - datetime.timedelta(days=1)).isoformat() + 'Z'
self.invalid_answer_end = (
now - datetime.timedelta(days=1)).isoformat() + 'Z'
self.invalid_answer_end2 = (
now + datetime.timedelta(days=91)).isoformat() + 'Z'
self.valid_compare_start = now.isoformat() + 'Z'
self.valid_compare_end = (
now + datetime.timedelta(days=90)).isoformat() + 'Z'
self.invalid_compare_start = (
now - datetime.timedelta(days=1)).isoformat() + 'Z'
self.invalid_compare_end = (
now - datetime.timedelta(days=1)).isoformat() + 'Z'
self.invalid_compare_end2 = (
now + datetime.timedelta(days=91)).isoformat() + 'Z'
self.validation_url = '/api/courses/' + self.valid_course.uuid + '/duplicate'
self.validate_expected_assignment = {
'id': self.valid_assignment.uuid,
'name': self.valid_assignment.name,
'answer_start': self.valid_answer_start,
'answer_end': self.valid_answer_end,
'compare_start': self.valid_compare_start,
'compare_end': self.valid_compare_end
}
self.validate_expected_course = {
'name': 'duplicate validation course',
'year': 2015,
'term': 'Winter',
'sandbox': False,
'start_date': self.valid_start_date,
'end_date': self.valid_end_date,
'assignments': [self.validate_expected_assignment]
}
class AlgorithmValidityTests(ComPAIRAPITestCase):
SCORING_ALGORITHM = ScoringAlgorithm.elo
PAIRING_ALGORITHM = PairingAlgorithm.adaptive_min_delta
NUMBER_OF_ANSWERS = 100
REPORT_PATH = None
WINNER_SELECTOR = WinnerSelector.always_correct
CORRECT_RATE = 1.0
ACTUAL_GRADES = None
def create_app(self):
settings = test_app_settings.copy()
settings['SQLALCHEMY_DATABASE_URI'] = 'sqlite:///'+self._sqlite_db()
app = create_app(settings_override=settings)
return app
def setUp(self):
# remove existing sqlite db if exists
self._delete_sqlite_db()
# get a new seed (needed for ConcurrentTestSuite so they don't produce the same results)
random.seed()
numpy.random.seed()
super(AlgorithmValidityTests, self).setUp()
self.data = ComparisonTestData()
self.MAX_COMPARSIONS = (NUMBER_OF_ANSWERS - 1) * (NUMBER_OF_ANSWERS - 2) / 2
self.TOTAL_MAX_ROUNDS = 50
self.COMPARISONS_IN_ROUND = math.ceil(NUMBER_OF_ANSWERS / 2)
# stop after lowest of total comparisons possible or 100 rounds worth of comparisons are complete
self.TOTAL_MAX_COMPARISONS = min(
self.COMPARISONS_IN_ROUND * self.TOTAL_MAX_ROUNDS,
NUMBER_OF_ANSWERS * self.MAX_COMPARSIONS
)
self.course = self.data.create_course()
self.instructor = self.data.create_instructor()
self.data.enrol_instructor(self.instructor, self.course)
self.assignment = self.data.create_assignment_in_comparison_period(
self.course, self.instructor,
number_of_comparisons=self.MAX_COMPARSIONS,
scoring_algorithm=AlgorithmValidityTests.SCORING_ALGORITHM,
pairing_algorithm=AlgorithmValidityTests.PAIRING_ALGORITHM)
self.students = []
self.answers = []
self.grade_by_answer_uuid = {}
actual_grades = ACTUAL_GRADES
if not actual_grades:
actual_grades = numpy.random.normal(0.78, 0.1, self.NUMBER_OF_ANSWERS)
for grade in actual_grades:
student = self.data.create_normal_user()
self.data.enrol_student(student, self.course)
self.students.append(student)
answer = self.data.create_answer(self.assignment, student, with_score=False)
self.answers.append(answer)
self.grade_by_answer_uuid[answer.uuid] = grade
self.base_url = self._build_url(self.course.uuid, self.assignment.uuid)
db.session.commit()
def tearDown(self):
self._delete_sqlite_db()
def _sqlite_db(self):
return 'test_comparison'+str(os.getpid())+'.db'
def _delete_sqlite_db(self):
file_path = os.path.join(os.getcwd(), 'compair', self._sqlite_db())
if os.path.isfile(file_path):
try:
os.remove(file_path)
except Exception as e:
print(e)
def _decide_winner(self, answer1_uuid, answer2_uuid):
answer1_grade = self.grade_by_answer_uuid[answer1_uuid]
answer2_grade = self.grade_by_answer_uuid[answer2_uuid]
if AlgorithmValidityTests.WINNER_SELECTOR == WinnerSelector.always_correct:
return self.always_correct(answer1_grade, answer2_grade)
elif AlgorithmValidityTests.WINNER_SELECTOR == WinnerSelector.guessing:
return self.guessing()
elif AlgorithmValidityTests.WINNER_SELECTOR == WinnerSelector.correct_with_error:
return self.correct_with_error(answer1_grade, answer2_grade, AlgorithmValidityTests.CORRECT_RATE)
elif AlgorithmValidityTests.WINNER_SELECTOR == WinnerSelector.closely_matched_errors:
return self.closely_matched_errors(answer1_grade, answer2_grade, AlgorithmValidityTests.CORRECT_RATE)
else:
raise Exception()
def always_correct(self, value1, value2):
return self.correct_with_error(value1, value2, 1.0)
def correct_with_error(self, value1, value2, correct_rate):
if value1 == value2:
return self.guessing()
correct_answer = WinningAnswer.answer1 if value1 > value2 else WinningAnswer.answer2
incorrect_answer = WinningAnswer.answer1 if value1 < value2 else WinningAnswer.answer2
return correct_answer if random.random() <= correct_rate else incorrect_answer
def guessing(self):
return WinningAnswer.answer1 if random.random() <= 0.5 else WinningAnswer.answer2
def closely_matched_errors(self, value1, value2, sigma):
# make the actual values of answers fuzzy (represents perceived value errors)
fuzzy_value1 = numpy.random.normal(value1, sigma, 1)[0]
fuzzy_value2 = numpy.random.normal(value2, sigma, 1)[0]
# return the correct winner using fuzzy perceived values
return self.always_correct(fuzzy_value1, fuzzy_value2)
def _build_url(self, course_uuid, assignment_uuid, tail=""):
url = '/api/courses/' + course_uuid + '/assignments/' + assignment_uuid + '/comparisons' + tail
return url
def _build_comparison_submit(self, winner, draft=False):
submit = {
'comparison_criteria': [],
'draft': draft
}
for criterion in self.assignment.criteria:
submit['comparison_criteria'].append({
'criterion_id': criterion.uuid,
'winner': winner,
'content': None
})
return submit
def test_random_students_perform_comparisons(self):
self.student_comparison_count = {
student.id: 0 for student in self.students
}
comparison_count = 0
round_count = 0
results = []
while comparison_count < self.TOTAL_MAX_COMPARISONS:
# select a random student to answer
student = random.choice(self.students)
with self.login(student.username):
# perform selection algorithm
rv = self.client.get(self.base_url)
self.assert200(rv)
winner = self._decide_winner(rv.json['comparison']['answer1_id'], rv.json['comparison']['answer2_id'])
comparison_submit = self._build_comparison_submit(winner.value)
rv = self.client.post(self.base_url, data=json.dumps(comparison_submit), content_type='application/json')
self.assert200(rv)
comparison_count += 1
# remove students who have completed all comparisons
self.student_comparison_count[student.id] += 1
if self.student_comparison_count[student.id] >= self.MAX_COMPARSIONS:
indexes = [i for i, s in enumerate(self.students) if student.id == s.id]
del self.students[indexes[0]]
if comparison_count % self.COMPARISONS_IN_ROUND == 0:
round_count += 1
actual_grades = []
current_scores = []
for answer in self.answers:
answer_score = AnswerScore.query.filter_by(answer_id=answer.id).first()
if answer_score:
current_scores.append(answer_score.score)
actual_grades.append(self.grade_by_answer_uuid[answer.uuid])
r_value, pearsonr_p_value = pearsonr(actual_grades, current_scores)
#rho, spearmanr_p_value = spearmanr(actual_grades, current_scores)
#tau, kendalltau_p_value = kendalltau(actual_grades, current_scores)
results.append(str(r_value))
#results.append(str(rho))
#results.append(str(tau))
#print("Round {} ----------- pearsonr={} value=={} spearmanr={} value=={} kendalltau={} value=={}".format(
# round_count, r_value, pearsonr_p_value, rho, spearmanr_p_value, tau, kendalltau_p_value
#))
if r_value >= ACCEPTABLE_CORRELATION:
break
with open(AlgorithmValidityTests.REPORT_PATH, "a") as csvfile:
out = csv.writer(csvfile)
out.writerow(results)
class AlgorithmValidityTests(ComPAIRAPITestCase):
SCORING_ALGORITHM = ScoringAlgorithm.elo
PAIRING_ALGORITHM = PairingAlgorithm.adaptive_min_delta
NUMBER_OF_ANSWERS = 100
REPORT_PATH = None
WINNER_SELECTOR = WinnerSelector.always_correct
CORRECT_RATE = 1.0
ACTUAL_GRADES = None
def create_app(self):
settings = test_app_settings.copy()
settings['SQLALCHEMY_DATABASE_URI'] = 'sqlite:///' + self._sqlite_db()
app = create_app(settings_override=settings)
return app
def setUp(self):
# remove existing sqlite db if exists
self._delete_sqlite_db()
# get a new seed (needed for ConcurrentTestSuite so they don't produce the same results)
random.seed()
numpy.random.seed()
super(AlgorithmValidityTests, self).setUp()
self.data = ComparisonTestData()
self.MAX_COMPARSIONS = (NUMBER_OF_ANSWERS - 1) * (NUMBER_OF_ANSWERS -
2) / 2
self.TOTAL_MAX_ROUNDS = 50
self.COMPARISONS_IN_ROUND = math.ceil(NUMBER_OF_ANSWERS / 2)
# stop after lowest of total comparisons possible or 100 rounds worth of comparisons are complete
self.TOTAL_MAX_COMPARISONS = min(
self.COMPARISONS_IN_ROUND * self.TOTAL_MAX_ROUNDS,
NUMBER_OF_ANSWERS * self.MAX_COMPARSIONS)
self.course = self.data.create_course()
self.instructor = self.data.create_instructor()
self.data.enrol_instructor(self.instructor, self.course)
self.assignment = self.data.create_assignment_in_comparison_period(
self.course,
self.instructor,
number_of_comparisons=self.MAX_COMPARSIONS,
scoring_algorithm=AlgorithmValidityTests.SCORING_ALGORITHM,
pairing_algorithm=AlgorithmValidityTests.PAIRING_ALGORITHM)
self.students = []
self.answers = []
self.grade_by_answer_uuid = {}
actual_grades = ACTUAL_GRADES
if not actual_grades:
actual_grades = numpy.random.normal(0.78, 0.1,
self.NUMBER_OF_ANSWERS)
for grade in actual_grades:
student = self.data.create_normal_user()
self.data.enrol_student(student, self.course)
self.students.append(student)
answer = self.data.create_answer(self.assignment,
student,
with_score=False)
self.answers.append(answer)
self.grade_by_answer_uuid[answer.uuid] = grade
self.base_url = self._build_url(self.course.uuid, self.assignment.uuid)
db.session.commit()
def tearDown(self):
self._delete_sqlite_db()
def _sqlite_db(self):
return 'test_comparison' + str(os.getpid()) + '.db'
def _delete_sqlite_db(self):
file_path = os.path.join(os.getcwd(), 'compair', self._sqlite_db())
if os.path.isfile(file_path):
try:
os.remove(file_path)
except Exception as e:
print(e)
def _decide_winner(self, answer1_uuid, answer2_uuid):
answer1_grade = self.grade_by_answer_uuid[answer1_uuid]
answer2_grade = self.grade_by_answer_uuid[answer2_uuid]
if AlgorithmValidityTests.WINNER_SELECTOR == WinnerSelector.always_correct:
return self.always_correct(answer1_grade, answer2_grade)
elif AlgorithmValidityTests.WINNER_SELECTOR == WinnerSelector.guessing:
return self.guessing()
elif AlgorithmValidityTests.WINNER_SELECTOR == WinnerSelector.correct_with_error:
return self.correct_with_error(answer1_grade, answer2_grade,
AlgorithmValidityTests.CORRECT_RATE)
elif AlgorithmValidityTests.WINNER_SELECTOR == WinnerSelector.closely_matched_errors:
return self.closely_matched_errors(
answer1_grade, answer2_grade,
AlgorithmValidityTests.CORRECT_RATE)
else:
raise Exception()
def always_correct(self, value1, value2):
return self.correct_with_error(value1, value2, 1.0)
def correct_with_error(self, value1, value2, correct_rate):
if value1 == value2:
return self.guessing()
correct_answer = WinningAnswer.answer1 if value1 > value2 else WinningAnswer.answer2
incorrect_answer = WinningAnswer.answer1 if value1 < value2 else WinningAnswer.answer2
return correct_answer if random.random(
) <= correct_rate else incorrect_answer
def guessing(self):
return WinningAnswer.answer1 if random.random(
) <= 0.5 else WinningAnswer.answer2
def closely_matched_errors(self, value1, value2, sigma):
# make the actual values of answers fuzzy (represents perceived value errors)
fuzzy_value1 = numpy.random.normal(value1, sigma, 1)[0]
fuzzy_value2 = numpy.random.normal(value2, sigma, 1)[0]
# return the correct winner using fuzzy perceived values
return self.always_correct(fuzzy_value1, fuzzy_value2)
def _build_url(self, course_uuid, assignment_uuid, tail=""):
url = '/api/courses/' + course_uuid + '/assignments/' + assignment_uuid + '/comparisons' + tail
return url
def _build_comparison_submit(self, winner, draft=False):
submit = {'comparison_criteria': [], 'draft': draft}
for criterion in self.assignment.criteria:
submit['comparison_criteria'].append({
'criterion_id': criterion.uuid,
'winner': winner,
'content': None
})
return submit
def test_random_students_perform_comparisons(self):
self.student_comparison_count = {
student.id: 0
for student in self.students
}
comparison_count = 0
round_count = 0
results = []
while comparison_count < self.TOTAL_MAX_COMPARISONS:
# select a random student to answer
student = random.choice(self.students)
with self.login(student.username):
# perform selection algorithm
rv = self.client.get(self.base_url)
self.assert200(rv)
winner = self._decide_winner(
rv.json['comparison']['answer1_id'],
rv.json['comparison']['answer2_id'])
comparison_submit = self._build_comparison_submit(winner.value)
rv = self.client.post(self.base_url,
data=json.dumps(comparison_submit),
content_type='application/json')
self.assert200(rv)
comparison_count += 1
# remove students who have completed all comparisons
self.student_comparison_count[student.id] += 1
if self.student_comparison_count[
student.id] >= self.MAX_COMPARSIONS:
indexes = [
i for i, s in enumerate(self.students)
if student.id == s.id
]
del self.students[indexes[0]]
if comparison_count % self.COMPARISONS_IN_ROUND == 0:
round_count += 1
actual_grades = []
current_scores = []
for answer in self.answers:
answer_score = AnswerScore.query.filter_by(
answer_id=answer.id).first()
if answer_score:
current_scores.append(answer_score.score)
actual_grades.append(
self.grade_by_answer_uuid[answer.uuid])
r_value, pearsonr_p_value = pearsonr(actual_grades,
current_scores)
#rho, spearmanr_p_value = spearmanr(actual_grades, current_scores)
#tau, kendalltau_p_value = kendalltau(actual_grades, current_scores)
results.append(str(r_value))
#results.append(str(rho))
#results.append(str(tau))
#print("Round {} ----------- pearsonr={} value=={} spearmanr={} value=={} kendalltau={} value=={}".format(
# round_count, r_value, pearsonr_p_value, rho, spearmanr_p_value, tau, kendalltau_p_value
#))
if r_value >= ACCEPTABLE_CORRELATION:
break
with open(AlgorithmValidityTests.REPORT_PATH, "a") as csvfile:
out = csv.writer(csvfile)
out.writerow(results)
def setUp(self):
super(CoursesDuplicateComplexAPITests, self).setUp()
self.data = ComparisonTestData()
self.course = self.data.get_course()
# add start_date
self.course.start_date = datetime.datetime.utcnow()
self.course.end_date = datetime.datetime.utcnow() + datetime.timedelta(days=90)
index = 0
self.assertGreaterEqual(self.course.assignments.count(), 4) # make sure we have enough assignments to play with
for assignment in self.course.assignments:
assignment.answer_start = self.course.start_date + datetime.timedelta(days=index)
assignment.answer_end = self.course.start_date + datetime.timedelta(days=index+7)
if assignment.compare_start != None:
assignment.compare_start = self.course.start_date + datetime.timedelta(days=index+14)
if assignment.compare_end != None:
assignment.compare_end = self.course.start_date + datetime.timedelta(days=index+21)
# modify some assignments with self-evaluation
if index % 2 == 0:
assignment.enable_self_evaluation = True
else:
assignment.enable_self_evaluation = False
if index % 4 == 0:
assignment.self_eval_start = assignment.answer_end + datetime.timedelta(days=1)
assignment.self_eval_end = assignment.self_eval_start + datetime.timedelta(days=1)
assignment.self_eval_instructions = ''.join([choice(ascii_letters) for i in range(10)])
else:
assignment.self_eval_start = None
assignment.self_eval_end = None
assignment.self_eval_instructions = None
index+=1
db.session.commit()
self.url = '/api/courses/' + self.course.uuid + '/duplicate'
self.date_delta = datetime.timedelta(days=180)
self.expected = {
'name': 'duplicate course',
'year': 2015,
'term': 'Winter',
'sandbox': False,
'start_date': (self.course.start_date + self.date_delta).isoformat() + 'Z',
'end_date': (self.course.end_date + self.date_delta).isoformat() + 'Z',
'assignments': []
}
for assignment in self.course.assignments:
if not assignment.active:
continue
assignment_data = {
'id': assignment.uuid,
'name': assignment.name,
'answer_start': (assignment.answer_start + self.date_delta).isoformat() + 'Z',
'answer_end': (assignment.answer_end + self.date_delta).isoformat() + 'Z',
'enable_self_evaluation': assignment.enable_self_evaluation,
'self_eval_start': (assignment.self_eval_start + self.date_delta).isoformat() + 'Z' if assignment.self_eval_start else assignment.self_eval_start,
'self_eval_end': (assignment.self_eval_end + self.date_delta).isoformat() + 'Z' if assignment.self_eval_end else assignment.self_eval_end,
'self_eval_instructions': assignment.self_eval_instructions,
}
if assignment.compare_start != None:
assignment_data['compare_start'] = (assignment.compare_start + self.date_delta).isoformat() + 'Z'
if assignment.compare_end != None:
assignment_data['compare_end'] = (assignment.compare_end + self.date_delta).isoformat() + 'Z'
self.expected['assignments'].append(assignment_data)
now = datetime.datetime.utcnow()
self.valid_course = self.data.create_course()
self.data.enrol_instructor(self.data.get_authorized_instructor(), self.valid_course)
self.valid_assignment = self.data.create_assignment_in_answer_period(self.valid_course, self.data.get_authorized_instructor())
self.valid_start_date = now.isoformat() + 'Z'
self.valid_end_date = (now + datetime.timedelta(days=120)).isoformat() + 'Z'
self.invalid_end_date = (now - datetime.timedelta(days=1)).isoformat() + 'Z'
self.valid_answer_start = now.isoformat() + 'Z'
self.valid_answer_end = (now + datetime.timedelta(days=90)).isoformat() + 'Z'
self.invalid_answer_start = (now - datetime.timedelta(days=1)).isoformat() + 'Z'
self.invalid_answer_end = (now - datetime.timedelta(days=1)).isoformat() + 'Z'
self.invalid_answer_end2 = (now + datetime.timedelta(days=121)).isoformat() + 'Z'
self.valid_compare_start = now.isoformat() + 'Z'
self.valid_compare_end = (now + datetime.timedelta(days=90)).isoformat() + 'Z'
self.invalid_compare_start = (now - datetime.timedelta(days=1)).isoformat() + 'Z'
self.invalid_compare_end = (now - datetime.timedelta(days=1)).isoformat() + 'Z'
self.invalid_compare_end2 = (now + datetime.timedelta(days=121)).isoformat() + 'Z'
self.valid_self_eval_start = (now + datetime.timedelta(days=92)).isoformat() + 'Z'
self.valid_self_eval_end = (now + datetime.timedelta(days=120)).isoformat() + 'Z'
self.invalid_self_eval_start = now.isoformat() + 'Z' # = compare_start
self.invalid_self_eval_start2 = (now + datetime.timedelta(days=90)).isoformat() + 'Z' # = answer_end
self.invalid_self_eval_end = (now + datetime.timedelta(days=91)).isoformat() + 'Z' # < self_eval_start
self.invalid_self_eval_end2 = (now + datetime.timedelta(days=121)).isoformat() + 'Z' # > course_end
self.validation_url = '/api/courses/' + self.valid_course.uuid + '/duplicate'
self.validate_expected_assignment = {
'id': self.valid_assignment.uuid,
'name': self.valid_assignment.name,
'answer_start': self.valid_answer_start,
'answer_end': self.valid_answer_end,
'compare_start': self.valid_compare_start,
'compare_end': self.valid_compare_end
}
self.validate_expected_course = {
'name': 'duplicate validation course',
'year': 2015,
'term': 'Winter',
'sandbox': False,
'start_date': self.valid_start_date,
'end_date': self.valid_end_date,
'assignments': [self.validate_expected_assignment]
}
class AlgorithmValidityTests(ComPAIRAPITestCase):
# def create_app(self):
# settings = test_app_settings.copy()
# settings['SQLALCHEMY_DATABASE_URI'] = 'sqlite:///'+self._sqlite_db()
# app = create_app(settings_override=settings)
# return app
def setUp(self):
if SKIP_VALIDITY_TEST:
self.skipTest("scipy and numpy not installed. run `make deps`")
# remove existing sqlite db if exists
# self._delete_sqlite_db()
# TODO: Modify conditions to be more fuzzy (closely_matched_errors with 0.05 correct rate)
# Depends on results of research
super(AlgorithmValidityTests, self).setUp()
self.data = ComparisonTestData()
self.ACCEPTABLE_CORRELATION = 0.8
self.NUMBER_OF_ANSWERS = 40
self.WINNER_SELECTOR = WinnerSelector.always_correct
self.CORRECT_RATE = 1.0
self.MAX_COMPARSIONS = (self.NUMBER_OF_ANSWERS - 1) * (self.NUMBER_OF_ANSWERS - 2) / 2
self.TOTAL_MAX_ROUNDS = 6 # 3 comparisons per student
self.COMPARISONS_IN_ROUND = math.ceil(self.NUMBER_OF_ANSWERS / 2)
# stop after lowest of total comparisons possible or 100 rounds worth of comparisons are complete
self.TOTAL_MAX_COMPARISONS = min(
self.COMPARISONS_IN_ROUND * self.TOTAL_MAX_ROUNDS,
self.NUMBER_OF_ANSWERS * self.MAX_COMPARSIONS
)
self.course = self.data.create_course()
self.instructor = self.data.create_instructor()
self.data.enrol_instructor(self.instructor, self.course)
self.assignment = self.data.create_assignment_in_comparison_period(
self.course, self.instructor,
number_of_comparisons=self.MAX_COMPARSIONS,
scoring_algorithm=ScoringAlgorithm.elo,
pairing_algorithm=PairingAlgorithm.adaptive_min_delta
)
self.students = []
self.answers = []
self.grade_by_answer_uuid = {}
actual_grades = numpy.random.normal(0.78, 0.1, self.NUMBER_OF_ANSWERS)
for grade in actual_grades:
student = self.data.create_normal_user()
self.data.enrol_student(student, self.course)
self.students.append(student)
answer = self.data.create_answer(self.assignment, student, with_score=False)
self.answers.append(answer)
self.grade_by_answer_uuid[answer.uuid] = grade
self.base_url = self._build_url(self.course.uuid, self.assignment.uuid)
db.session.commit()
# def tearDown(self):
# self._delete_sqlite_db()
# def _sqlite_db(self):
# return 'test_comparison'+str(os.getpid())+'.db'
# def _delete_sqlite_db(self):
# file_path = os.path.join(os.getcwd(), 'compair', self._sqlite_db())
# if os.path.isfile(file_path):
# try:
# os.remove(file_path)
# except Exception as e:
# print(e)
def _decide_winner(self, answer1_uuid, answer2_uuid):
answer1_grade = self.grade_by_answer_uuid[answer1_uuid]
answer2_grade = self.grade_by_answer_uuid[answer2_uuid]
if self.WINNER_SELECTOR == WinnerSelector.always_correct:
return self.always_correct(answer1_grade, answer2_grade)
elif self.WINNER_SELECTOR == WinnerSelector.guessing:
return self.guessing()
elif self.WINNER_SELECTOR == WinnerSelector.correct_with_error:
return self.correct_with_error(answer1_grade, answer2_grade, self.CORRECT_RATE)
elif self.WINNER_SELECTOR == WinnerSelector.closely_matched_errors:
return self.closely_matched_errors(answer1_grade, answer2_grade, self.CORRECT_RATE)
else:
raise Exception()
def always_correct(self, value1, value2):
return self.correct_with_error(value1, value2, 1.0)
def correct_with_error(self, value1, value2, correct_rate):
if value1 == value2:
return self.guessing()
correct_answer = WinningAnswer.answer1 if value1 > value2 else WinningAnswer.answer2
incorrect_answer = WinningAnswer.answer1 if value1 < value2 else WinningAnswer.answer2
return correct_answer if random.random() <= correct_rate else incorrect_answer
def guessing(self):
return WinningAnswer.answer1 if random.random() <= 0.5 else WinningAnswer.answer2
def closely_matched_errors(self, value1, value2, sigma):
# make the actual values of answers fuzzy (represents perceived value errors)
fuzzy_value1 = numpy.random.normal(value1, sigma, 1)[0]
fuzzy_value2 = numpy.random.normal(value2, sigma, 1)[0]
# return the correct winner using fuzzy perceived values
return self.always_correct(fuzzy_value1, fuzzy_value2)
def _build_url(self, course_uuid, assignment_uuid, tail=""):
url = '/api/courses/' + course_uuid + '/assignments/' + assignment_uuid + '/comparisons' + tail
return url
def _build_comparison_submit(self, winner, draft=False):
submit = {
'comparison_criteria': [],
'draft': draft
}
for criterion in self.assignment.criteria:
submit['comparison_criteria'].append({
'criterion_id': criterion.uuid,
'winner': winner,
'content': None
})
return submit
def test_random_students_perform_comparisons(self):
self.student_comparison_count = {
student.id: 0 for student in self.students
}
comparison_count = 0
round_count = 0
r_value = None
while comparison_count < self.TOTAL_MAX_COMPARISONS:
# select a random student to answer
student = random.choice(self.students)
with self.login(student.username):
# perform selection algorithm
rv = self.client.get(self.base_url)
self.assert200(rv)
winner = self._decide_winner(rv.json['comparison']['answer1_id'], rv.json['comparison']['answer2_id'])
comparison_submit = self._build_comparison_submit(winner.value)
rv = self.client.post(self.base_url, data=json.dumps(comparison_submit), content_type='application/json')
self.assert200(rv)
comparison_count += 1
# remove students who have completed all comparisons
self.student_comparison_count[student.id] += 1
if self.student_comparison_count[student.id] >= self.MAX_COMPARSIONS:
indexes = [i for i, s in enumerate(self.students) if student.id == s.id]
del self.students[indexes[0]]
if comparison_count % self.COMPARISONS_IN_ROUND == 0:
round_count += 1
actual_grades = []
current_scores = []
for answer in self.answers:
answer_score = AnswerScore.query.filter_by(answer_id=answer.id).first()
if answer_score:
current_scores.append(answer_score.score)
actual_grades.append(self.grade_by_answer_uuid[answer.uuid])
r_value, pearsonr_p_value = pearsonr(actual_grades, current_scores)
if r_value >= self.ACCEPTABLE_CORRELATION:
break
self.assertGreaterEqual(r_value, self.ACCEPTABLE_CORRELATION)
self.assertLessEqual(round_count, self.TOTAL_MAX_ROUNDS)
class CoursesDuplicateComplexAPITests(ComPAIRAPITestCase):
def setUp(self):
super(CoursesDuplicateComplexAPITests, self).setUp()
self.data = ComparisonTestData()
self.course = self.data.get_course()
# add start_date
self.course.start_date = datetime.datetime.utcnow()
self.course.end_date = datetime.datetime.utcnow() + datetime.timedelta(days=90)
db.session.commit()
self.url = '/api/courses/' + self.course.uuid + '/duplicate'
self.date_delta = datetime.timedelta(days=180)
self.expected = {
'name': 'duplicate course',
'year': 2015,
'term': 'Winter',
'start_date': (self.course.start_date + self.date_delta).isoformat() + 'Z',
'end_date': (self.course.end_date + self.date_delta).isoformat() + 'Z',
'assignments': []
}
for assignment in self.course.assignments:
if not assignment.active:
continue
assignment_data = {
'id': assignment.uuid,
'answer_start': (assignment.answer_start + self.date_delta).isoformat() + 'Z',
'answer_end': (assignment.answer_end + self.date_delta).isoformat() + 'Z'
}
if assignment.compare_start != None:
assignment_data['compare_start'] = (assignment.compare_start + self.date_delta).isoformat() + 'Z'
if assignment.compare_end != None:
assignment_data['compare_end'] = (assignment.compare_end + self.date_delta).isoformat() + 'Z'
self.expected['assignments'].append(assignment_data)
def test_duplicate_course_complex(self):
original_course = self.data.get_course()
# test authorized user
with self.login(self.data.get_authorized_instructor().username):
rv = self.client.post(self.url, data=json.dumps(self.expected), content_type='application/json')
self.assert200(rv)
duplicate_course = Course.query.filter_by(uuid=rv.json['id']).first()
self.assertIsNotNone(duplicate_course)
# verify course duplicated correctly
self.assertNotEqual(original_course.id, duplicate_course.id)
self.assertNotEqual(original_course.uuid, duplicate_course.uuid)
self.assertEqual(self.expected['name'], duplicate_course.name)
self.assertEqual(self.expected['year'], duplicate_course.year)
self.assertEqual(self.expected['term'], duplicate_course.term)
self.assertEqual(self.expected['start_date'].replace('Z', ''), rv.json['start_date'].replace('+00:00', ''))
self.assertEqual(self.expected['end_date'].replace('Z', ''), rv.json['end_date'].replace('+00:00', ''))
self.assertEqual(original_course.description, duplicate_course.description)
# verify instructor added to duplicate course
user_course = UserCourse.query \
.filter_by(
user_id=self.data.get_authorized_instructor().id,
course_uuid=rv.json['id']
) \
.one_or_none()
self.assertIsNotNone(user_course)
original_assignments = original_course.assignments.all()
duplicate_assignments = duplicate_course.assignments.all()
self.assertEqual(len(original_assignments), 3)
self.assertEqual(len(original_assignments), len(duplicate_assignments))
for index, original_assignment in enumerate(original_assignments):
duplicate_assignment = duplicate_assignments[index]
self.assertNotEqual(original_assignment.id, duplicate_assignment.id)
self.assertNotEqual(original_assignment.uuid, duplicate_assignment.uuid)
self.assertEqual(duplicate_course.id, duplicate_assignment.course_id)
self.assertEqual(original_assignment.name, duplicate_assignment.name)
self.assertEqual(original_assignment.description, duplicate_assignment.description)
self.assertEqual(original_assignment.number_of_comparisons, duplicate_assignment.number_of_comparisons)
self.assertEqual(original_assignment.students_can_reply, duplicate_assignment.students_can_reply)
self.assertEqual(original_assignment.enable_self_evaluation, duplicate_assignment.enable_self_evaluation)
self.assertEqual(original_assignment.pairing_algorithm, duplicate_assignment.pairing_algorithm)
self.assertEqual(original_assignment.answer_start,
(duplicate_assignment.answer_start - self.date_delta))
self.assertEqual(original_assignment.answer_end,
(duplicate_assignment.answer_end - self.date_delta))
if original_assignment.compare_start != None:
self.assertEqual(original_assignment.compare_start,
(duplicate_assignment.compare_start - self.date_delta))
else:
self.assertIsNone(duplicate_assignment.compare_start)
if original_assignment.compare_end != None:
self.assertEqual(original_assignment.compare_end,
(duplicate_assignment.compare_end - self.date_delta))
else:
self.assertIsNone(duplicate_assignment.compare_end)
self.assertEqual(len(original_assignment.criteria), 1)
self.assertEqual(len(original_assignment.criteria), len(duplicate_assignment.criteria))
for index, original_criteria in enumerate(original_assignment.criteria):
duplicate_criteria = duplicate_assignment.criteria[index]
self.assertEqual(original_criteria.id, duplicate_criteria.id)
original_comparison_examples = original_assignment.comparison_examples.all()
duplicate_comparison_examples = duplicate_assignment.comparison_examples.all()
if original_assignment.id in [1,2]:
self.assertEqual(len(original_comparison_examples), 1)
else:
self.assertEqual(len(original_comparison_examples), 0)
self.assertEqual(len(original_comparison_examples), len(duplicate_comparison_examples))
for index, original_comparison_example in enumerate(original_comparison_examples):
duplicate_comparison_example = duplicate_comparison_examples[index]
self.assertNotEqual(original_comparison_example.id, duplicate_comparison_example.id)
self.assertNotEqual(original_comparison_example.answer1_id, duplicate_comparison_example.answer1_id)
self.assertNotEqual(original_comparison_example.answer2_id, duplicate_comparison_example.answer2_id)
self.assertEqual(duplicate_assignment.id, duplicate_comparison_example.assignment_id)
original_answer1 = original_comparison_example.answer1
duplicate_answer1 = duplicate_comparison_example.answer1
self.assertNotEqual(original_answer1.id, duplicate_answer1.id)
self.assertEqual(duplicate_assignment.id, duplicate_answer1.assignment_id)
self.assertEqual(original_answer1.content, duplicate_answer1.content)
self.assertEqual(original_answer1.practice, duplicate_answer1.practice)
self.assertEqual(original_answer1.active, duplicate_answer1.active)
self.assertEqual(original_answer1.draft, duplicate_answer1.draft)
original_answer2 = original_comparison_example.answer2
duplicate_answer2 = duplicate_comparison_example.answer2
self.assertNotEqual(original_answer2.id, duplicate_answer2.id)
self.assertEqual(duplicate_assignment.id, duplicate_answer2.assignment_id)
self.assertEqual(original_answer2.content, duplicate_answer2.content)
self.assertEqual(original_answer2.practice, original_answer2.practice)
self.assertEqual(original_answer2.active, original_answer2.active)
self.assertEqual(original_answer2.draft, original_answer2.draft)
def setUp(self):
super(UsersCourseStatusAPITests, self).setUp()
self.data = ComparisonTestData()
class UsersCourseStatusAPITests(ComPAIRAPITestCase):
def setUp(self):
super(UsersCourseStatusAPITests, self).setUp()
self.data = ComparisonTestData()
def _create_enough_answers_for_assignment(self, assignment):
course = assignment.course
for i in range(assignment.number_of_comparisons*2):
student = self.data.create_normal_user()
self.data.enrol_student(student, course)
self.data.create_answer(assignment, student)
def _submit_all_comparisons_for_assignment(self, assignment, user_id):
submit_count = 0
for comparison_example in assignment.comparison_examples:
comparisons = Comparison.create_new_comparison_set(assignment.id, user_id, False)
self.assertEqual(comparisons[0].answer1_id, comparison_example.answer1_id)
self.assertEqual(comparisons[0].answer2_id, comparison_example.answer2_id)
for comparison in comparisons:
comparison.completed = True
comparison.winner_id = min([comparisons[0].answer1_id, comparisons[0].answer2_id])
db.session.add(comparison)
submit_count += 1
db.session.commit()
for i in range(assignment.number_of_comparisons):
comparisons = Comparison.create_new_comparison_set(assignment.id, user_id, False)
for comparison in comparisons:
comparison.completed = True
comparison.winner_id = min([comparisons[0].answer1_id, comparisons[0].answer2_id])
db.session.add(comparison)
submit_count += 1
db.session.commit()
Comparison.calculate_scores(assignment.id)
return submit_count
def test_get_course_list(self):
# test login required
course = self.data.get_course()
url = '/api/users/courses/status?ids='+course.uuid
rv = self.client.get(url)
self.assert401(rv)
with self.login(self.data.get_authorized_student().username):
# ids missing
rv = self.client.get('/api/users/courses/status')
self.assert400(rv)
# empty ids
rv = self.client.get('/api/users/courses/status?ids=')
self.assert400(rv)
# empty course doesn't exists
rv = self.client.get('/api/users/courses/status?ids=999')
self.assert400(rv)
# test authorized student
rv = self.client.get(url)
self.assert200(rv)
self.assertEqual(1, len(rv.json['statuses']))
self.assertIn(course.uuid, rv.json['statuses'])
# test course with no assignments
course = self.data.create_course()
self.data.enrol_student(self.data.get_authorized_student(), course)
self.data.enrol_instructor(self.data.get_authorized_instructor(), course)
url = '/api/users/courses/status?ids='+course.uuid
rv = self.client.get(url)
self.assert200(rv)
self.assertEqual(1, len(rv.json['statuses']))
self.assertIn(course.uuid, rv.json['statuses'])
self.assertEqual(0, rv.json['statuses'][course.uuid]['incomplete_assignments'])
# test course with 2 assignment
self.data.create_assignment(course, self.data.get_authorized_instructor(),
datetime.datetime.now() + datetime.timedelta(days=7), datetime.datetime.now() + datetime.timedelta(days=14))
past_assignment = self.data.create_assignment(course, self.data.get_authorized_instructor(),
datetime.datetime.now() - datetime.timedelta(days=28), datetime.datetime.now() - datetime.timedelta(days=21))
past_assignment.compare_start = datetime.datetime.now() - datetime.timedelta(days=14)
past_assignment.compare_end = datetime.datetime.now() - datetime.timedelta(days=7)
db.session.commit()
rv = self.client.get(url)
self.assert200(rv)
self.assertEqual(1, len(rv.json['statuses']))
self.assertEqual(0, rv.json['statuses'][course.uuid]['incomplete_assignments'])
# test course with 3 assignment
# - 1 in answering period
answering_assignment = self.data.create_assignment_in_answer_period(course, self.data.get_authorized_instructor())
rv = self.client.get(url)
self.assert200(rv)
self.assertEqual(1, len(rv.json['statuses']))
self.assertEqual(1, rv.json['statuses'][course.uuid]['incomplete_assignments'])
# test course with 5 assignment
# - 1 in answering period
# - 1 in comparison_period
# - 1 in comparison_period with self evaluation
comparing_assignment = self.data.create_assignment_in_comparison_period(course, self.data.get_authorized_instructor())
comparing_assignment_self_eval = self.data.create_assignment_in_comparison_period(course, self.data.get_authorized_instructor())
comparing_assignment_self_eval.enable_self_evaluation = True
db.session.commit()
rv = self.client.get(url)
self.assert200(rv)
self.assertEqual(1, len(rv.json['statuses']))
self.assertEqual(3, rv.json['statuses'][course.uuid]['incomplete_assignments'])
# test course with 5 assignment
# - 1 in answering period (answered)
# - 1 in comparison_period
# - 1 in comparison_period with self evaluation
self.data.create_answer(answering_assignment, self.data.get_authorized_student())
rv = self.client.get(url)
self.assert200(rv)
self.assertEqual(1, len(rv.json['statuses']))
self.assertEqual(2, rv.json['statuses'][course.uuid]['incomplete_assignments'])
# test course with 5 assignment
# - 1 in answering period (answered)
# - 1 in comparison_period (fully compared)
# - 1 in comparison_period with self evaluation
self._create_enough_answers_for_assignment(comparing_assignment)
self._submit_all_comparisons_for_assignment(comparing_assignment, self.data.get_authorized_student().id)
rv = self.client.get(url)
self.assert200(rv)
self.assertEqual(1, len(rv.json['statuses']))
self.assertEqual(1, rv.json['statuses'][course.uuid]['incomplete_assignments'])
# test course with 5 assignment
# - 1 in answering period (answered)
# - 1 in comparison_period (fully compared)
# - 1 in comparison_period with self evaluation (fully compared, not self-evaulated)
self._create_enough_answers_for_assignment(comparing_assignment_self_eval)
self._submit_all_comparisons_for_assignment(comparing_assignment_self_eval, self.data.get_authorized_student().id)
rv = self.client.get(url)
self.assert200(rv)
self.assertEqual(1, len(rv.json['statuses']))
self.assertEqual(1, rv.json['statuses'][course.uuid]['incomplete_assignments'])
# test course with 5 assignment
# - 1 in answering period (answered)
# - 1 in comparison_period (fully compared)
# - 1 in comparison_period with self evaluation (fully compared and self-evaulated)
answer = self.data.create_answer(comparing_assignment_self_eval, self.data.get_authorized_student())
answer_comment = AnswerComment(
user_id=self.data.get_authorized_student().id,
answer_id=answer.id,
comment_type=AnswerCommentType.self_evaluation,
draft=False
)
db.session.add(answer_comment)
db.session.commit()
rv = self.client.get(url)
self.assert200(rv)
self.assertEqual(1, len(rv.json['statuses']))
self.assertEqual(0, rv.json['statuses'][course.uuid]['incomplete_assignments'])
class CoursesDuplicateComplexAPITests(ComPAIRAPITestCase):
def setUp(self):
super(CoursesDuplicateComplexAPITests, self).setUp()
self.data = ComparisonTestData()
self.course = self.data.get_course()
# add start_date
self.course.start_date = datetime.datetime.utcnow()
self.course.end_date = datetime.datetime.utcnow() + datetime.timedelta(days=90)
index = 0
self.assertGreaterEqual(self.course.assignments.count(), 4) # make sure we have enough assignments to play with
for assignment in self.course.assignments:
assignment.answer_start = self.course.start_date + datetime.timedelta(days=index)
assignment.answer_end = self.course.start_date + datetime.timedelta(days=index+7)
if assignment.compare_start != None:
assignment.compare_start = self.course.start_date + datetime.timedelta(days=index+14)
if assignment.compare_end != None:
assignment.compare_end = self.course.start_date + datetime.timedelta(days=index+21)
# modify some assignments with self-evaluation
if index % 2 == 0:
assignment.enable_self_evaluation = True
else:
assignment.enable_self_evaluation = False
if index % 4 == 0:
assignment.self_eval_start = assignment.answer_end + datetime.timedelta(days=1)
assignment.self_eval_end = assignment.self_eval_start + datetime.timedelta(days=1)
assignment.self_eval_instructions = ''.join([choice(ascii_letters) for i in range(10)])
else:
assignment.self_eval_start = None
assignment.self_eval_end = None
assignment.self_eval_instructions = None
index+=1
db.session.commit()
self.url = '/api/courses/' + self.course.uuid + '/duplicate'
self.date_delta = datetime.timedelta(days=180)
self.expected = {
'name': 'duplicate course',
'year': 2015,
'term': 'Winter',
'sandbox': False,
'start_date': (self.course.start_date + self.date_delta).isoformat() + 'Z',
'end_date': (self.course.end_date + self.date_delta).isoformat() + 'Z',
'assignments': []
}
for assignment in self.course.assignments:
if not assignment.active:
continue
assignment_data = {
'id': assignment.uuid,
'name': assignment.name,
'answer_start': (assignment.answer_start + self.date_delta).isoformat() + 'Z',
'answer_end': (assignment.answer_end + self.date_delta).isoformat() + 'Z',
'enable_self_evaluation': assignment.enable_self_evaluation,
'self_eval_start': (assignment.self_eval_start + self.date_delta).isoformat() + 'Z' if assignment.self_eval_start else assignment.self_eval_start,
'self_eval_end': (assignment.self_eval_end + self.date_delta).isoformat() + 'Z' if assignment.self_eval_end else assignment.self_eval_end,
'self_eval_instructions': assignment.self_eval_instructions,
}
if assignment.compare_start != None:
assignment_data['compare_start'] = (assignment.compare_start + self.date_delta).isoformat() + 'Z'
if assignment.compare_end != None:
assignment_data['compare_end'] = (assignment.compare_end + self.date_delta).isoformat() + 'Z'
self.expected['assignments'].append(assignment_data)
now = datetime.datetime.utcnow()
self.valid_course = self.data.create_course()
self.data.enrol_instructor(self.data.get_authorized_instructor(), self.valid_course)
self.valid_assignment = self.data.create_assignment_in_answer_period(self.valid_course, self.data.get_authorized_instructor())
self.valid_start_date = now.isoformat() + 'Z'
self.valid_end_date = (now + datetime.timedelta(days=120)).isoformat() + 'Z'
self.invalid_end_date = (now - datetime.timedelta(days=1)).isoformat() + 'Z'
self.valid_answer_start = now.isoformat() + 'Z'
self.valid_answer_end = (now + datetime.timedelta(days=90)).isoformat() + 'Z'
self.invalid_answer_start = (now - datetime.timedelta(days=1)).isoformat() + 'Z'
self.invalid_answer_end = (now - datetime.timedelta(days=1)).isoformat() + 'Z'
self.invalid_answer_end2 = (now + datetime.timedelta(days=121)).isoformat() + 'Z'
self.valid_compare_start = now.isoformat() + 'Z'
self.valid_compare_end = (now + datetime.timedelta(days=90)).isoformat() + 'Z'
self.invalid_compare_start = (now - datetime.timedelta(days=1)).isoformat() + 'Z'
self.invalid_compare_end = (now - datetime.timedelta(days=1)).isoformat() + 'Z'
self.invalid_compare_end2 = (now + datetime.timedelta(days=121)).isoformat() + 'Z'
self.valid_self_eval_start = (now + datetime.timedelta(days=92)).isoformat() + 'Z'
self.valid_self_eval_end = (now + datetime.timedelta(days=120)).isoformat() + 'Z'
self.invalid_self_eval_start = now.isoformat() + 'Z' # = compare_start
self.invalid_self_eval_start2 = (now + datetime.timedelta(days=90)).isoformat() + 'Z' # = answer_end
self.invalid_self_eval_end = (now + datetime.timedelta(days=91)).isoformat() + 'Z' # < self_eval_start
self.invalid_self_eval_end2 = (now + datetime.timedelta(days=121)).isoformat() + 'Z' # > course_end
self.validation_url = '/api/courses/' + self.valid_course.uuid + '/duplicate'
self.validate_expected_assignment = {
'id': self.valid_assignment.uuid,
'name': self.valid_assignment.name,
'answer_start': self.valid_answer_start,
'answer_end': self.valid_answer_end,
'compare_start': self.valid_compare_start,
'compare_end': self.valid_compare_end
}
self.validate_expected_course = {
'name': 'duplicate validation course',
'year': 2015,
'term': 'Winter',
'sandbox': False,
'start_date': self.valid_start_date,
'end_date': self.valid_end_date,
'assignments': [self.validate_expected_assignment]
}
def test_duplicate_course_complex(self):
original_course = self.data.get_course()
# test authorized user
with self.login(self.data.get_authorized_instructor().username):
# test valid expected
rv = self.client.post(self.validation_url, data=json.dumps(self.validate_expected_course), content_type='application/json')
self.assert200(rv)
# test invalid course start/end dates
invalid_expected = self.validate_expected_course.copy()
invalid_expected['end_date'] = self.invalid_end_date
rv = self.client.post(self.validation_url, data=json.dumps(invalid_expected), content_type='application/json')
self.assert400(rv)
self.assertEqual(rv.json["title"], "Course Not Saved")
self.assertEqual(rv.json["message"], "Course end time must be after course start time.")
# test invalid assignment answer start
invalid_expected = self.validate_expected_course.copy()
invalid_expected['assignments'][0] = self.validate_expected_assignment.copy()
invalid_expected['assignments'][0]['answer_start'] = None
rv = self.client.post(self.validation_url, data=json.dumps(invalid_expected), content_type='application/json')
self.assert400(rv)
self.assertEqual(rv.json["title"], "Course Not Saved")
self.assertEqual(rv.json["message"], "No answer period start time provided for assignment "+self.valid_assignment.name+".")
# test invalid assignment answer end
invalid_expected = self.validate_expected_course.copy()
invalid_expected['assignments'][0] = self.validate_expected_assignment.copy()
invalid_expected['assignments'][0]['answer_end'] = None
rv = self.client.post(self.validation_url, data=json.dumps(invalid_expected), content_type='application/json')
self.assert400(rv)
self.assertEqual(rv.json["title"], "Course Not Saved")
self.assertEqual(rv.json["message"], "No answer period end time provided for assignment "+self.valid_assignment.name+".")
invalid_expected = self.validate_expected_course.copy()
invalid_expected['assignments'][0] = self.validate_expected_assignment.copy()
invalid_expected['assignments'][0]['answer_end'] = self.invalid_answer_end
rv = self.client.post(self.validation_url, data=json.dumps(invalid_expected), content_type='application/json')
self.assert400(rv)
self.assertEqual(rv.json["title"], "Course Not Saved")
self.assertEqual(rv.json["message"], "Answer period end time must be after the answer start time for assignment "+self.valid_assignment.name+".")
invalid_expected = self.validate_expected_course.copy()
invalid_expected['assignments'][0] = self.validate_expected_assignment.copy()
invalid_expected['assignments'][0]['answer_end'] = self.invalid_answer_end2
rv = self.client.post(self.validation_url, data=json.dumps(invalid_expected), content_type='application/json')
self.assert400(rv)
self.assertEqual(rv.json["title"], "Course Not Saved")
self.assertEqual(rv.json["message"], "Answer period end time must be before the course end time for assignment "+self.valid_assignment.name+".")
# test invalid assignment compare start
invalid_expected = self.validate_expected_course.copy()
invalid_expected['assignments'][0] = self.validate_expected_assignment.copy()
invalid_expected['assignments'][0]['compare_start'] = None
rv = self.client.post(self.validation_url, data=json.dumps(invalid_expected), content_type='application/json')
self.assert400(rv)
self.assertEqual(rv.json["title"], "Course Not Saved")
self.assertEqual(rv.json["message"], "No compare period start time provided for assignment "+self.valid_assignment.name+".")
invalid_expected = self.validate_expected_course.copy()
invalid_expected['assignments'][0] = self.validate_expected_assignment.copy()
invalid_expected['assignments'][0]['compare_start'] = self.invalid_compare_start
rv = self.client.post(self.validation_url, data=json.dumps(invalid_expected), content_type='application/json')
self.assert400(rv)
self.assertEqual(rv.json["title"], "Course Not Saved")
self.assertEqual(rv.json["message"], "Compare period start time must be after the answer start time for assignment "+self.valid_assignment.name+".")
# test invalid assignment compare end
invalid_expected = self.validate_expected_course.copy()
invalid_expected['assignments'][0] = self.validate_expected_assignment.copy()
invalid_expected['assignments'][0]['compare_end'] = None
rv = self.client.post(self.validation_url, data=json.dumps(invalid_expected), content_type='application/json')
self.assert400(rv)
self.assertEqual(rv.json["title"], "Course Not Saved")
self.assertEqual(rv.json["message"], "No compare period end time provided for assignment "+self.valid_assignment.name+".")
invalid_expected = self.validate_expected_course.copy()
invalid_expected['assignments'][0] = self.validate_expected_assignment.copy()
invalid_expected['assignments'][0]['compare_end'] = self.invalid_compare_end
rv = self.client.post(self.validation_url, data=json.dumps(invalid_expected), content_type='application/json')
self.assert400(rv)
self.assertEqual(rv.json["title"], "Course Not Saved")
self.assertEqual(rv.json["message"], "Compare period end time must be after the compare start time for assignment "+self.valid_assignment.name+".")
invalid_expected = self.validate_expected_course.copy()
invalid_expected['assignments'][0] = self.validate_expected_assignment.copy()
invalid_expected['assignments'][0]['compare_end'] = self.invalid_compare_end2
rv = self.client.post(self.validation_url, data=json.dumps(invalid_expected), content_type='application/json')
self.assert400(rv)
self.assertEqual(rv.json["title"], "Course Not Saved")
self.assertEqual(rv.json["message"], "Compare period end time must be before the course end time for assignment "+self.valid_assignment.name+".")
# test invalid assignment self-eval start
invalid_expected = self.validate_expected_course.copy()
invalid_expected['assignments'][0] = self.validate_expected_assignment.copy()
invalid_expected['assignments'][0]['self_eval_start'] = None
invalid_expected['assignments'][0]['self_eval_end'] = self.valid_self_eval_end
rv = self.client.post(self.validation_url, data=json.dumps(invalid_expected), content_type='application/json')
self.assert400(rv)
self.assertEqual(rv.json["title"], "Course Not Saved")
self.assertEqual(rv.json["message"], "No self-evaluation start time provided for assignment "+self.valid_assignment.name+".")
invalid_expected = self.validate_expected_course.copy()
invalid_expected['assignments'][0] = self.validate_expected_assignment.copy()
invalid_expected['assignments'][0]['self_eval_start'] = self.invalid_self_eval_start
invalid_expected['assignments'][0]['self_eval_end'] = self.valid_self_eval_end
rv = self.client.post(self.validation_url, data=json.dumps(invalid_expected), content_type='application/json')
self.assert400(rv)
self.assertEqual(rv.json["title"], "Course Not Saved")
self.assertEqual(rv.json["message"], "Self-evaluation start time must be after the compare start time for assignment "+self.valid_assignment.name+".")
invalid_expected = self.validate_expected_course.copy()
invalid_expected['assignments'][0] = self.validate_expected_assignment.copy()
invalid_expected['assignments'][0]['compare_start'] = None
invalid_expected['assignments'][0]['compare_end'] = None
invalid_expected['assignments'][0]['self_eval_start'] = self.invalid_self_eval_start2
invalid_expected['assignments'][0]['self_eval_end'] = self.valid_self_eval_end
rv = self.client.post(self.validation_url, data=json.dumps(invalid_expected), content_type='application/json')
self.assert400(rv)
self.assertEqual(rv.json["title"], "Course Not Saved")
self.assertEqual(rv.json["message"], "Self-evaluation start time must be after the answer end time for assignment "+self.valid_assignment.name+".")
# test invalid assignment self-eval end
invalid_expected = self.validate_expected_course.copy()
invalid_expected['assignments'][0] = self.validate_expected_assignment.copy()
invalid_expected['assignments'][0]['self_eval_start'] = self.valid_self_eval_start
invalid_expected['assignments'][0]['self_eval_end'] = None
rv = self.client.post(self.validation_url, data=json.dumps(invalid_expected), content_type='application/json')
self.assert400(rv)
self.assertEqual(rv.json["title"], "Course Not Saved")
self.assertEqual(rv.json["message"], "No self-evaluation end time provided for assignment "+self.valid_assignment.name+".")
invalid_expected = self.validate_expected_course.copy()
invalid_expected['assignments'][0] = self.validate_expected_assignment.copy()
invalid_expected['assignments'][0]['self_eval_start'] = self.valid_self_eval_start
invalid_expected['assignments'][0]['self_eval_end'] = self.invalid_self_eval_end
rv = self.client.post(self.validation_url, data=json.dumps(invalid_expected), content_type='application/json')
self.assert400(rv)
self.assertEqual(rv.json["title"], "Course Not Saved")
self.assertEqual(rv.json["message"], "Self-evaluation end time must be after the self-evaluation start time for assignment "+self.valid_assignment.name+".")
invalid_expected = self.validate_expected_course.copy()
invalid_expected['assignments'][0] = self.validate_expected_assignment.copy()
invalid_expected['assignments'][0]['self_eval_start'] = self.valid_self_eval_start
invalid_expected['assignments'][0]['self_eval_end'] = self.invalid_self_eval_end2
rv = self.client.post(self.validation_url, data=json.dumps(invalid_expected), content_type='application/json')
self.assert400(rv)
self.assertEqual(rv.json["title"], "Course Not Saved")
self.assertEqual(rv.json["message"], "Self-evaluation end time must be before the course end time for assignment "+self.valid_assignment.name+".")
# test deep copy assignments
rv = self.client.post(self.url, data=json.dumps(self.expected), content_type='application/json')
self.assert200(rv)
duplicate_course = Course.query.filter_by(uuid=rv.json['id']).first()
self.assertIsNotNone(duplicate_course)
# verify course duplicated correctly
self.assertNotEqual(original_course.id, duplicate_course.id)
self.assertNotEqual(original_course.uuid, duplicate_course.uuid)
self.assertEqual(self.expected['name'], duplicate_course.name)
self.assertEqual(self.expected['year'], duplicate_course.year)
self.assertEqual(self.expected['term'], duplicate_course.term)
self.assertEqual(self.expected['sandbox'], duplicate_course.sandbox)
self.assertEqual(self.expected['start_date'].replace('Z', ''), rv.json['start_date'].replace('+00:00', ''))
self.assertEqual(self.expected['end_date'].replace('Z', ''), rv.json['end_date'].replace('+00:00', ''))
# verify instructor added to duplicate course
user_course = UserCourse.query \
.filter_by(
user_id=self.data.get_authorized_instructor().id,
course_uuid=rv.json['id']
) \
.one_or_none()
self.assertIsNotNone(user_course)
original_assignments = original_course.assignments.all()
duplicate_assignments = duplicate_course.assignments.all()
self.assertEqual(len(original_assignments), 5)
self.assertEqual(len(original_assignments), len(duplicate_assignments))
for index, original_assignment in enumerate(original_assignments):
duplicate_assignment = duplicate_assignments[index]
self.assertNotEqual(original_assignment.id, duplicate_assignment.id)
self.assertNotEqual(original_assignment.uuid, duplicate_assignment.uuid)
self.assertEqual(duplicate_course.id, duplicate_assignment.course_id)
self.assertEqual(original_assignment.name, duplicate_assignment.name)
self.assertEqual(original_assignment.description, duplicate_assignment.description)
self.assertEqual(original_assignment.number_of_comparisons, duplicate_assignment.number_of_comparisons)
self.assertEqual(original_assignment.students_can_reply, duplicate_assignment.students_can_reply)
self.assertEqual(original_assignment.enable_self_evaluation, duplicate_assignment.enable_self_evaluation)
self.assertEqual(original_assignment.pairing_algorithm, duplicate_assignment.pairing_algorithm)
self.assertEqual(original_assignment.answer_grade_weight, duplicate_assignment.answer_grade_weight)
self.assertEqual(original_assignment.comparison_grade_weight, duplicate_assignment.comparison_grade_weight)
self.assertEqual(original_assignment.self_evaluation_grade_weight, duplicate_assignment.self_evaluation_grade_weight)
self.assertEqual(original_assignment.enable_group_answers, duplicate_assignment.enable_group_answers)
self.assertEqual(original_assignment.scoring_algorithm, duplicate_assignment.scoring_algorithm)
self.assertEqual(original_assignment.peer_feedback_prompt, duplicate_assignment.peer_feedback_prompt)
self.assertEqual(original_assignment.educators_can_compare, duplicate_assignment.educators_can_compare)
self.assertEqual(original_assignment.rank_display_limit, duplicate_assignment.rank_display_limit)
self.assertEqual(original_assignment.answer_start,
(duplicate_assignment.answer_start - self.date_delta))
self.assertEqual(original_assignment.answer_end,
(duplicate_assignment.answer_end - self.date_delta))
if original_assignment.compare_start != None:
self.assertEqual(original_assignment.compare_start,
(duplicate_assignment.compare_start - self.date_delta))
else:
self.assertIsNone(duplicate_assignment.compare_start)
if original_assignment.compare_end != None:
self.assertEqual(original_assignment.compare_end,
(duplicate_assignment.compare_end - self.date_delta))
else:
self.assertIsNone(duplicate_assignment.compare_end)
self.assertEqual(len(original_assignment.criteria), 1)
self.assertEqual(len(original_assignment.criteria), len(duplicate_assignment.criteria))
for index, original_criteria in enumerate(original_assignment.criteria):
duplicate_criteria = duplicate_assignment.criteria[index]
self.assertEqual(original_criteria.id, duplicate_criteria.id)
self.assertEqual(original_assignment.enable_self_evaluation,
duplicate_assignment.enable_self_evaluation)
if original_assignment.self_eval_start:
self.assertEqual(original_assignment.self_eval_start,
(duplicate_assignment.self_eval_start - self.date_delta))
else:
self.assertIsNone(duplicate_assignment.self_eval_start)
if original_assignment.self_eval_end:
self.assertEqual(original_assignment.self_eval_end,
(duplicate_assignment.self_eval_end - self.date_delta))
else:
self.assertIsNone(duplicate_assignment.self_eval_end)
self.assertEqual(original_assignment.self_eval_instructions,
duplicate_assignment.self_eval_instructions)
original_comparison_examples = original_assignment.comparison_examples.all()
duplicate_comparison_examples = duplicate_assignment.comparison_examples.all()
if original_assignment.id in [1,2,3,4]:
self.assertEqual(len(original_comparison_examples), 1)
else:
self.assertEqual(len(original_comparison_examples), 0)
self.assertEqual(len(original_comparison_examples), len(duplicate_comparison_examples))
for index, original_comparison_example in enumerate(original_comparison_examples):
duplicate_comparison_example = duplicate_comparison_examples[index]
self.assertNotEqual(original_comparison_example.id, duplicate_comparison_example.id)
self.assertNotEqual(original_comparison_example.answer1_id, duplicate_comparison_example.answer1_id)
self.assertNotEqual(original_comparison_example.answer2_id, duplicate_comparison_example.answer2_id)
self.assertEqual(duplicate_assignment.id, duplicate_comparison_example.assignment_id)
original_answer1 = original_comparison_example.answer1
duplicate_answer1 = duplicate_comparison_example.answer1
self.assertNotEqual(original_answer1.id, duplicate_answer1.id)
self.assertEqual(duplicate_assignment.id, duplicate_answer1.assignment_id)
self.assertEqual(original_answer1.content, duplicate_answer1.content)
self.assertEqual(original_answer1.practice, duplicate_answer1.practice)
self.assertEqual(original_answer1.active, duplicate_answer1.active)
self.assertEqual(original_answer1.draft, duplicate_answer1.draft)
original_answer2 = original_comparison_example.answer2
duplicate_answer2 = duplicate_comparison_example.answer2
self.assertNotEqual(original_answer2.id, duplicate_answer2.id)
self.assertEqual(duplicate_assignment.id, duplicate_answer2.assignment_id)
self.assertEqual(original_answer2.content, duplicate_answer2.content)
self.assertEqual(original_answer2.practice, original_answer2.practice)
self.assertEqual(original_answer2.active, original_answer2.active)
self.assertEqual(original_answer2.draft, original_answer2.draft)
class ComparisonAPITests(ComPAIRAPITestCase):
def setUp(self):
super(ComparisonAPITests, self).setUp()
self.data = ComparisonTestData()
self.course = self.data.get_course()
self.assignment = self.data.get_assignments()[0]
self.base_url = self._build_url(self.course.uuid, self.assignment.uuid)
self.lti_data = LTITestData()
secondary_criterion = self.data.create_criterion(
self.data.authorized_instructor)
AssignmentCriterionFactory(criterion=secondary_criterion,
assignment=self.assignment)
db.session.commit()
def _build_url(self, course_uuid, assignment_uuid, tail=""):
url = '/api/courses/' + course_uuid + '/assignments/' + assignment_uuid + '/comparisons' + tail
return url
def _build_comparison_submit(self, winner, draft=False):
submit = {'comparison_criteria': [], 'draft': draft}
for criterion in self.assignment.criteria:
submit['comparison_criteria'].append({
'criterion_id': criterion.uuid,
'winner': winner,
'content': None
})
return submit
def test_get_answer_pair_access_control(self):
# test login required
rv = self.client.get(self.base_url)
self.assert401(rv)
# test deny access to unenroled users
with self.login(self.data.get_unauthorized_student().username):
rv = self.client.get(self.base_url)
self.assert403(rv)
with self.login(self.data.get_unauthorized_instructor().username):
rv = self.client.get(self.base_url)
self.assert403(rv)
# enroled user from this point on
with self.login(self.data.get_authorized_student().username):
# test non-existent course
rv = self.client.get(
self._build_url("9993929", self.assignment.uuid))
self.assert404(rv)
# test non-existent assignment
rv = self.client.get(self._build_url(self.course.uuid, "23902390"))
self.assert404(rv)
# no comparisons has been entered yet, assignment is not in comparing period
rv = self.client.get(
self._build_url(
self.course.uuid,
self.data.get_assignment_in_answer_period().uuid))
self.assert403(rv)
def test_submit_comparison_access_control(self):
# test login required
rv = self.client.post(self.base_url,
data=json.dumps({}),
content_type='application/json')
self.assert401(rv)
# establish expected data by first getting an answer pair
with self.login(self.data.get_authorized_student().username):
rv = self.client.get(self.base_url)
self.assert200(rv)
# expected_comparisons = rv.json
comparison_submit = self._build_comparison_submit(
WinningAnswer.answer1.value)
# test deny access to unenroled users
with self.login(self.data.get_unauthorized_student().username):
rv = self.client.post(self.base_url,
data=json.dumps(comparison_submit),
content_type='application/json')
self.assert403(rv)
with self.login(self.data.get_unauthorized_instructor().username):
rv = self.client.post(self.base_url,
data=json.dumps(comparison_submit),
content_type='application/json')
self.assert403(rv)
# authorized user from this point
with self.login(self.data.get_authorized_student().username):
# test non-existent course
rv = self.client.post(self._build_url("9999999",
self.assignment.uuid),
data=json.dumps(comparison_submit),
content_type='application/json')
self.assert404(rv)
# test non-existent assignment
rv = self.client.post(self._build_url(self.course.uuid, "9999999"),
data=json.dumps(comparison_submit),
content_type='application/json')
self.assert404(rv)
# test reject missing criteria
faulty_comparisons = copy.deepcopy(comparison_submit)
faulty_comparisons['comparison_criteria'] = []
rv = self.client.post(self.base_url,
data=json.dumps(faulty_comparisons),
content_type='application/json')
self.assert400(rv)
# test reject missing course criteria id
faulty_comparisons = copy.deepcopy(comparison_submit)
del faulty_comparisons['comparison_criteria'][0]['criterion_id']
rv = self.client.post(self.base_url,
data=json.dumps(faulty_comparisons),
content_type='application/json')
self.assert400(rv)
# test invalid criterion id
faulty_comparisons = copy.deepcopy(comparison_submit)
faulty_comparisons['comparison_criteria'][0][
'criterion_id'] = 3930230
rv = self.client.post(self.base_url,
data=json.dumps(faulty_comparisons),
content_type='application/json')
self.assert400(rv)
# test invalid winner
faulty_comparisons = copy.deepcopy(comparison_submit)
faulty_comparisons['comparison_criteria'][0]['winner'] = "2382301"
rv = self.client.post(self.base_url,
data=json.dumps(faulty_comparisons),
content_type='application/json')
self.assert400(rv)
# test past grace period
self.assignment.compare_start = datetime.datetime.utcnow(
) - datetime.timedelta(days=7)
self.assignment.compare_end = datetime.datetime.utcnow(
) - datetime.timedelta(minutes=2)
db.session.add(self.assignment)
db.session.commit()
ok_comparisons = copy.deepcopy(comparison_submit)
rv = self.client.post(self.base_url,
data=json.dumps(ok_comparisons),
content_type='application/json')
self.assert403(rv)
self.assertEqual("Comparison Not Saved", rv.json['title'])
self.assertEqual(
"Sorry, the comparison deadline has passed. No comparisons can be done after the deadline.",
rv.json['message'])
# test within grace period
self.assignment.compare_start = datetime.datetime.utcnow(
) - datetime.timedelta(days=7)
self.assignment.compare_end = datetime.datetime.utcnow(
) - datetime.timedelta(seconds=15)
db.session.add(self.assignment)
db.session.commit()
ok_comparisons = copy.deepcopy(comparison_submit)
rv = self.client.post(self.base_url,
data=json.dumps(ok_comparisons),
content_type='application/json')
self.assert200(rv)
self.assignment.educators_can_compare = False
db.session.commit()
# instructors can access
with self.login(self.data.get_authorized_instructor().username):
rv = self.client.get(self.base_url)
self.assert403(rv)
self.assignment.educators_can_compare = True
db.session.commit()
rv = self.client.get(self.base_url)
self.assert200(rv)
# expected_comparisons = rv.json
comparison_submit = self._build_comparison_submit(
WinningAnswer.answer1.value)
ok_comparisons = copy.deepcopy(comparison_submit)
rv = self.client.post(self.base_url,
data=json.dumps(ok_comparisons),
content_type='application/json')
self.assert200(rv)
self.assignment.educators_can_compare = False
db.session.commit()
# ta can access
with self.login(self.data.get_authorized_ta().username):
rv = self.client.get(self.base_url)
self.assert403(rv)
self.assignment.educators_can_compare = True
db.session.commit()
rv = self.client.get(self.base_url)
self.assert200(rv)
# expected_comparisons = rv.json
comparison_submit = self._build_comparison_submit(
WinningAnswer.answer1.value)
ok_comparisons = copy.deepcopy(comparison_submit)
rv = self.client.post(self.base_url,
data=json.dumps(ok_comparisons),
content_type='application/json')
self.assert200(rv)
@mock.patch('compair.tasks.lti_outcomes.update_lti_course_grades.run')
@mock.patch('compair.tasks.lti_outcomes.update_lti_assignment_grades.run')
def test_get_and_submit_comparison(self,
mocked_update_assignment_grades_run,
mocked_update_course_grades_run):
lti_consumer = self.lti_data.lti_consumer
(lti_user_resource_link1, lti_user_resource_link2
) = self.lti_data.setup_student_user_resource_links(
self.data.get_authorized_student(), self.course, self.assignment)
users = [
self.data.get_authorized_student(),
self.data.get_authorized_instructor(),
self.data.get_authorized_ta()
]
for user in users:
max_comparisons = 0
other_student_answers = 0
valid_answer_uuids = set()
for answer in self.data.get_student_answers():
if answer.assignment.id == self.assignment.id and answer.user_id != user.id:
other_student_answers += 1
valid_answer_uuids.add(answer.uuid)
max_comparisons = int(other_student_answers *
(other_student_answers - 1) / 2)
if user.id == self.data.get_authorized_student().id:
for comparison_example in self.data.comparisons_examples:
if comparison_example.assignment_id == self.assignment.id:
max_comparisons += 1
valid_answer_uuids.add(comparison_example.answer1_uuid)
valid_answer_uuids.add(comparison_example.answer2_uuid)
with self.login(user.username):
if user.id in [
self.data.get_authorized_instructor().id,
self.data.get_authorized_ta().id
]:
self.assignment.educators_can_compare = False
db.session.commit()
# cannot compare answers unless educators_can_compare is set for assignment
rv = self.client.get(self.base_url)
self.assert403(rv)
self.assignment.educators_can_compare = True
db.session.commit()
current = 0
while current < max_comparisons:
current += 1
if user.id == self.data.get_authorized_student().id:
course_grade = CourseGrade.get_user_course_grade(
self.course, user).grade
assignment_grade = AssignmentGrade.get_user_assignment_grade(
self.assignment, user).grade
# establish expected data by first getting an answer pair
rv = self.client.get(self.base_url)
self.assert200(rv)
actual_answer1_uuid = rv.json['comparison']['answer1_id']
actual_answer2_uuid = rv.json['comparison']['answer2_id']
self.assertIn(actual_answer1_uuid, valid_answer_uuids)
self.assertIn(actual_answer2_uuid, valid_answer_uuids)
self.assertNotEqual(actual_answer1_uuid,
actual_answer2_uuid)
self.assertTrue(rv.json['new_pair'])
self.assertEqual(rv.json['current'], current)
# fetch again
rv = self.client.get(self.base_url)
self.assert200(rv)
expected_comparison = rv.json['comparison']
self.assertEqual(actual_answer1_uuid,
rv.json['comparison']['answer1_id'])
self.assertEqual(actual_answer2_uuid,
rv.json['comparison']['answer2_id'])
self.assertFalse(rv.json['new_pair'])
self.assertEqual(rv.json['current'], current)
# test draft post
comparison_submit = self._build_comparison_submit(
WinningAnswer.answer1.value, True)
rv = self.client.post(self.base_url,
data=json.dumps(comparison_submit),
content_type='application/json')
self.assert200(rv)
actual_comparison = rv.json['comparison']
self._validate_comparison_submit(comparison_submit,
actual_comparison,
expected_comparison)
# test draft post (no winner)
comparison_submit = self._build_comparison_submit(None)
rv = self.client.post(self.base_url,
data=json.dumps(comparison_submit),
content_type='application/json')
self.assert200(rv)
actual_comparison = rv.json['comparison']
self._validate_comparison_submit(comparison_submit,
actual_comparison,
expected_comparison)
# test normal post
comparison_submit = self._build_comparison_submit(
WinningAnswer.answer1.value)
rv = self.client.post(self.base_url,
data=json.dumps(comparison_submit),
content_type='application/json')
self.assert200(rv)
actual_comparison = rv.json['comparison']
self._validate_comparison_submit(comparison_submit,
actual_comparison,
expected_comparison)
# grades should increase for every comparison
if user.id == self.data.get_authorized_student().id:
new_course_grade = CourseGrade.get_user_course_grade(
self.course, user)
new_assignment_grade = AssignmentGrade.get_user_assignment_grade(
self.assignment, user)
self.assertGreater(new_course_grade.grade,
course_grade)
self.assertGreater(new_assignment_grade.grade,
assignment_grade)
mocked_update_course_grades_run.assert_called_once_with(
lti_consumer.id,
[(lti_user_resource_link1.lis_result_sourcedid,
new_course_grade.id)])
mocked_update_course_grades_run.reset_mock()
mocked_update_assignment_grades_run.assert_called_once_with(
lti_consumer.id,
[(lti_user_resource_link2.lis_result_sourcedid,
new_assignment_grade.id)])
mocked_update_assignment_grades_run.reset_mock()
else:
new_course_grade = CourseGrade.get_user_course_grade(
self.course, user)
new_assignment_grade = AssignmentGrade.get_user_assignment_grade(
self.assignment, user)
self.assertIsNone(new_course_grade)
self.assertIsNone(new_assignment_grade)
mocked_update_assignment_grades_run.assert_not_called()
mocked_update_course_grades_run.assert_not_called()
# Resubmit of same comparison should fail
rv = self.client.post(self.base_url,
data=json.dumps(comparison_submit),
content_type='application/json')
self.assert400(rv)
# all answers has been compared by the user, errors out when trying to get another pair
rv = self.client.get(self.base_url)
self.assert400(rv)
def _validate_comparison_submit(self, comparison_submit, actual_comparison,
expected_comparison):
self.assertEqual(
len(actual_comparison['comparison_criteria']),
len(comparison_submit['comparison_criteria']),
"The number of comparisons saved does not match the number sent")
self.assertEqual(
expected_comparison['answer1_id'], actual_comparison['answer1_id'],
"Expected and actual comparison answer1 id did not match")
self.assertEqual(
expected_comparison['answer2_id'], actual_comparison['answer2_id'],
"Expected and actual comparison answer2 id did not match")
for actual_comparison_criterion in actual_comparison[
'comparison_criteria']:
found_comparison = False
for expected_comparison_criterion in comparison_submit[
'comparison_criteria']:
if expected_comparison_criterion[
'criterion_id'] != actual_comparison_criterion[
'criterion_id']:
continue
self.assertEqual(expected_comparison_criterion['winner'],
actual_comparison_criterion['winner'],
"Expected and actual winner did not match.")
found_comparison = True
self.assertTrue(
found_comparison,
"Actual comparison received contains a comparison that was not sent."
)
def _submit_all_possible_comparisons_for_user(self, user_id):
example_winner_ids = []
example_loser_ids = []
for comparison_example in self.data.comparisons_examples:
if comparison_example.assignment_id == self.assignment.id:
comparison = Comparison.create_new_comparison(
self.assignment.id, user_id, False)
self.assertEqual(comparison.answer1_id,
comparison_example.answer1_id)
self.assertEqual(comparison.answer2_id,
comparison_example.answer2_id)
min_id = min([comparison.answer1_id, comparison.answer2_id])
max_id = max([comparison.answer1_id, comparison.answer2_id])
example_winner_ids.append(min_id)
example_loser_ids.append(max_id)
comparison.completed = True
comparison.winner = WinningAnswer.answer1 if comparison.answer1_id < comparison.answer2_id else WinningAnswer.answer2
for comparison_criterion in comparison.comparison_criteria:
comparison_criterion.winner = comparison.winner
db.session.add(comparison)
db.session.commit()
# self.login(username)
# calculate number of comparisons to do before user has compared all the pairs it can
num_eligible_answers = 0 # need to minus one to exclude the logged in user's own answer
for answer in self.data.get_student_answers():
if answer.assignment_id == self.assignment.id and answer.user_id != user_id:
num_eligible_answers += 1
# n(n-1)/2 possible pairs before all answers have been compared
num_possible_comparisons = int(num_eligible_answers *
(num_eligible_answers - 1) / 2)
winner_ids = []
loser_ids = []
for i in range(num_possible_comparisons):
comparison = Comparison.create_new_comparison(
self.assignment.id, user_id, False)
min_id = min([comparison.answer1_id, comparison.answer2_id])
max_id = max([comparison.answer1_id, comparison.answer2_id])
winner_ids.append(min_id)
loser_ids.append(max_id)
comparison.completed = True
comparison.winner = WinningAnswer.answer1 if comparison.answer1_id < comparison.answer2_id else WinningAnswer.answer2
for comparison_criterion in comparison.comparison_criteria:
comparison_criterion.winner = comparison.winner
db.session.add(comparison)
db.session.commit()
Comparison.calculate_scores(self.assignment.id)
return {
'comparisons': {
'winners': winner_ids,
'losers': loser_ids
},
'comparison_examples': {
'winners': example_winner_ids,
'losers': example_loser_ids
}
}
@mock.patch('random.shuffle')
def test_score_calculation(self, mock_shuffle):
"""
This is just a rough check on whether score calculations are correct. Answers
that has more wins should have the highest scores.
"""
# Make sure all answers are compared first
comparisons_auth = self._submit_all_possible_comparisons_for_user(
self.data.get_authorized_student().id)
comparisons_secondary = self._submit_all_possible_comparisons_for_user(
self.data.get_secondary_authorized_student().id)
loser_ids = comparisons_auth['comparisons'][
'losers'] + comparisons_secondary['comparisons']['losers']
winner_ids = comparisons_auth['comparisons'][
'winners'] + comparisons_secondary['comparisons']['winners']
# Count the number of wins each answer has had
num_wins_by_id = {}
for loser_id in loser_ids:
num_wins_by_id[loser_id] = num_wins_by_id.setdefault(loser_id, 0)
for winner_id in winner_ids:
num_wins = num_wins_by_id.setdefault(winner_id, 0)
num_wins_by_id[winner_id] = num_wins + 1
# Get the actual score calculated for each answer
answers = self.data.get_student_answers()
answer_scores = {}
for answer in answers:
if answer.assignment.id == self.assignment.id:
answer_scores[answer.id] = answer.score.score
# Check that ranking by score and by wins match, this only works for low number of
# comparisons
sorted_expect_ranking = sorted(num_wins_by_id.items(),
key=operator.itemgetter(0))
sorted_expect_ranking = sorted(sorted_expect_ranking,
key=operator.itemgetter(1))
expected_ranking_by_wins = [
answer_id for (answer_id, wins) in sorted_expect_ranking
]
sorted_actual_ranking = sorted(answer_scores.items(),
key=operator.itemgetter(1))
actual_ranking_by_scores = [
answer_id for (answer_id, score) in sorted_actual_ranking
]
self.assertSequenceEqual(actual_ranking_by_scores,
expected_ranking_by_wins)
def test_comparison_count_matched_pairing(self):
# Make sure all answers are compared first
answer_ids = self._submit_all_possible_comparisons_for_user(
self.data.get_authorized_student().id)
answer_ids2 = self._submit_all_possible_comparisons_for_user(
self.data.get_secondary_authorized_student().id)
compared_ids = \
answer_ids['comparisons']['winners'] + answer_ids2['comparisons']['winners'] + \
answer_ids['comparisons']['losers'] + answer_ids2['comparisons']['losers'] + \
answer_ids['comparison_examples']['winners'] + answer_ids2['comparison_examples']['winners'] + \
answer_ids['comparison_examples']['losers'] + answer_ids2['comparison_examples']['losers']
# Just a simple test for now, make sure that answers with the smaller number of
# comparisons are matched up with each other
# Count number of comparisons done for each answer
num_comp_by_id = {}
for answer_id in compared_ids:
num_comp = num_comp_by_id.setdefault(answer_id, 0)
num_comp_by_id[answer_id] = num_comp + 1
comp_groups = {}
for answerId in num_comp_by_id:
count = num_comp_by_id[answerId]
comp_groups.setdefault(count, [])
comp_groups[count].append(answerId)
counts = sorted(comp_groups)
# get the answerIds with the lowest count of comparisons
possible_answer_ids = comp_groups[counts[0]]
if len(possible_answer_ids) < 2:
# if the lowest count group does not have enough to create a pair - add the next group
possible_answer_ids += comp_groups[counts[1]]
# Check that the 2 answers with 1 win gets returned
with self.login(
self.data.get_authorized_student_with_no_answers().username):
rv = self.client.get(self.base_url)
self.assert200(rv)
answer1 = Answer.query.filter_by(
uuid=rv.json['comparison']['answer1_id']).first()
answer2 = Answer.query.filter_by(
uuid=rv.json['comparison']['answer2_id']).first()
self.assertIsNotNone(answer1)
self.assertIsNotNone(answer2)
self.assertIn(answer1.id, possible_answer_ids)
self.assertIn(answer2.id, possible_answer_ids)
def test_comparison_winners(self):
# disable current criteria
for assignment_criterion in self.assignment.assignment_criteria:
assignment_criterion.active = False
# test 1 criterion: answer1, answer2 (draw not possible)
criterion = self.data.create_criterion(self.data.authorized_instructor)
AssignmentCriterionFactory(criterion=criterion,
assignment=self.assignment,
weight=100)
student = self.data.create_user(SystemRole.student)
self.data.enrol_student(student, self.course)
db.session.commit()
# test winner = answer1
with self.login(student.username):
rv = self.client.get(self.base_url)
self.assert200(rv)
comparison_submit = self._build_comparison_submit(
WinningAnswer.answer1.value)
rv = self.client.post(self.base_url,
data=json.dumps(comparison_submit),
content_type='application/json')
self.assert200(rv)
actual_comparison = rv.json['comparison']
self.assertEqual(actual_comparison['winner'],
WinningAnswer.answer1.value)
self.assertEqual(len(actual_comparison['comparison_criteria']), 1)
self.assertEqual(
actual_comparison['comparison_criteria'][0]['winner'],
WinningAnswer.answer1.value)
# test winner = answer2
rv = self.client.get(self.base_url)
self.assert200(rv)
comparison_submit = self._build_comparison_submit(
WinningAnswer.answer2.value)
rv = self.client.post(self.base_url,
data=json.dumps(comparison_submit),
content_type='application/json')
self.assert200(rv)
actual_comparison = rv.json['comparison']
self.assertEqual(actual_comparison['winner'],
WinningAnswer.answer2.value)
self.assertEqual(len(actual_comparison['comparison_criteria']), 1)
self.assertEqual(
actual_comparison['comparison_criteria'][0]['winner'],
WinningAnswer.answer2.value)
# test 2 criterion: answer1, answer2, draw
for assignment_criterion in self.assignment.assignment_criteria:
assignment_criterion.active = False
criterion1 = self.data.create_criterion(
self.data.authorized_instructor)
criterion2 = self.data.create_criterion(
self.data.authorized_instructor)
AssignmentCriterionFactory(criterion=criterion1,
assignment=self.assignment,
weight=100)
AssignmentCriterionFactory(criterion=criterion2,
assignment=self.assignment,
weight=100)
student = self.data.create_user(SystemRole.student)
self.data.enrol_student(student, self.course)
db.session.commit()
# test winner = answer1
with self.login(student.username):
rv = self.client.get(self.base_url)
self.assert200(rv)
comparison_submit = self._build_comparison_submit(
WinningAnswer.answer1.value)
rv = self.client.post(self.base_url,
data=json.dumps(comparison_submit),
content_type='application/json')
self.assert200(rv)
actual_comparison = rv.json['comparison']
self.assertEqual(actual_comparison['winner'],
WinningAnswer.answer1.value)
self.assertEqual(len(actual_comparison['comparison_criteria']), 2)
self.assertEqual(
actual_comparison['comparison_criteria'][0]['winner'],
WinningAnswer.answer1.value)
self.assertEqual(
actual_comparison['comparison_criteria'][1]['winner'],
WinningAnswer.answer1.value)
# test winner = answer2
rv = self.client.get(self.base_url)
self.assert200(rv)
comparison_submit = self._build_comparison_submit(
WinningAnswer.answer2.value)
rv = self.client.post(self.base_url,
data=json.dumps(comparison_submit),
content_type='application/json')
self.assert200(rv)
actual_comparison = rv.json['comparison']
self.assertEqual(actual_comparison['winner'],
WinningAnswer.answer2.value)
self.assertEqual(len(actual_comparison['comparison_criteria']), 2)
self.assertEqual(
actual_comparison['comparison_criteria'][0]['winner'],
WinningAnswer.answer2.value)
self.assertEqual(
actual_comparison['comparison_criteria'][1]['winner'],
WinningAnswer.answer2.value)
# test winner = draw
rv = self.client.get(self.base_url)
self.assert200(rv)
comparison_submit = self._build_comparison_submit(
WinningAnswer.answer1.value)
comparison_submit['comparison_criteria'][1][
'winner'] = WinningAnswer.answer2.value
rv = self.client.post(self.base_url,
data=json.dumps(comparison_submit),
content_type='application/json')
self.assert200(rv)
actual_comparison = rv.json['comparison']
self.assertEqual(actual_comparison['winner'],
WinningAnswer.draw.value)
self.assertEqual(len(actual_comparison['comparison_criteria']), 2)
self.assertEqual(
actual_comparison['comparison_criteria'][0]['winner'],
WinningAnswer.answer1.value)
self.assertEqual(
actual_comparison['comparison_criteria'][1]['winner'],
WinningAnswer.answer2.value)
# test 3 criterion: answer1, answer2, draw (with with different weights)
for assignment_criterion in self.assignment.assignment_criteria:
assignment_criterion.active = False
criterion1 = self.data.create_criterion(
self.data.authorized_instructor)
criterion2 = self.data.create_criterion(
self.data.authorized_instructor)
criterion3 = self.data.create_criterion(
self.data.authorized_instructor)
AssignmentCriterionFactory(criterion=criterion1,
assignment=self.assignment,
weight=200)
AssignmentCriterionFactory(criterion=criterion2,
assignment=self.assignment,
weight=100)
AssignmentCriterionFactory(criterion=criterion3,
assignment=self.assignment,
weight=100)
student = self.data.create_user(SystemRole.student)
self.data.enrol_student(student, self.course)
db.session.commit()
# test winner = answer1
with self.login(student.username):
rv = self.client.get(self.base_url)
self.assert200(rv)
comparison_submit = self._build_comparison_submit(
WinningAnswer.answer1.value)
comparison_submit['comparison_criteria'][1][
'winner'] = WinningAnswer.answer2.value
rv = self.client.post(self.base_url,
data=json.dumps(comparison_submit),
content_type='application/json')
self.assert200(rv)
actual_comparison = rv.json['comparison']
self.assertEqual(actual_comparison['winner'],
WinningAnswer.answer1.value)
self.assertEqual(len(actual_comparison['comparison_criteria']), 3)
self.assertEqual(
actual_comparison['comparison_criteria'][0]['winner'],
WinningAnswer.answer1.value)
self.assertEqual(
actual_comparison['comparison_criteria'][1]['winner'],
WinningAnswer.answer2.value)
self.assertEqual(
actual_comparison['comparison_criteria'][2]['winner'],
WinningAnswer.answer1.value)
# test winner = answer2
rv = self.client.get(self.base_url)
self.assert200(rv)
comparison_submit = self._build_comparison_submit(
WinningAnswer.answer2.value)
comparison_submit['comparison_criteria'][1][
'winner'] = WinningAnswer.answer1.value
rv = self.client.post(self.base_url,
data=json.dumps(comparison_submit),
content_type='application/json')
self.assert200(rv)
actual_comparison = rv.json['comparison']
self.assertEqual(actual_comparison['winner'],
WinningAnswer.answer2.value)
self.assertEqual(len(actual_comparison['comparison_criteria']), 3)
self.assertEqual(
actual_comparison['comparison_criteria'][0]['winner'],
WinningAnswer.answer2.value)
self.assertEqual(
actual_comparison['comparison_criteria'][1]['winner'],
WinningAnswer.answer1.value)
self.assertEqual(
actual_comparison['comparison_criteria'][2]['winner'],
WinningAnswer.answer2.value)
# test winner = draw
rv = self.client.get(self.base_url)
self.assert200(rv)
comparison_submit = self._build_comparison_submit(
WinningAnswer.answer1.value)
comparison_submit['comparison_criteria'][1][
'winner'] = WinningAnswer.answer2.value
comparison_submit['comparison_criteria'][2][
'winner'] = WinningAnswer.answer2.value
rv = self.client.post(self.base_url,
data=json.dumps(comparison_submit),
content_type='application/json')
self.assert200(rv)
actual_comparison = rv.json['comparison']
self.assertEqual(actual_comparison['winner'],
WinningAnswer.draw.value)
self.assertEqual(len(actual_comparison['comparison_criteria']), 3)
self.assertEqual(
actual_comparison['comparison_criteria'][0]['winner'],
WinningAnswer.answer1.value)
self.assertEqual(
actual_comparison['comparison_criteria'][1]['winner'],
WinningAnswer.answer2.value)
self.assertEqual(
actual_comparison['comparison_criteria'][2]['winner'],
WinningAnswer.answer2.value)
