def setUp(self):
super(JudgementAPITests, self).setUp()
self.data = JudgmentsTestData()
self.course = self.data.get_course()
self.question = self.data.get_questions()[0]
self.base_url = self._build_url(self.course.id, self.question.id)
self.answer_pair_url = self.base_url + '/pair'
def setUp(self):
super(JudgementAPITests, self).setUp()
self.data = JudgmentsTestData()
self.course = self.data.get_course()
self.question = self.data.get_questions()[0]
self.base_url = self._build_url(self.course.id, self.question.id)
self.answer_pair_url = self.base_url + '/pair'
class JudgementAPITests(ACJAPITestCase):
def setUp(self):
super(JudgementAPITests, self).setUp()
self.data = JudgmentsTestData()
self.course = self.data.get_course()
self.question = self.data.get_questions()[0]
self.base_url = self._build_url(self.course.id, self.question.id)
self.answer_pair_url = self.base_url + '/pair'
def _build_url(self, course_id, question_id, tail=""):
url = \
'/api/courses/' + str(course_id) + '/questions/' + str(question_id) + '/judgements' + \
tail
return url
def _build_judgement_submit(self, answerpair_id, winner_id):
submit = {
'answerpair_id': answerpair_id,
'judgements': [
{
'question_criterion_id': self.question.criteria[0].id,
'answer_id_winner': winner_id
}
]
}
return submit
def test_get_answer_pair_access_control(self):
# test login required
rv = self.client.get(self.answer_pair_url)
self.assert401(rv)
# test deny access to unenroled users
with self.login(self.data.get_unauthorized_student().username):
rv = self.client.get(self.answer_pair_url)
self.assert403(rv)
with self.login(self.data.get_unauthorized_instructor().username):
rv = self.client.get(self.answer_pair_url)
self.assert403(rv)
# enroled user from this point on
with self.login(self.data.get_authorized_student().username):
# test non-existent course
rv = self.client.get(self._build_url(9993929, self.question.id, '/pair'))
self.assert404(rv)
# test non-existent question
rv = self.client.get(self._build_url(self.course.id, 23902390, '/pair'))
self.assert404(rv)
# no judgements has been entered yet, question is not in judging period
rv = self.client.get(self._build_url(
self.course.id, self.data.get_question_in_answer_period().id, '/pair'))
self.assert403(rv)
def test_get_answer_pair_basic(self):
with self.login(self.data.get_authorized_student().username):
# no judgements has been entered yet
rv = self.client.get(self.answer_pair_url)
self.assert200(rv)
actual_answer_pair = rv.json
actual_answer1 = actual_answer_pair['answers'][0]
actual_answer2 = actual_answer_pair['answers'][1]
expected_answer_ids = [answer.id for answer in self.data.get_student_answers()]
# make sure that we actually got answers for the question we're targetting
self.assertIn(actual_answer1['id'], expected_answer_ids)
self.assertIn(actual_answer2['id'], expected_answer_ids)
def test_get_answer_pair_answer_exclusions_for_answers_with_no_scores(self):
"""
The user doing judgements should not see their own answer in a judgement.
Instructor and TA answers should not show up.
Answers cannot be paired with itself.
For answers that don't have a score yet, which means they're randomly matched up.
"""
with self.login(self.data.get_authorized_student().username):
excluded_student_answer = PostsForAnswers.query.join(Posts).filter(
Posts.users_id == self.data.get_authorized_student().id,
PostsForAnswers.questions_id == self.question.id).first()
self.assertTrue(excluded_student_answer, "Missing authorized student's answer.")
excluded_instructor_answer = PostsForAnswers.query.join(Posts).filter(
Posts.users_id == self.data.get_authorized_instructor().id,
PostsForAnswers.questions_id == self.question.id).first()
self.assertTrue(excluded_instructor_answer, "Missing instructor answer")
excluded_ta_answer = PostsForAnswers.query.join(Posts).filter(
Posts.users_id == self.data.get_authorized_ta().id,
PostsForAnswers.questions_id == self.question.id).first()
self.assertTrue(excluded_ta_answer, "Missing TA answer")
# no judgements has been entered yet, this tests the randomized pairing when no answers has
# scores, since it's randomized though, we'll have to run it lots of times to be sure
for i in range(50):
rv = self.client.get(self.answer_pair_url)
self.assert200(rv)
actual_answer_pair = rv.json
actual_answer1 = actual_answer_pair['answers'][0]
actual_answer2 = actual_answer_pair['answers'][1]
# exclude student's own answer
self.assertNotEqual(actual_answer1['id'], excluded_student_answer.id)
self.assertNotEqual(actual_answer2['id'], excluded_student_answer.id)
# exclude instructor answer
self.assertNotEqual(actual_answer1['id'], excluded_instructor_answer.id)
self.assertNotEqual(actual_answer2['id'], excluded_instructor_answer.id)
# exclude ta answer
self.assertNotEqual(actual_answer1['id'], excluded_ta_answer.id)
self.assertNotEqual(actual_answer2['id'], excluded_ta_answer.id)
# need a user with no answers submitted, otherwise pairs with the same answers
# won't be generated since we have too few answers
with self.login(self.data.get_authorized_student_with_no_answers().username):
for i in range(50):
rv = self.client.get(self.answer_pair_url)
self.assert200(rv)
# answer cannot be paired with itself
self.assertNotEqual(rv.json['answers'][0]['id'], rv.json['answers'][1]['id'])
def test_submit_judgement_access_control(self):
# test login required
rv = self.client.post(
self.base_url,
data=json.dumps({}),
content_type='application/json')
self.assert401(rv)
# establish expected data by first getting an answer pair
with self.login(self.data.get_authorized_student().username):
rv = self.client.get(self.answer_pair_url)
self.assert200(rv)
# expected_answer_pair = rv.json
judgement_submit = self._build_judgement_submit(rv.json['id'], rv.json['answers'][0]['id'])
# test deny access to unenroled users
with self.login(self.data.get_unauthorized_student().username):
rv = self.client.post(
self.base_url,
data=json.dumps(judgement_submit),
content_type='application/json')
self.assert403(rv)
with self.login(self.data.get_unauthorized_instructor().username):
rv = self.client.post(
self.base_url,
data=json.dumps(judgement_submit),
content_type='application/json')
self.assert403(rv)
# test deny access to non-students
with self.login(self.data.get_authorized_instructor().username):
rv = self.client.post(
self.base_url,
data=json.dumps(judgement_submit),
content_type='application/json')
self.assert403(rv)
# authorized user from this point
with self.login(self.data.get_authorized_student().username):
# test non-existent course
rv = self.client.post(
self._build_url(9999999, self.question.id),
data=json.dumps(judgement_submit),
content_type='application/json')
self.assert404(rv)
# test non-existent question
rv = self.client.post(
self._build_url(self.course.id, 9999999),
data=json.dumps(judgement_submit),
content_type='application/json')
self.assert404(rv)
# test reject missing criteria
faulty_judgements = copy.deepcopy(judgement_submit)
faulty_judgements['judgements'] = []
rv = self.client.post(
self.base_url,
data=json.dumps(faulty_judgements),
content_type='application/json')
self.assert400(rv)
# test reject missing course criteria id
faulty_judgements = copy.deepcopy(judgement_submit)
del faulty_judgements['judgements'][0]['question_criterion_id']
rv = self.client.post(
self.base_url,
data=json.dumps(faulty_judgements),
content_type='application/json')
self.assert400(rv)
# test reject missing winner
faulty_judgements = copy.deepcopy(judgement_submit)
del faulty_judgements['judgements'][0]['answer_id_winner']
rv = self.client.post(
self.base_url,
data=json.dumps(faulty_judgements),
content_type='application/json')
self.assert400(rv)
# test invalid criteria id
faulty_judgements = copy.deepcopy(judgement_submit)
faulty_judgements['judgements'][0]['question_criterion_id'] = 3930230
rv = self.client.post(
self.base_url,
data=json.dumps(faulty_judgements),
content_type='application/json')
self.assert400(rv)
# test invalid winner id
faulty_judgements = copy.deepcopy(judgement_submit)
faulty_judgements['judgements'][0]['answer_id_winner'] = 2382301
rv = self.client.post(
self.base_url,
data=json.dumps(faulty_judgements),
content_type='application/json')
self.assert400(rv)
# test invalid answer pair
faulty_judgements = copy.deepcopy(judgement_submit)
faulty_judgements['answerpair_id'] = 2382301
rv = self.client.post(
self.base_url,
data=json.dumps(faulty_judgements),
content_type='application/json')
self.assert404(rv)
def test_submit_judgement_basic(self):
with self.login(self.data.get_authorized_student().username):
# calculate number of judgements to do before user has judged all the pairs it can
num_eligible_answers = -1 # need to minus one to exclude the logged in user's own answer
for answer in self.data.get_student_answers():
if answer.question.id == self.question.id:
num_eligible_answers += 1
# n - 1 possible pairs before all answers have been judged
num_possible_judgements = num_eligible_answers - 1
winner_ids = []
for i in range(num_possible_judgements):
# establish expected data by first getting an answer pair
rv = self.client.get(self.answer_pair_url)
self.assert200(rv)
expected_answer_pair = rv.json
judgement_submit = self._build_judgement_submit(rv.json['id'], rv.json['answers'][0]['id'])
winner_ids.append(rv.json['answers'][0]['id'])
# test normal post
rv = self.client.post(
self.base_url,
data=json.dumps(judgement_submit),
content_type='application/json')
self.assert200(rv)
actual_judgements = rv.json['objects']
self._validate_judgement_submit(judgement_submit, actual_judgements, expected_answer_pair)
# Resubmit of same judgement should fail
rv = self.client.post(
self.base_url,
data=json.dumps(judgement_submit),
content_type='application/json')
self.assert400(rv)
# all answers has been judged by the user, errors out when trying to get another pair
rv = self.client.get(self.answer_pair_url)
self.assert400(rv)
def _validate_judgement_submit(self, judgement_submit, actual_judgements, expected_answer_pair):
self.assertEqual(
len(actual_judgements), len(judgement_submit['judgements']),
"The number of judgements saved does not match the number sent")
for actual_judgement in actual_judgements:
self.assertEqual(
expected_answer_pair['answers'][0]['id'],
actual_judgement['answerpairing']['answers_id1'],
"Expected and actual judgement answer1 id did not match")
self.assertEqual(
expected_answer_pair['answers'][1]['id'],
actual_judgement['answerpairing']['answers_id2'],
"Expected and actual judgement answer2 id did not match")
found_judgement = False
for expected_judgement in judgement_submit['judgements']:
if expected_judgement['question_criterion_id'] != \
actual_judgement['question_criterion']['id']:
continue
self.assertEqual(
expected_judgement['answer_id_winner'],
actual_judgement['answers_id_winner'],
"Expected and actual winner answer id did not match.")
found_judgement = True
self.assertTrue(
found_judgement,
"Actual judgement received contains a judgement that was not sent.")
def test_get_answer_pair_answer_exclusion_with_scored_answers(self):
"""
The user doing judgements should not see their own answer in a judgement.
Instructor and TA answers should not show up.
Answers cannot be paired with itself.
Scored answer pairing means answers should be matched up to similar scores.
"""
# Make sure all answers are judged first
self._submit_all_possible_judgements_for_user(
self.data.get_authorized_student().id)
self._submit_all_possible_judgements_for_user(
self.data.get_secondary_authorized_student().id)
with self.login(self.data.get_authorized_student_with_no_answers().username):
excluded_instructor_answer = PostsForAnswers.query.join(Posts).filter(
Posts.users_id == self.data.get_authorized_instructor().id,
PostsForAnswers.questions_id == self.question.id).first()
self.assertTrue(excluded_instructor_answer, "Missing instructor answer")
excluded_ta_answer = PostsForAnswers.query.join(Posts).filter(
Posts.users_id == self.data.get_authorized_ta().id,
PostsForAnswers.questions_id == self.question.id).first()
self.assertTrue(excluded_ta_answer, "Missing TA answer")
# no judgements has been entered yet, this tests the randomized pairing when no answers has
# scores, since it's randomized though, we'll have to run it lots of times to be sure
for i in range(50):
rv = self.client.get(self.answer_pair_url)
self.assert200(rv)
actual_answer_pair = rv.json
actual_answer1 = actual_answer_pair['answers'][0]
actual_answer2 = actual_answer_pair['answers'][1]
# exclude instructor answer
self.assertNotEqual(actual_answer1['id'], excluded_instructor_answer.id)
self.assertNotEqual(actual_answer2['id'], excluded_instructor_answer.id)
# exclude ta answer
self.assertNotEqual(actual_answer1['id'], excluded_ta_answer.id)
self.assertNotEqual(actual_answer2['id'], excluded_ta_answer.id)
# answer cannot be paired with itself
self.assertNotEqual(actual_answer1['id'], actual_answer2['id'])
def _submit_all_possible_judgements_for_user(self, user_id):
# self.login(username)
# calculate number of judgements to do before user has judged all the pairs it can
num_eligible_answers = -1 # need to minus one to exclude the logged in user's own answer
for answer in self.data.get_student_answers():
if answer.question.id == self.question.id:
num_eligible_answers += 1
# n - 1 possible pairs before all answers have been judged
num_possible_judgements = num_eligible_answers - 1
winner_ids = []
loser_ids = []
for i in range(num_possible_judgements):
pair_generator = AnswerPairGenerator(self.course.id, self.question, user_id)
answerpairing = pair_generator.get_pair()
# answer_pair = AnswerPairings.query.get(answerpairing.id)
# establish expected data by first getting an answer pair
# rv = self.client.get(self.answer_pair_url)
# self.assert200(rv)
# expected_answer_pair = rv.json
min_id = min([answerpairing.answers_id1, answerpairing.answers_id2])
max_id = max([answerpairing.answers_id1, answerpairing.answers_id2])
judgement_submit = self._build_judgement_submit(answerpairing.id, min_id)
winner_ids.append(min_id)
loser_ids.append(max_id)
Judgements.create_judgement(judgement_submit, answerpairing, user_id)
Judgements.calculate_scores(self.question.id)
# test normal post
# rv = self.client.post(self.base_url, data=json.dumps(judgement_submit),
# content_type='application/json')
# self.assert200(rv)
# self.logout()
return {'winners': winner_ids, 'losers': loser_ids}
def test_score_calculation(self):
"""
This is just a rough check on whether score calculations are correct. Answers
that has more wins should have the highest scores.
"""
# Make sure all answers are judged first
winner_ids = self._submit_all_possible_judgements_for_user(
self.data.get_authorized_student().id)['winners']
winner_ids.extend(self._submit_all_possible_judgements_for_user(
self.data.get_secondary_authorized_student().id)['winners'])
# Count the number of wins each answer has had
num_wins_by_id = {}
for winner_id in winner_ids:
num_wins = num_wins_by_id.setdefault(winner_id, 0)
num_wins_by_id[winner_id] = num_wins + 1
# Get the actual score calculated for each answer
answers = self.data.get_student_answers()
answer_scores = {}
for answer in answers:
if answer.question.id == self.question.id:
answer_scores[answer.id] = answer.scores[0].score
# Check that ranking by score and by wins match, this only works for low number of
# judgements
expected_ranking_by_wins = [answer_id for (answer_id, wins) in sorted(
num_wins_by_id.items(),
key=operator.itemgetter(1))]
actual_ranking_by_scores = [answer_id for (answer_id, score) in sorted(
answer_scores.items(),
key=operator.itemgetter(1)) if score > 0]
self.assertSequenceEqual(actual_ranking_by_scores, expected_ranking_by_wins)
def test_comparison_count_matched_pairing(self):
# Make sure all answers are judged first
answer_ids = self._submit_all_possible_judgements_for_user(
self.data.get_authorized_student().id)
answer_ids2 = self._submit_all_possible_judgements_for_user(
self.data.get_secondary_authorized_student().id)
compared_ids = \
answer_ids['winners'] + answer_ids2['winners'] + \
answer_ids['losers'] + answer_ids2['losers']
# Just a simple test for now, make sure that answers with the smaller number of
# comparisons are matched up with each other
# Count number of comparisons done for each answer
num_comp_by_id = {}
for answer_id in compared_ids:
num_comp = num_comp_by_id.setdefault(answer_id, 0)
num_comp_by_id[answer_id] = num_comp + 1
comp_groups = {}
for answerId in num_comp_by_id:
count = num_comp_by_id[answerId]
comp_groups.setdefault(count, [])
comp_groups[count].append(answerId)
counts = sorted(comp_groups)
# get the answerIds with the lowest count of comparisons
possible_answer_ids = comp_groups[counts[0]]
if len(possible_answer_ids) < 2:
# if the lowest count group does not have enough to create a pair - add the next group
possible_answer_ids += comp_groups[counts[1]]
# Check that the 2 answers with 1 win gets returned
with self.login(self.data.get_authorized_student_with_no_answers().username):
rv = self.client.get(self.answer_pair_url)
self.assert200(rv)
self.assertIn(rv.json['answers'][0]['id'], possible_answer_ids)
self.assertIn(rv.json['answers'][1]['id'], possible_answer_ids)
def test_get_judgement_count(self):
url = self._build_url(self.data.get_course().id, self.question.id)
# test login required
tail = '/users/' + str(self.data.get_authorized_student().id) + '/count'
rv = self.client.get(url + tail)
self.assert401(rv)
# test unauthorized user
with self.login(self.data.get_unauthorized_student().username):
tail = '/users/' + str(self.data.get_unauthorized_student().id) + '/count'
rv = self.client.get(url + tail)
self.assert403(rv)
with self.login(self.data.get_authorized_instructor().username):
tail = '/users/' + str(self.data.get_authorized_instructor().id) + '/count'
# test invalid course id
invalid_url = self._build_url(999, self.question.id)
rv = self.client.get(invalid_url + tail)
self.assert404(rv)
# test invalid question id
invalid_url = self._build_url(self.data.get_course().id, 999)
rv = self.client.get(invalid_url + tail)
self.assert404(rv)
# test authorized instructor
rv = self.client.get(url + tail)
self.assert200(rv)
self.assertEqual(rv.json['count'], 0)
# test authorized student
winners = self._submit_all_possible_judgements_for_user(
self.data.get_authorized_student().id)['winners']
tail = '/users/' + str(self.data.get_authorized_student().id) + '/count'
with self.login(self.data.get_authorized_student().username):
rv = self.client.get(url + tail)
self.assert200(rv)
self.assertEqual(rv.json['count'], len(winners))
def test_get_all_judgement_count(self):
url = '/api/courses/' + str(self.data.get_course().id) + '/judgements/count'
# test login required
rv = self.client.get(url)
self.assert401(rv)
# test unauthorized user
with self.login(self.data.get_unauthorized_instructor().username):
rv = self.client.get(url)
self.assert403(rv)
with self.login(self.data.get_authorized_instructor().username):
# test invalid course id
rv = self.client.get('/api/courses/999/judgements/count')
self.assert404(rv)
questions = self.data.get_questions()
# test authorized instructor
rv = self.client.get(url)
self.assert200(rv)
count = rv.json['judgements']
for ques in questions:
question_id = str(ques.id)
self.assertTrue(question_id in count)
self.assertEqual(count[question_id], 0)
# test authorized student
winners = self._submit_all_possible_judgements_for_user(
self.data.get_authorized_student().id)['winners']
judgement_count = len(winners)
with self.login(self.data.get_authorized_student().username):
rv = self.client.get(url)
self.assert200(rv)
count = rv.json['judgements']
for ques in questions:
question_id = str(ques.id)
self.assertTrue(question_id in count)
jcount = judgement_count if ques.id == self.question.id else 0
self.assertEqual(count[question_id], jcount)
def test_get_all_availPair_logic(self):
url = '/api/courses/' + str(self.data.get_course().id) + '/judgements/availpair'
# test login required
rv = self.client.get(url)
self.assert401(rv)
# test unauthorized user
with self.login(self.data.get_unauthorized_student().username):
rv = self.client.get(url)
self.assert403(rv)
with self.login(self.data.get_authorized_student().username):
# test invalid course id
invalid_url = '/api/courses/999/judgements/availpair'
rv = self.client.get(invalid_url)
self.assert404(rv)
first_ques = self.data.get_questions()[0]
last_ques = self.data.get_questions()[-1]
expected = {ques.id: True for ques in self.data.get_questions()}
expected[last_ques.id] = False
# test authorized student - when haven't judged
rv = self.client.get(url)
self.assert200(rv)
logic = rv.json['availPairsLogic']
for ques in self.data.get_questions():
self.assertEqual(logic[str(ques.id)], expected[ques.id])
self._submit_all_possible_judgements_for_user(self.data.get_authorized_student().id)
with self.login(self.data.get_authorized_student().username):
# test authorized student - when have judged all
rv = self.client.get(url)
self.assert200(rv)
logic = rv.json['availPairsLogic']
expected[first_ques.id] = False
for ques in self.data.get_questions():
self.assertEqual(logic[str(ques.id)], expected[ques.id])
def test_get_availPair_logic(self):
url = self._build_url(self.data.get_course().id, self.question.id)
tail = '/users/' + str(self.data.get_unauthorized_student().id) + '/availpair'
# test login required
rv = self.client.get(url + tail)
self.assert401(rv)
# test unauthorized user
with self.login(self.data.get_unauthorized_student().username):
rv = self.client.get(url + tail)
self.assert403(rv)
# test invalid course id
tail = '/users/' + str(self.data.get_authorized_student().id) + '/availpair'
with self.login(self.data.get_authorized_student().username):
invalid_url = self._build_url(999, self.question.id)
rv = self.client.get(invalid_url + tail)
self.assert404(rv)
# test invalid question id
invalid_url = self._build_url(self.data.get_course().id, 999)
rv = self.client.get(invalid_url + tail)
self.assert404(rv)
with self.login(self.data.get_authorized_student().username):
# test authorized student - when haven't judged
rv = self.client.get(url + tail)
self.assert200(rv)
self.assertTrue(rv.json['availPairsLogic'])
self._submit_all_possible_judgements_for_user(self.data.get_authorized_student().id)
# test authorized student - when have judged all
self.login(self.data.get_authorized_student().username)
rv = self.client.get(url + tail)
self.assert200(rv)
self.assertFalse(rv.json['availPairsLogic'])
class JudgementAPITests(ACJAPITestCase):
def setUp(self):
super(JudgementAPITests, self).setUp()
self.data = JudgmentsTestData()
self.course = self.data.get_course()
self.question = self.data.get_questions()[0]
self.base_url = self._build_url(self.course.id, self.question.id)
self.answer_pair_url = self.base_url + '/pair'
def _build_url(self, course_id, question_id, tail=""):
url = \
'/api/courses/' + str(course_id) + '/questions/' + str(question_id) + '/judgements' + \
tail
return url
def _build_judgement_submit(self, answerpair_id, winner_id):
submit = {
'answerpair_id':
answerpair_id,
'judgements': [{
'question_criterion_id': self.question.criteria[0].id,
'answer_id_winner': winner_id
}]
}
return submit
def test_get_answer_pair_access_control(self):
# test login required
rv = self.client.get(self.answer_pair_url)
self.assert401(rv)
# test deny access to unenroled users
with self.login(self.data.get_unauthorized_student().username):
rv = self.client.get(self.answer_pair_url)
self.assert403(rv)
with self.login(self.data.get_unauthorized_instructor().username):
rv = self.client.get(self.answer_pair_url)
self.assert403(rv)
# enroled user from this point on
with self.login(self.data.get_authorized_student().username):
# test non-existent course
rv = self.client.get(
self._build_url(9993929, self.question.id, '/pair'))
self.assert404(rv)
# test non-existent question
rv = self.client.get(
self._build_url(self.course.id, 23902390, '/pair'))
self.assert404(rv)
# no judgements has been entered yet, question is not in judging period
rv = self.client.get(
self._build_url(self.course.id,
self.data.get_question_in_answer_period().id,
'/pair'))
self.assert403(rv)
def test_get_answer_pair_basic(self):
with self.login(self.data.get_authorized_student().username):
# no judgements has been entered yet
rv = self.client.get(self.answer_pair_url)
self.assert200(rv)
actual_answer_pair = rv.json
actual_answer1 = actual_answer_pair['answers'][0]
actual_answer2 = actual_answer_pair['answers'][1]
expected_answer_ids = [
answer.id for answer in self.data.get_student_answers()
]
# make sure that we actually got answers for the question we're targetting
self.assertIn(actual_answer1['id'], expected_answer_ids)
self.assertIn(actual_answer2['id'], expected_answer_ids)
def test_get_answer_pair_answer_exclusions_for_answers_with_no_scores(
self):
"""
The user doing judgements should not see their own answer in a judgement.
Instructor and TA answers should not show up.
Answers cannot be paired with itself.
For answers that don't have a score yet, which means they're randomly matched up.
"""
with self.login(self.data.get_authorized_student().username):
excluded_student_answer = PostsForAnswers.query.join(Posts).filter(
Posts.users_id == self.data.get_authorized_student().id,
PostsForAnswers.questions_id == self.question.id).first()
self.assertTrue(excluded_student_answer,
"Missing authorized student's answer.")
excluded_instructor_answer = PostsForAnswers.query.join(
Posts).filter(
Posts.users_id == self.data.get_authorized_instructor().id,
PostsForAnswers.questions_id == self.question.id).first()
self.assertTrue(excluded_instructor_answer,
"Missing instructor answer")
excluded_ta_answer = PostsForAnswers.query.join(Posts).filter(
Posts.users_id == self.data.get_authorized_ta().id,
PostsForAnswers.questions_id == self.question.id).first()
self.assertTrue(excluded_ta_answer, "Missing TA answer")
# no judgements has been entered yet, this tests the randomized pairing when no answers has
# scores, since it's randomized though, we'll have to run it lots of times to be sure
for i in range(50):
rv = self.client.get(self.answer_pair_url)
self.assert200(rv)
actual_answer_pair = rv.json
actual_answer1 = actual_answer_pair['answers'][0]
actual_answer2 = actual_answer_pair['answers'][1]
# exclude student's own answer
self.assertNotEqual(actual_answer1['id'],
excluded_student_answer.id)
self.assertNotEqual(actual_answer2['id'],
excluded_student_answer.id)
# exclude instructor answer
self.assertNotEqual(actual_answer1['id'],
excluded_instructor_answer.id)
self.assertNotEqual(actual_answer2['id'],
excluded_instructor_answer.id)
# exclude ta answer
self.assertNotEqual(actual_answer1['id'],
excluded_ta_answer.id)
self.assertNotEqual(actual_answer2['id'],
excluded_ta_answer.id)
# need a user with no answers submitted, otherwise pairs with the same answers
# won't be generated since we have too few answers
with self.login(
self.data.get_authorized_student_with_no_answers().username):
for i in range(50):
rv = self.client.get(self.answer_pair_url)
self.assert200(rv)
# answer cannot be paired with itself
self.assertNotEqual(rv.json['answers'][0]['id'],
rv.json['answers'][1]['id'])
def test_submit_judgement_access_control(self):
# test login required
rv = self.client.post(self.base_url,
data=json.dumps({}),
content_type='application/json')
self.assert401(rv)
# establish expected data by first getting an answer pair
with self.login(self.data.get_authorized_student().username):
rv = self.client.get(self.answer_pair_url)
self.assert200(rv)
# expected_answer_pair = rv.json
judgement_submit = self._build_judgement_submit(
rv.json['id'], rv.json['answers'][0]['id'])
# test deny access to unenroled users
with self.login(self.data.get_unauthorized_student().username):
rv = self.client.post(self.base_url,
data=json.dumps(judgement_submit),
content_type='application/json')
self.assert403(rv)
with self.login(self.data.get_unauthorized_instructor().username):
rv = self.client.post(self.base_url,
data=json.dumps(judgement_submit),
content_type='application/json')
self.assert403(rv)
# test deny access to non-students
with self.login(self.data.get_authorized_instructor().username):
rv = self.client.post(self.base_url,
data=json.dumps(judgement_submit),
content_type='application/json')
self.assert403(rv)
# authorized user from this point
with self.login(self.data.get_authorized_student().username):
# test non-existent course
rv = self.client.post(self._build_url(9999999, self.question.id),
data=json.dumps(judgement_submit),
content_type='application/json')
self.assert404(rv)
# test non-existent question
rv = self.client.post(self._build_url(self.course.id, 9999999),
data=json.dumps(judgement_submit),
content_type='application/json')
self.assert404(rv)
# test reject missing criteria
faulty_judgements = copy.deepcopy(judgement_submit)
faulty_judgements['judgements'] = []
rv = self.client.post(self.base_url,
data=json.dumps(faulty_judgements),
content_type='application/json')
self.assert400(rv)
# test reject missing course criteria id
faulty_judgements = copy.deepcopy(judgement_submit)
del faulty_judgements['judgements'][0]['question_criterion_id']
rv = self.client.post(self.base_url,
data=json.dumps(faulty_judgements),
content_type='application/json')
self.assert400(rv)
# test reject missing winner
faulty_judgements = copy.deepcopy(judgement_submit)
del faulty_judgements['judgements'][0]['answer_id_winner']
rv = self.client.post(self.base_url,
data=json.dumps(faulty_judgements),
content_type='application/json')
self.assert400(rv)
# test invalid criteria id
faulty_judgements = copy.deepcopy(judgement_submit)
faulty_judgements['judgements'][0][
'question_criterion_id'] = 3930230
rv = self.client.post(self.base_url,
data=json.dumps(faulty_judgements),
content_type='application/json')
self.assert400(rv)
# test invalid winner id
faulty_judgements = copy.deepcopy(judgement_submit)
faulty_judgements['judgements'][0]['answer_id_winner'] = 2382301
rv = self.client.post(self.base_url,
data=json.dumps(faulty_judgements),
content_type='application/json')
self.assert400(rv)
# test invalid answer pair
faulty_judgements = copy.deepcopy(judgement_submit)
faulty_judgements['answerpair_id'] = 2382301
rv = self.client.post(self.base_url,
data=json.dumps(faulty_judgements),
content_type='application/json')
self.assert404(rv)
def test_submit_judgement_basic(self):
with self.login(self.data.get_authorized_student().username):
# calculate number of judgements to do before user has judged all the pairs it can
num_eligible_answers = -1 # need to minus one to exclude the logged in user's own answer
for answer in self.data.get_student_answers():
if answer.question.id == self.question.id:
num_eligible_answers += 1
# n - 1 possible pairs before all answers have been judged
num_possible_judgements = num_eligible_answers - 1
winner_ids = []
for i in range(num_possible_judgements):
# establish expected data by first getting an answer pair
rv = self.client.get(self.answer_pair_url)
self.assert200(rv)
expected_answer_pair = rv.json
judgement_submit = self._build_judgement_submit(
rv.json['id'], rv.json['answers'][0]['id'])
winner_ids.append(rv.json['answers'][0]['id'])
# test normal post
rv = self.client.post(self.base_url,
data=json.dumps(judgement_submit),
content_type='application/json')
self.assert200(rv)
actual_judgements = rv.json['objects']
self._validate_judgement_submit(judgement_submit,
actual_judgements,
expected_answer_pair)
# Resubmit of same judgement should fail
rv = self.client.post(self.base_url,
data=json.dumps(judgement_submit),
content_type='application/json')
self.assert400(rv)
# all answers has been judged by the user, errors out when trying to get another pair
rv = self.client.get(self.answer_pair_url)
self.assert400(rv)
def _validate_judgement_submit(self, judgement_submit, actual_judgements,
expected_answer_pair):
self.assertEqual(
len(actual_judgements), len(judgement_submit['judgements']),
"The number of judgements saved does not match the number sent")
for actual_judgement in actual_judgements:
self.assertEqual(
expected_answer_pair['answers'][0]['id'],
actual_judgement['answerpairing']['answers_id1'],
"Expected and actual judgement answer1 id did not match")
self.assertEqual(
expected_answer_pair['answers'][1]['id'],
actual_judgement['answerpairing']['answers_id2'],
"Expected and actual judgement answer2 id did not match")
found_judgement = False
for expected_judgement in judgement_submit['judgements']:
if expected_judgement['question_criterion_id'] != \
actual_judgement['question_criterion']['id']:
continue
self.assertEqual(
expected_judgement['answer_id_winner'],
actual_judgement['answers_id_winner'],
"Expected and actual winner answer id did not match.")
found_judgement = True
self.assertTrue(
found_judgement,
"Actual judgement received contains a judgement that was not sent."
)
def test_get_answer_pair_answer_exclusion_with_scored_answers(self):
"""
The user doing judgements should not see their own answer in a judgement.
Instructor and TA answers should not show up.
Answers cannot be paired with itself.
Scored answer pairing means answers should be matched up to similar scores.
"""
# Make sure all answers are judged first
self._submit_all_possible_judgements_for_user(
self.data.get_authorized_student().id)
self._submit_all_possible_judgements_for_user(
self.data.get_secondary_authorized_student().id)
with self.login(
self.data.get_authorized_student_with_no_answers().username):
excluded_instructor_answer = PostsForAnswers.query.join(
Posts).filter(
Posts.users_id == self.data.get_authorized_instructor().id,
PostsForAnswers.questions_id == self.question.id).first()
self.assertTrue(excluded_instructor_answer,
"Missing instructor answer")
excluded_ta_answer = PostsForAnswers.query.join(Posts).filter(
Posts.users_id == self.data.get_authorized_ta().id,
PostsForAnswers.questions_id == self.question.id).first()
self.assertTrue(excluded_ta_answer, "Missing TA answer")
# no judgements has been entered yet, this tests the randomized pairing when no answers has
# scores, since it's randomized though, we'll have to run it lots of times to be sure
for i in range(50):
rv = self.client.get(self.answer_pair_url)
self.assert200(rv)
actual_answer_pair = rv.json
actual_answer1 = actual_answer_pair['answers'][0]
actual_answer2 = actual_answer_pair['answers'][1]
# exclude instructor answer
self.assertNotEqual(actual_answer1['id'],
excluded_instructor_answer.id)
self.assertNotEqual(actual_answer2['id'],
excluded_instructor_answer.id)
# exclude ta answer
self.assertNotEqual(actual_answer1['id'],
excluded_ta_answer.id)
self.assertNotEqual(actual_answer2['id'],
excluded_ta_answer.id)
# answer cannot be paired with itself
self.assertNotEqual(actual_answer1['id'], actual_answer2['id'])
def _submit_all_possible_judgements_for_user(self, user_id):
# self.login(username)
# calculate number of judgements to do before user has judged all the pairs it can
num_eligible_answers = -1 # need to minus one to exclude the logged in user's own answer
for answer in self.data.get_student_answers():
if answer.question.id == self.question.id:
num_eligible_answers += 1
# n - 1 possible pairs before all answers have been judged
num_possible_judgements = num_eligible_answers - 1
winner_ids = []
loser_ids = []
for i in range(num_possible_judgements):
pair_generator = AnswerPairGenerator(self.course.id, self.question,
user_id)
answerpairing = pair_generator.get_pair()
# answer_pair = AnswerPairings.query.get(answerpairing.id)
# establish expected data by first getting an answer pair
# rv = self.client.get(self.answer_pair_url)
# self.assert200(rv)
# expected_answer_pair = rv.json
min_id = min(
[answerpairing.answers_id1, answerpairing.answers_id2])
max_id = max(
[answerpairing.answers_id1, answerpairing.answers_id2])
judgement_submit = self._build_judgement_submit(
answerpairing.id, min_id)
winner_ids.append(min_id)
loser_ids.append(max_id)
Judgements.create_judgement(judgement_submit, answerpairing,
user_id)
Judgements.calculate_scores(self.question.id)
# test normal post
# rv = self.client.post(self.base_url, data=json.dumps(judgement_submit),
# content_type='application/json')
# self.assert200(rv)
# self.logout()
return {'winners': winner_ids, 'losers': loser_ids}
def test_score_calculation(self):
"""
This is just a rough check on whether score calculations are correct. Answers
that has more wins should have the highest scores.
"""
# Make sure all answers are judged first
winner_ids = self._submit_all_possible_judgements_for_user(
self.data.get_authorized_student().id)['winners']
winner_ids.extend(
self._submit_all_possible_judgements_for_user(
self.data.get_secondary_authorized_student().id)['winners'])
# Count the number of wins each answer has had
num_wins_by_id = {}
for winner_id in winner_ids:
num_wins = num_wins_by_id.setdefault(winner_id, 0)
num_wins_by_id[winner_id] = num_wins + 1
# Get the actual score calculated for each answer
answers = self.data.get_student_answers()
answer_scores = {}
for answer in answers:
if answer.question.id == self.question.id:
answer_scores[answer.id] = answer.scores[0].score
# Check that ranking by score and by wins match, this only works for low number of
# judgements
expected_ranking_by_wins = [
answer_id
for (answer_id, wins) in sorted(num_wins_by_id.items(),
key=operator.itemgetter(1))
]
actual_ranking_by_scores = [
answer_id
for (answer_id, score
) in sorted(answer_scores.items(), key=operator.itemgetter(1))
if score > 0
]
self.assertSequenceEqual(actual_ranking_by_scores,
expected_ranking_by_wins)
def test_comparison_count_matched_pairing(self):
# Make sure all answers are judged first
answer_ids = self._submit_all_possible_judgements_for_user(
self.data.get_authorized_student().id)
answer_ids2 = self._submit_all_possible_judgements_for_user(
self.data.get_secondary_authorized_student().id)
compared_ids = \
answer_ids['winners'] + answer_ids2['winners'] + \
answer_ids['losers'] + answer_ids2['losers']
# Just a simple test for now, make sure that answers with the smaller number of
# comparisons are matched up with each other
# Count number of comparisons done for each answer
num_comp_by_id = {}
for answer_id in compared_ids:
num_comp = num_comp_by_id.setdefault(answer_id, 0)
num_comp_by_id[answer_id] = num_comp + 1
comp_groups = {}
for answerId in num_comp_by_id:
count = num_comp_by_id[answerId]
comp_groups.setdefault(count, [])
comp_groups[count].append(answerId)
counts = sorted(comp_groups)
# get the answerIds with the lowest count of comparisons
possible_answer_ids = comp_groups[counts[0]]
if len(possible_answer_ids) < 2:
# if the lowest count group does not have enough to create a pair - add the next group
possible_answer_ids += comp_groups[counts[1]]
# Check that the 2 answers with 1 win gets returned
with self.login(
self.data.get_authorized_student_with_no_answers().username):
rv = self.client.get(self.answer_pair_url)
self.assert200(rv)
self.assertIn(rv.json['answers'][0]['id'], possible_answer_ids)
self.assertIn(rv.json['answers'][1]['id'], possible_answer_ids)
def test_get_judgement_count(self):
url = self._build_url(self.data.get_course().id, self.question.id)
# test login required
tail = '/users/' + str(
self.data.get_authorized_student().id) + '/count'
rv = self.client.get(url + tail)
self.assert401(rv)
# test unauthorized user
with self.login(self.data.get_unauthorized_student().username):
tail = '/users/' + str(
self.data.get_unauthorized_student().id) + '/count'
rv = self.client.get(url + tail)
self.assert403(rv)
with self.login(self.data.get_authorized_instructor().username):
tail = '/users/' + str(
self.data.get_authorized_instructor().id) + '/count'
# test invalid course id
invalid_url = self._build_url(999, self.question.id)
rv = self.client.get(invalid_url + tail)
self.assert404(rv)
# test invalid question id
invalid_url = self._build_url(self.data.get_course().id, 999)
rv = self.client.get(invalid_url + tail)
self.assert404(rv)
# test authorized instructor
rv = self.client.get(url + tail)
self.assert200(rv)
self.assertEqual(rv.json['count'], 0)
# test authorized student
winners = self._submit_all_possible_judgements_for_user(
self.data.get_authorized_student().id)['winners']
tail = '/users/' + str(
self.data.get_authorized_student().id) + '/count'
with self.login(self.data.get_authorized_student().username):
rv = self.client.get(url + tail)
self.assert200(rv)
self.assertEqual(rv.json['count'], len(winners))
def test_get_all_judgement_count(self):
url = '/api/courses/' + str(
self.data.get_course().id) + '/judgements/count'
# test login required
rv = self.client.get(url)
self.assert401(rv)
# test unauthorized user
with self.login(self.data.get_unauthorized_instructor().username):
rv = self.client.get(url)
self.assert403(rv)
with self.login(self.data.get_authorized_instructor().username):
# test invalid course id
rv = self.client.get('/api/courses/999/judgements/count')
self.assert404(rv)
questions = self.data.get_questions()
# test authorized instructor
rv = self.client.get(url)
self.assert200(rv)
count = rv.json['judgements']
for ques in questions:
question_id = str(ques.id)
self.assertTrue(question_id in count)
self.assertEqual(count[question_id], 0)
# test authorized student
winners = self._submit_all_possible_judgements_for_user(
self.data.get_authorized_student().id)['winners']
judgement_count = len(winners)
with self.login(self.data.get_authorized_student().username):
rv = self.client.get(url)
self.assert200(rv)
count = rv.json['judgements']
for ques in questions:
question_id = str(ques.id)
self.assertTrue(question_id in count)
jcount = judgement_count if ques.id == self.question.id else 0
self.assertEqual(count[question_id], jcount)
def test_get_all_availPair_logic(self):
url = '/api/courses/' + str(
self.data.get_course().id) + '/judgements/availpair'
# test login required
rv = self.client.get(url)
self.assert401(rv)
# test unauthorized user
with self.login(self.data.get_unauthorized_student().username):
rv = self.client.get(url)
self.assert403(rv)
with self.login(self.data.get_authorized_student().username):
# test invalid course id
invalid_url = '/api/courses/999/judgements/availpair'
rv = self.client.get(invalid_url)
self.assert404(rv)
first_ques = self.data.get_questions()[0]
last_ques = self.data.get_questions()[-1]
expected = {ques.id: True for ques in self.data.get_questions()}
expected[last_ques.id] = False
# test authorized student - when haven't judged
rv = self.client.get(url)
self.assert200(rv)
logic = rv.json['availPairsLogic']
for ques in self.data.get_questions():
self.assertEqual(logic[str(ques.id)], expected[ques.id])
self._submit_all_possible_judgements_for_user(
self.data.get_authorized_student().id)
with self.login(self.data.get_authorized_student().username):
# test authorized student - when have judged all
rv = self.client.get(url)
self.assert200(rv)
logic = rv.json['availPairsLogic']
expected[first_ques.id] = False
for ques in self.data.get_questions():
self.assertEqual(logic[str(ques.id)], expected[ques.id])
def test_get_availPair_logic(self):
url = self._build_url(self.data.get_course().id, self.question.id)
tail = '/users/' + str(
self.data.get_unauthorized_student().id) + '/availpair'
# test login required
rv = self.client.get(url + tail)
self.assert401(rv)
# test unauthorized user
with self.login(self.data.get_unauthorized_student().username):
rv = self.client.get(url + tail)
self.assert403(rv)
# test invalid course id
tail = '/users/' + str(
self.data.get_authorized_student().id) + '/availpair'
with self.login(self.data.get_authorized_student().username):
invalid_url = self._build_url(999, self.question.id)
rv = self.client.get(invalid_url + tail)
self.assert404(rv)
# test invalid question id
invalid_url = self._build_url(self.data.get_course().id, 999)
rv = self.client.get(invalid_url + tail)
self.assert404(rv)
with self.login(self.data.get_authorized_student().username):
# test authorized student - when haven't judged
rv = self.client.get(url + tail)
self.assert200(rv)
self.assertTrue(rv.json['availPairsLogic'])
self._submit_all_possible_judgements_for_user(
self.data.get_authorized_student().id)
# test authorized student - when have judged all
self.login(self.data.get_authorized_student().username)
rv = self.client.get(url + tail)
self.assert200(rv)
self.assertFalse(rv.json['availPairsLogic'])