def setUp(self): super(ComparisonAPITests, self).setUp() self.data = ComparisonTestData() self.course = self.data.get_course() self.assignment = self.data.get_assignments()[0] self.base_url = self._build_url(self.course.uuid, self.assignment.uuid) self.lti_data = LTITestData() secondary_criterion = self.data.create_criterion( self.data.authorized_instructor) AssignmentCriterionFactory(criterion=secondary_criterion, assignment=self.assignment) db.session.commit()
def setUp(self): # remove existing sqlite db if exists self._delete_sqlite_db() # get a new seed (needed for ConcurrentTestSuite so they don't produce the same results) random.seed() numpy.random.seed() super(AlgorithmValidityTests, self).setUp() self.data = ComparisonTestData() self.MAX_COMPARSIONS = (NUMBER_OF_ANSWERS - 1) * (NUMBER_OF_ANSWERS - 2) / 2 self.TOTAL_MAX_ROUNDS = 50 self.COMPARISONS_IN_ROUND = math.ceil(NUMBER_OF_ANSWERS / 2) # stop after lowest of total comparisons possible or 100 rounds worth of comparisons are complete self.TOTAL_MAX_COMPARISONS = min( self.COMPARISONS_IN_ROUND * self.TOTAL_MAX_ROUNDS, NUMBER_OF_ANSWERS * self.MAX_COMPARSIONS) self.course = self.data.create_course() self.instructor = self.data.create_instructor() self.data.enrol_instructor(self.instructor, self.course) self.assignment = self.data.create_assignment_in_comparison_period( self.course, self.instructor, number_of_comparisons=self.MAX_COMPARSIONS, scoring_algorithm=AlgorithmValidityTests.SCORING_ALGORITHM, pairing_algorithm=AlgorithmValidityTests.PAIRING_ALGORITHM) self.students = [] self.answers = [] self.grade_by_answer_uuid = {} actual_grades = ACTUAL_GRADES if not actual_grades: actual_grades = numpy.random.normal(0.78, 0.1, self.NUMBER_OF_ANSWERS) for grade in actual_grades: student = self.data.create_normal_user() self.data.enrol_student(student, self.course) self.students.append(student) answer = self.data.create_answer(self.assignment, student, with_score=False) self.answers.append(answer) self.grade_by_answer_uuid[answer.uuid] = grade self.base_url = self._build_url(self.course.uuid, self.assignment.uuid) db.session.commit()
def setUp(self): if SKIP_VALIDITY_TEST: self.skipTest("scipy and numpy not installed. run `make deps`") # remove existing sqlite db if exists # self._delete_sqlite_db() # TODO: Modify conditions to be more fuzzy (closely_matched_errors with 0.05 correct rate) # Depends on results of research super(AlgorithmValidityTests, self).setUp() self.data = ComparisonTestData() self.ACCEPTABLE_CORRELATION = 0.8 self.NUMBER_OF_ANSWERS = 40 self.WINNER_SELECTOR = WinnerSelector.always_correct self.CORRECT_RATE = 1.0 self.MAX_COMPARSIONS = (self.NUMBER_OF_ANSWERS - 1) * (self.NUMBER_OF_ANSWERS - 2) / 2 self.TOTAL_MAX_ROUNDS = 6 # 3 comparisons per student self.COMPARISONS_IN_ROUND = math.ceil(self.NUMBER_OF_ANSWERS / 2) # stop after lowest of total comparisons possible or 100 rounds worth of comparisons are complete self.TOTAL_MAX_COMPARISONS = min( self.COMPARISONS_IN_ROUND * self.TOTAL_MAX_ROUNDS, self.NUMBER_OF_ANSWERS * self.MAX_COMPARSIONS ) self.course = self.data.create_course() self.instructor = self.data.create_instructor() self.data.enrol_instructor(self.instructor, self.course) self.assignment = self.data.create_assignment_in_comparison_period( self.course, self.instructor, number_of_comparisons=self.MAX_COMPARSIONS, scoring_algorithm=ScoringAlgorithm.elo, pairing_algorithm=PairingAlgorithm.adaptive_min_delta ) self.students = [] self.answers = [] self.grade_by_answer_uuid = {} actual_grades = numpy.random.normal(0.78, 0.1, self.NUMBER_OF_ANSWERS) for grade in actual_grades: student = self.data.create_normal_user() self.data.enrol_student(student, self.course) self.students.append(student) answer = self.data.create_answer(self.assignment, student, with_score=False) self.answers.append(answer) self.grade_by_answer_uuid[answer.uuid] = grade self.base_url = self._build_url(self.course.uuid, self.assignment.uuid) db.session.commit()
def setUp(self): super(ComparisonAPITests, self).setUp() self.data = ComparisonTestData() self.course = self.data.get_course() self.assignment = self.data.get_assignments()[0] self.base_url = self._build_url(self.course.uuid, self.assignment.uuid) self.lti_data = LTITestData() secondary_criterion = self.data.create_criterion(self.data.authorized_instructor) AssignmentCriterionFactory(criterion=secondary_criterion, assignment=self.assignment) db.session.commit()
def setUp(self): super(ComPAIRXAPITestCase, self).setUp() self.data = ComparisonTestData() self.user = self.data.authorized_student self.course = self.data.main_course self.assignment = self.data.assignments[0] self.answer1 = self.data.answers[0] self.answer2 = self.data.answers[1] self.comparison_example = self.data.comparisons_examples[0] self.example_comparison = ComparisonFactory( assignment=self.assignment, user=self.user, answer1_id=self.comparison_example.answer1.id, answer2_id=self.comparison_example.answer2.id, winner=None, completed=False) self.example_comparison_criteria = [] self.comparison = ComparisonFactory(assignment=self.assignment, user=self.user, answer1_id=self.answer1.id, answer2_id=self.answer2.id, winner=None, completed=False) self.comparison_criteria = [] for criterion in self.assignment.criteria: self.example_comparison_criteria.append( ComparisonCriterionFactory( comparison=self.example_comparison, criterion=criterion, winner=WinningAnswer.answer1, )) self.comparison_criteria.append( ComparisonCriterionFactory( comparison=self.comparison, criterion=criterion, winner=WinningAnswer.answer1, )) db.session.commit() self.maxDiff = None
def setUp(self): # remove existing sqlite db if exists self._delete_sqlite_db() # get a new seed (needed for ConcurrentTestSuite so they don't produce the same results) random.seed() numpy.random.seed() super(AlgorithmValidityTests, self).setUp() self.data = ComparisonTestData() self.MAX_COMPARSIONS = (NUMBER_OF_ANSWERS - 1) * (NUMBER_OF_ANSWERS - 2) / 2 self.TOTAL_MAX_ROUNDS = 50 self.COMPARISONS_IN_ROUND = math.ceil(NUMBER_OF_ANSWERS / 2) # stop after lowest of total comparisons possible or 100 rounds worth of comparisons are complete self.TOTAL_MAX_COMPARISONS = min( self.COMPARISONS_IN_ROUND * self.TOTAL_MAX_ROUNDS, NUMBER_OF_ANSWERS * self.MAX_COMPARSIONS ) self.course = self.data.create_course() self.instructor = self.data.create_instructor() self.data.enrol_instructor(self.instructor, self.course) self.assignment = self.data.create_assignment_in_comparison_period( self.course, self.instructor, number_of_comparisons=self.MAX_COMPARSIONS, scoring_algorithm=AlgorithmValidityTests.SCORING_ALGORITHM, pairing_algorithm=AlgorithmValidityTests.PAIRING_ALGORITHM) self.students = [] self.answers = [] self.grade_by_answer_uuid = {} actual_grades = ACTUAL_GRADES if not actual_grades: actual_grades = numpy.random.normal(0.78, 0.1, self.NUMBER_OF_ANSWERS) for grade in actual_grades: student = self.data.create_normal_user() self.data.enrol_student(student, self.course) self.students.append(student) answer = self.data.create_answer(self.assignment, student, with_score=False) self.answers.append(answer) self.grade_by_answer_uuid[answer.uuid] = grade self.base_url = self._build_url(self.course.uuid, self.assignment.uuid) db.session.commit()
def setUp(self): super(CoursesDuplicateComplexAPITests, self).setUp() self.data = ComparisonTestData() self.course = self.data.get_course() # add start_date self.course.start_date = datetime.datetime.utcnow() self.course.end_date = datetime.datetime.utcnow() + datetime.timedelta(days=90) db.session.commit() self.url = '/api/courses/' + self.course.uuid + '/duplicate' self.date_delta = datetime.timedelta(days=180) self.expected = { 'name': 'duplicate course', 'year': 2015, 'term': 'Winter', 'start_date': (self.course.start_date + self.date_delta).isoformat() + 'Z', 'end_date': (self.course.end_date + self.date_delta).isoformat() + 'Z', 'assignments': [] } for assignment in self.course.assignments: if not assignment.active: continue assignment_data = { 'id': assignment.uuid, 'answer_start': (assignment.answer_start + self.date_delta).isoformat() + 'Z', 'answer_end': (assignment.answer_end + self.date_delta).isoformat() + 'Z' } if assignment.compare_start != None: assignment_data['compare_start'] = (assignment.compare_start + self.date_delta).isoformat() + 'Z' if assignment.compare_end != None: assignment_data['compare_end'] = (assignment.compare_end + self.date_delta).isoformat() + 'Z' self.expected['assignments'].append(assignment_data)
class ComparisonAPITests(ComPAIRAPITestCase): def setUp(self): super(ComparisonAPITests, self).setUp() self.data = ComparisonTestData() self.course = self.data.get_course() self.assignment = self.data.get_assignments()[0] self.base_url = self._build_url(self.course.uuid, self.assignment.uuid) self.lti_data = LTITestData() secondary_criterion = self.data.create_criterion(self.data.authorized_instructor) AssignmentCriterionFactory(criterion=secondary_criterion, assignment=self.assignment) db.session.commit() def _build_url(self, course_uuid, assignment_uuid, tail=""): url = '/api/courses/' + course_uuid + '/assignments/' + assignment_uuid + '/comparisons' + tail return url def _build_comparison_submit(self, winner_uuid, draft=False): submit = { 'comparisons': [ { 'criterion_id': self.assignment.criteria[0].uuid, 'winner_id': winner_uuid, 'draft': draft }, { 'criterion_id': self.assignment.criteria[1].uuid, 'winner_id': winner_uuid, 'draft': draft } ] } return submit def test_get_answer_pair_access_control(self): # test login required rv = self.client.get(self.base_url) self.assert401(rv) # test deny access to unenroled users with self.login(self.data.get_unauthorized_student().username): rv = self.client.get(self.base_url) self.assert403(rv) with self.login(self.data.get_unauthorized_instructor().username): rv = self.client.get(self.base_url) self.assert403(rv) # enroled user from this point on with self.login(self.data.get_authorized_student().username): # test non-existent course rv = self.client.get(self._build_url("9993929", self.assignment.uuid)) self.assert404(rv) # test non-existent assignment rv = self.client.get(self._build_url(self.course.uuid, "23902390")) self.assert404(rv) # no comparisons has been entered yet, assignment is not in comparing period rv = self.client.get(self._build_url( self.course.uuid, self.data.get_assignment_in_answer_period().uuid)) self.assert403(rv) def test_submit_comparison_access_control(self): # test login required rv = self.client.post( self.base_url, data=json.dumps({}), content_type='application/json') self.assert401(rv) # establish expected data by first getting an answer pair with self.login(self.data.get_authorized_student().username): rv = self.client.get(self.base_url) self.assert200(rv) # expected_comparisons = rv.json comparison_submit = self._build_comparison_submit(rv.json['objects'][0]['answer1_id']) # test deny access to unenroled users with self.login(self.data.get_unauthorized_student().username): rv = self.client.post( self.base_url, data=json.dumps(comparison_submit), content_type='application/json') self.assert403(rv) with self.login(self.data.get_unauthorized_instructor().username): rv = self.client.post( self.base_url, data=json.dumps(comparison_submit), content_type='application/json') self.assert403(rv) # authorized user from this point with self.login(self.data.get_authorized_student().username): # test non-existent course rv = self.client.post( self._build_url("9999999", self.assignment.uuid), data=json.dumps(comparison_submit), content_type='application/json') self.assert404(rv) # test non-existent assignment rv = self.client.post( self._build_url(self.course.uuid, "9999999"), data=json.dumps(comparison_submit), content_type='application/json') self.assert404(rv) # test reject missing criteria faulty_comparisons = copy.deepcopy(comparison_submit) faulty_comparisons['comparisons'] = [] rv = self.client.post( self.base_url, data=json.dumps(faulty_comparisons), content_type='application/json') self.assert400(rv) # test reject missing course criteria id faulty_comparisons = copy.deepcopy(comparison_submit) del faulty_comparisons['comparisons'][0]['criterion_id'] rv = self.client.post( self.base_url, data=json.dumps(faulty_comparisons), content_type='application/json') self.assert400(rv) # test invalid criterion id faulty_comparisons = copy.deepcopy(comparison_submit) faulty_comparisons['comparisons'][0]['criterion_id'] = 3930230 rv = self.client.post( self.base_url, data=json.dumps(faulty_comparisons), content_type='application/json') self.assert400(rv) # test invalid winner id faulty_comparisons = copy.deepcopy(comparison_submit) faulty_comparisons['comparisons'][0]['winner_id'] = 2382301 rv = self.client.post( self.base_url, data=json.dumps(faulty_comparisons), content_type='application/json') self.assert400(rv) # test past grace period self.assignment.compare_start = datetime.datetime.utcnow() - datetime.timedelta(days=7) self.assignment.compare_end = datetime.datetime.utcnow() - datetime.timedelta(minutes=2) db.session.add(self.assignment) db.session.commit() ok_comparisons = copy.deepcopy(comparison_submit) rv = self.client.post( self.base_url, data=json.dumps(ok_comparisons), content_type='application/json') self.assert403(rv) self.assertEqual("Assignment comparison deadline has passed.", rv.json['error']) # test within grace period self.assignment.compare_start = datetime.datetime.utcnow() - datetime.timedelta(days=7) self.assignment.compare_end = datetime.datetime.utcnow() - datetime.timedelta(seconds=15) db.session.add(self.assignment) db.session.commit() ok_comparisons = copy.deepcopy(comparison_submit) rv = self.client.post( self.base_url, data=json.dumps(ok_comparisons), content_type='application/json') self.assert200(rv) self.assignment.educators_can_compare = False db.session.commit() # instructors can access with self.login(self.data.get_authorized_instructor().username): rv = self.client.get(self.base_url) self.assert403(rv) self.assignment.educators_can_compare = True db.session.commit() rv = self.client.get(self.base_url) self.assert200(rv) # expected_comparisons = rv.json comparison_submit = self._build_comparison_submit(rv.json['objects'][0]['answer1_id']) ok_comparisons = copy.deepcopy(comparison_submit) rv = self.client.post( self.base_url, data=json.dumps(ok_comparisons), content_type='application/json') self.assert200(rv) self.assignment.educators_can_compare = False db.session.commit() # ta can access with self.login(self.data.get_authorized_ta().username): rv = self.client.get(self.base_url) self.assert403(rv) self.assignment.educators_can_compare = True db.session.commit() rv = self.client.get(self.base_url) self.assert200(rv) # expected_comparisons = rv.json comparison_submit = self._build_comparison_submit(rv.json['objects'][0]['answer1_id']) ok_comparisons = copy.deepcopy(comparison_submit) rv = self.client.post( self.base_url, data=json.dumps(ok_comparisons), content_type='application/json') self.assert200(rv) @mock.patch('compair.tasks.lti_outcomes.update_lti_course_grades.run') @mock.patch('compair.tasks.lti_outcomes.update_lti_assignment_grades.run') def test_get_and_submit_comparison(self, mocked_update_assignment_grades_run, mocked_update_course_grades_run): lti_consumer = self.lti_data.lti_consumer (lti_user_resource_link1, lti_user_resource_link2) = self.lti_data.setup_student_user_resource_links( self.data.get_authorized_student(), self.course, self.assignment) users = [self.data.get_authorized_student(), self.data.get_authorized_instructor(), self.data.get_authorized_ta()] for user in users: compared_answer_uuids = set() valid_answer_uuids = set() for answer in self.data.get_student_answers(): if answer.assignment.id == self.assignment.id and answer.user_id != user.id: valid_answer_uuids.add(answer.uuid) if user.id == self.data.get_authorized_student().id: for comparison_example in self.data.comparisons_examples: if comparison_example.assignment_id == self.assignment.id: valid_answer_uuids.add(comparison_example.answer1_uuid) valid_answer_uuids.add(comparison_example.answer2_uuid) with self.login(user.username): if user.id in [self.data.get_authorized_instructor().id, self.data.get_authorized_ta().id]: self.assignment.educators_can_compare = False db.session.commit() # cannot compare answers unless educators_can_compare is set for assignment rv = self.client.get(self.base_url) self.assert403(rv) self.assignment.educators_can_compare = True db.session.commit() current = 0 while len(valid_answer_uuids - compared_answer_uuids) > 0: current += 1 if user.id == self.data.get_authorized_student().id: course_grade = CourseGrade.get_user_course_grade(self.course, user).grade assignment_grade = AssignmentGrade.get_user_assignment_grade(self.assignment, user).grade # establish expected data by first getting an answer pair rv = self.client.get(self.base_url) self.assert200(rv) actual_answer1_uuid = rv.json['objects'][0]['answer1_id'] actual_answer2_uuid = rv.json['objects'][0]['answer2_id'] self.assertIn(actual_answer1_uuid, valid_answer_uuids) self.assertIn(actual_answer2_uuid, valid_answer_uuids) self.assertNotEqual(actual_answer1_uuid, actual_answer2_uuid) self.assertTrue(rv.json['new_pair']) self.assertEqual(rv.json['current'], current) # fetch again rv = self.client.get(self.base_url) self.assert200(rv) expected_comparisons = rv.json self.assertEqual(actual_answer1_uuid, rv.json['objects'][0]['answer1_id']) self.assertEqual(actual_answer2_uuid, rv.json['objects'][0]['answer2_id']) self.assertFalse(rv.json['new_pair']) self.assertEqual(rv.json['current'], current) # test draft post comparison_submit = self._build_comparison_submit(rv.json['objects'][0]['answer1_id'], True) rv = self.client.post( self.base_url, data=json.dumps(comparison_submit), content_type='application/json') self.assert200(rv) actual_comparisons = rv.json['objects'] self._validate_comparison_submit(comparison_submit, actual_comparisons, expected_comparisons) # test draft post (no answer id) comparison_submit = self._build_comparison_submit(None) rv = self.client.post( self.base_url, data=json.dumps(comparison_submit), content_type='application/json') self.assert200(rv) actual_comparisons = rv.json['objects'] self._validate_comparison_submit(comparison_submit, actual_comparisons, expected_comparisons) # test normal post comparison_submit = self._build_comparison_submit(rv.json['objects'][0]['answer1_id']) rv = self.client.post( self.base_url, data=json.dumps(comparison_submit), content_type='application/json') self.assert200(rv) actual_comparisons = rv.json['objects'] compared_answer_uuids.add(actual_comparisons[0]['answer1_id']) compared_answer_uuids.add(actual_comparisons[0]['answer2_id']) self._validate_comparison_submit(comparison_submit, actual_comparisons, expected_comparisons) # grades should increase for every comparison if user.id == self.data.get_authorized_student().id: new_course_grade = CourseGrade.get_user_course_grade(self.course, user) new_assignment_grade = AssignmentGrade.get_user_assignment_grade(self.assignment, user) self.assertGreater(new_course_grade.grade, course_grade) self.assertGreater(new_assignment_grade.grade, assignment_grade) mocked_update_course_grades_run.assert_called_once_with( lti_consumer.id, [(lti_user_resource_link1.lis_result_sourcedid, new_course_grade.id)] ) mocked_update_course_grades_run.reset_mock() mocked_update_assignment_grades_run.assert_called_once_with( lti_consumer.id, [(lti_user_resource_link2.lis_result_sourcedid, new_assignment_grade.id)] ) mocked_update_assignment_grades_run.reset_mock() else: new_course_grade = CourseGrade.get_user_course_grade(self.course, user) new_assignment_grade = AssignmentGrade.get_user_assignment_grade(self.assignment, user) self.assertIsNone(new_course_grade) self.assertIsNone(new_assignment_grade) mocked_update_assignment_grades_run.assert_not_called() mocked_update_course_grades_run.assert_not_called() # Resubmit of same comparison should fail rv = self.client.post( self.base_url, data=json.dumps(comparison_submit), content_type='application/json') self.assert400(rv) # all answers has been compared by the user, errors out when trying to get another pair rv = self.client.get(self.base_url) self.assert400(rv) def _validate_comparison_submit(self, comparison_submit, actual_comparisons, expected_comparisons): self.assertEqual( len(actual_comparisons), len(comparison_submit['comparisons']), "The number of comparisons saved does not match the number sent") for actual_comparison in actual_comparisons: self.assertEqual( expected_comparisons['objects'][0]['answer1_id'], actual_comparison['answer1_id'], "Expected and actual comparison answer1 id did not match") self.assertEqual( expected_comparisons['objects'][0]['answer2_id'], actual_comparison['answer2_id'], "Expected and actual comparison answer2 id did not match") found_comparison = False for expected_comparison in comparison_submit['comparisons']: if expected_comparison['criterion_id'] != \ actual_comparison['criterion_id']: continue self.assertEqual( expected_comparison['winner_id'], actual_comparison['winner_id'], "Expected and actual winner answer id did not match.") found_comparison = True self.assertTrue( found_comparison, "Actual comparison received contains a comparison that was not sent.") def _submit_all_possible_comparisons_for_user(self, user_id): example_winner_ids = [] example_loser_ids = [] for comparison_example in self.data.comparisons_examples: if comparison_example.assignment_id == self.assignment.id: comparisons = Comparison.create_new_comparison_set(self.assignment.id, user_id, False) self.assertEqual(comparisons[0].answer1_id, comparison_example.answer1_id) self.assertEqual(comparisons[0].answer2_id, comparison_example.answer2_id) min_id = min([comparisons[0].answer1_id, comparisons[0].answer2_id]) max_id = max([comparisons[0].answer1_id, comparisons[0].answer2_id]) example_winner_ids.append(min_id) example_loser_ids.append(max_id) for comparison in comparisons: comparison.completed = True comparison.winner_id = min_id db.session.add(comparison) db.session.commit() # self.login(username) # calculate number of comparisons to do before user has compared all the pairs it can num_eligible_answers = 0 # need to minus one to exclude the logged in user's own answer for answer in self.data.get_student_answers(): if answer.assignment_id == self.assignment.id and answer.user_id != user_id: num_eligible_answers += 1 # n - 1 possible pairs before all answers have been compared num_possible_comparisons = num_eligible_answers - 1 winner_ids = [] loser_ids = [] for i in range(num_possible_comparisons): comparisons = Comparison.create_new_comparison_set(self.assignment.id, user_id, False) answer1_id = comparisons[0].answer1_id answer2_id = comparisons[0].answer2_id min_id = min([answer1_id, answer2_id]) max_id = max([answer1_id, answer2_id]) winner_ids.append(min_id) loser_ids.append(max_id) for comparison in comparisons: comparison.completed = True comparison.winner_id = min_id db.session.add(comparison) db.session.commit() Comparison.calculate_scores(self.assignment.id) return { 'comparisons': { 'winners': winner_ids, 'losers': loser_ids }, 'comparison_examples': { 'winners': example_winner_ids, 'losers': example_loser_ids } } @mock.patch('random.shuffle') def test_score_calculation(self, mock_shuffle): """ This is just a rough check on whether score calculations are correct. Answers that has more wins should have the highest scores. """ # Make sure all answers are compared first comparisons_auth = self._submit_all_possible_comparisons_for_user( self.data.get_authorized_student().id) comparisons_secondary = self._submit_all_possible_comparisons_for_user( self.data.get_secondary_authorized_student().id) loser_ids = comparisons_auth['comparisons']['losers'] + comparisons_secondary['comparisons']['losers'] winner_ids = comparisons_auth['comparisons']['winners'] + comparisons_secondary['comparisons']['winners'] # Count the number of wins each answer has had num_wins_by_id = {} for loser_id in loser_ids: num_wins_by_id[loser_id] = num_wins_by_id.setdefault(loser_id, 0) for winner_id in winner_ids: num_wins = num_wins_by_id.setdefault(winner_id, 0) num_wins_by_id[winner_id] = num_wins + 1 # Get the actual score calculated for each answer answers = self.data.get_student_answers() answer_scores = {} for answer in answers: if answer.assignment.id == self.assignment.id: answer_scores[answer.id] = answer.scores[0].score # Check that ranking by score and by wins match, this only works for low number of # comparisons sorted_expect_ranking = sorted(num_wins_by_id.items(), key=operator.itemgetter(0), reverse=True) sorted_expect_ranking = sorted(sorted_expect_ranking, key=operator.itemgetter(1)) expected_ranking_by_wins = [answer_id for (answer_id, wins) in sorted_expect_ranking] sorted_actual_ranking = sorted(answer_scores.items(), key=operator.itemgetter(1)) actual_ranking_by_scores = [answer_id for (answer_id, score) in sorted_actual_ranking] self.assertSequenceEqual(actual_ranking_by_scores, expected_ranking_by_wins) def test_comparison_count_matched_pairing(self): # Make sure all answers are compared first answer_ids = self._submit_all_possible_comparisons_for_user( self.data.get_authorized_student().id) answer_ids2 = self._submit_all_possible_comparisons_for_user( self.data.get_secondary_authorized_student().id) compared_ids = \ answer_ids['comparisons']['winners'] + answer_ids2['comparisons']['winners'] + \ answer_ids['comparisons']['losers'] + answer_ids2['comparisons']['losers'] + \ answer_ids['comparison_examples']['winners'] + answer_ids2['comparison_examples']['winners'] + \ answer_ids['comparison_examples']['losers'] + answer_ids2['comparison_examples']['losers'] # Just a simple test for now, make sure that answers with the smaller number of # comparisons are matched up with each other # Count number of comparisons done for each answer num_comp_by_id = {} for answer_id in compared_ids: num_comp = num_comp_by_id.setdefault(answer_id, 0) num_comp_by_id[answer_id] = num_comp + 1 comp_groups = {} for answerId in num_comp_by_id: count = num_comp_by_id[answerId] comp_groups.setdefault(count, []) comp_groups[count].append(answerId) counts = sorted(comp_groups) # get the answerIds with the lowest count of comparisons possible_answer_ids = comp_groups[counts[0]] if len(possible_answer_ids) < 2: # if the lowest count group does not have enough to create a pair - add the next group possible_answer_ids += comp_groups[counts[1]] # Check that the 2 answers with 1 win gets returned with self.login(self.data.get_authorized_student_with_no_answers().username): rv = self.client.get(self.base_url) self.assert200(rv) answer1 = Answer.query.filter_by(uuid=rv.json['objects'][0]['answer1_id']).first() answer2 = Answer.query.filter_by(uuid=rv.json['objects'][0]['answer2_id']).first() self.assertIsNotNone(answer1) self.assertIsNotNone(answer2) self.assertIn(answer1.id, possible_answer_ids) self.assertIn(answer2.id, possible_answer_ids)
def setUp(self): super(ComPAIRLearningRecordTestCase, self).setUp() self.data = ComparisonTestData() self.lti_data = LTITestData() self.user = self.data.authorized_student self.setup_session_data(self.user) self.course = self.data.main_course self.lti_context = self.lti_data.create_context( self.lti_data.lti_consumer, compair_course_id=self.course.id, lis_course_offering_sourcedid="sis_course_id", lis_course_section_sourcedid="sis_section_id", ) self.assignment = self.data.assignments[0] self.criterion = self.assignment.criteria[0] self.answer1 = self.data.answers[0] self.answer2 = self.data.answers[1] self.example_comparison = ComparisonFactory(assignment=self.assignment, user=self.user, answer1_id=self.answer1.id, answer2_id=self.answer2.id, winner=None, completed=False) self.example_comparison_criterion = ComparisonCriterionFactory( comparison=self.example_comparison, criterion=self.criterion, winner=WinningAnswer.answer1, ) self.comparison = ComparisonFactory(assignment=self.assignment, user=self.user, answer1_id=self.answer1.id, answer2_id=self.answer2.id, winner=None, completed=False) self.comparison_criterion = ComparisonCriterionFactory( comparison=self.comparison, criterion=self.criterion, winner=WinningAnswer.answer1, ) db.session.commit() self.expected_caliper_course = { 'academicSession': self.course.term, 'dateCreated': self.course.created.replace(tzinfo=pytz.utc).isoformat(), 'dateModified': self.course.modified.replace(tzinfo=pytz.utc).isoformat(), 'id': "https://localhost:8888/app/course/" + self.course.uuid, 'name': self.course.name, 'type': 'CourseOffering', 'extensions': { 'ltiContexts': [{ 'context_id': self.lti_context.context_id, 'oauth_consumer_key': self.lti_data.lti_consumer.oauth_consumer_key, 'lis_course_offering_sourcedid': "sis_course_id", 'lis_course_section_sourcedid': "sis_section_id", }] } } self.expected_caliper_assignment = { 'name': self.assignment.name, 'type': 'Assessment', 'dateCreated': self.assignment.created.replace(tzinfo=pytz.utc).isoformat(), 'dateModified': self.assignment.modified.replace(tzinfo=pytz.utc).isoformat(), 'dateToStartOn': self.assignment.answer_start.replace(tzinfo=pytz.utc).isoformat(), 'description': self.assignment.description, 'id': "https://localhost:8888/app/course/" + self.course.uuid + "/assignment/" + self.assignment.uuid, 'isPartOf': self.expected_caliper_course, 'items': [{ 'id': "https://localhost:8888/app/course/" + self.course.uuid + "/assignment/" + self.assignment.uuid + "/question", 'type': 'AssessmentItem' }, { 'id': "https://localhost:8888/app/course/" + self.course.uuid + "/assignment/" + self.assignment.uuid + "/comparison/question/1", 'type': 'AssessmentItem' }, { 'id': "https://localhost:8888/app/course/" + self.course.uuid + "/assignment/" + self.assignment.uuid + "/evaluation/question/1", 'type': 'AssessmentItem' }, { 'id': "https://localhost:8888/app/course/" + self.course.uuid + "/assignment/" + self.assignment.uuid + "/evaluation/question/2", 'type': 'AssessmentItem' }, { 'id': "https://localhost:8888/app/course/" + self.course.uuid + "/assignment/" + self.assignment.uuid + "/comparison/question/2", 'type': 'AssessmentItem' }, { 'id': "https://localhost:8888/app/course/" + self.course.uuid + "/assignment/" + self.assignment.uuid + "/evaluation/question/3", 'type': 'AssessmentItem' }, { 'id': "https://localhost:8888/app/course/" + self.course.uuid + "/assignment/" + self.assignment.uuid + "/evaluation/question/4", 'type': 'AssessmentItem' }, { 'id': "https://localhost:8888/app/course/" + self.course.uuid + "/assignment/" + self.assignment.uuid + "/comparison/question/3", 'type': 'AssessmentItem' }, { 'id': "https://localhost:8888/app/course/" + self.course.uuid + "/assignment/" + self.assignment.uuid + "/evaluation/question/5", 'type': 'AssessmentItem' }, { 'id': "https://localhost:8888/app/course/" + self.course.uuid + "/assignment/" + self.assignment.uuid + "/evaluation/question/6", 'type': 'AssessmentItem' }], } self.expected_caliper_assignment_question = { 'name': self.assignment.name, 'type': 'AssessmentItem', 'dateCreated': self.assignment.created.replace(tzinfo=pytz.utc).isoformat(), 'dateModified': self.assignment.modified.replace(tzinfo=pytz.utc).isoformat(), 'dateToStartOn': self.assignment.answer_start.replace(tzinfo=pytz.utc).isoformat(), 'dateToSubmit': self.assignment.answer_end.replace(tzinfo=pytz.utc).isoformat(), 'description': self.assignment.description, 'id': "https://localhost:8888/app/course/" + self.course.uuid + "/assignment/" + self.assignment.uuid + "/question", 'isPartOf': self.expected_caliper_assignment, } self.expected_caliper_answer1_attempt = { 'assignable': self.expected_caliper_assignment_question, 'assignee': self.get_compair_caliper_actor(self.answer1.user), 'id': "https://localhost:8888/app/course/" + self.course.uuid + "/assignment/" + self.assignment.uuid + "/question/attempt/" + self.answer1.attempt_uuid, 'duration': "PT05M00S", 'startedAtTime': self.answer1.attempt_started.replace(tzinfo=pytz.utc).isoformat(), 'endedAtTime': self.answer1.attempt_ended.replace(tzinfo=pytz.utc).isoformat(), 'type': 'Attempt' } self.expected_caliper_answer1 = { 'attempt': self.expected_caliper_answer1_attempt, 'id': "https://localhost:8888/app/course/" + self.course.uuid + "/assignment/" + self.assignment.uuid + "/answer/" + self.answer1.uuid, 'type': 'Response', 'dateCreated': self.answer1.created.replace(tzinfo=pytz.utc).isoformat(), 'dateModified': self.answer1.modified.replace(tzinfo=pytz.utc).isoformat(), 'extensions': { 'characterCount': len(self.answer1.content), 'content': self.answer1.content, 'isDraft': False, 'wordCount': 8, 'scoreDetails': { 'algorithm': self.assignment.scoring_algorithm.value, 'loses': 0, 'opponents': 0, 'rounds': 0, 'score': 5, 'wins': 0, 'criteria': { "https://localhost:8888/app/criterion/" + self.criterion.uuid: { 'loses': 0, 'opponents': 0, 'rounds': 0, 'score': 5, 'wins': 0 }, } }, } } self.expected_caliper_answer2_attempt = { 'assignable': self.expected_caliper_assignment_question, 'assignee': self.get_compair_caliper_actor(self.answer2.user), 'id': "https://localhost:8888/app/course/" + self.course.uuid + "/assignment/" + self.assignment.uuid + "/question/attempt/" + self.answer2.attempt_uuid, 'duration': "PT05M00S", 'startedAtTime': self.answer2.attempt_started.replace(tzinfo=pytz.utc).isoformat(), 'endedAtTime': self.answer2.attempt_ended.replace(tzinfo=pytz.utc).isoformat(), 'type': 'Attempt' } self.expected_caliper_answer2 = { 'attempt': self.expected_caliper_answer2_attempt, 'id': "https://localhost:8888/app/course/" + self.course.uuid + "/assignment/" + self.assignment.uuid + "/answer/" + self.answer2.uuid, 'type': 'Response', 'dateCreated': self.answer2.created.replace(tzinfo=pytz.utc).isoformat(), 'dateModified': self.answer2.modified.replace(tzinfo=pytz.utc).isoformat(), 'extensions': { 'characterCount': len(self.answer2.content), 'content': self.answer2.content, 'isDraft': False, 'wordCount': 8, 'scoreDetails': { 'algorithm': self.assignment.scoring_algorithm.value, 'loses': 0, 'opponents': 0, 'rounds': 0, 'score': 5, 'wins': 0, 'criteria': { "https://localhost:8888/app/criterion/" + self.criterion.uuid: { 'loses': 0, 'opponents': 0, 'rounds': 0, 'score': 5, 'wins': 0 }, } }, } } self.expected_caliper_comparison_question = { 'name': "Assignment comparison #1", 'type': 'AssessmentItem', 'dateCreated': self.assignment.created.replace(tzinfo=pytz.utc).isoformat(), 'dateModified': self.assignment.modified.replace(tzinfo=pytz.utc).isoformat(), 'dateToStartOn': self.assignment.answer_end.replace(tzinfo=pytz.utc).isoformat(), 'id': "https://localhost:8888/app/course/" + self.course.uuid + "/assignment/" + self.assignment.uuid + "/comparison/question/1", 'isPartOf': self.expected_caliper_assignment, } self.expected_xapi_course = { 'id': "https://localhost:8888/app/course/" + self.course.uuid, 'definition': { 'type': 'http://adlnet.gov/expapi/activities/course', 'name': { 'en-US': self.course.name } }, 'objectType': 'Activity' } self.expected_xapi_sis_course = { 'id': 'https://localhost:8888/course/' + self.lti_context.lis_course_offering_sourcedid, 'objectType': 'Activity' } self.expected_xapi_sis_section = { 'id': 'https://localhost:8888/course/' + self.lti_context.lis_course_offering_sourcedid + '/section/' + self.lti_context.lis_course_section_sourcedid, 'objectType': 'Activity' } self.expected_xapi_assignment = { 'id': "https://localhost:8888/app/course/" + self.course.uuid + "/assignment/" + self.assignment.uuid, 'definition': { 'type': 'http://adlnet.gov/expapi/activities/assessment', 'name': { 'en-US': self.assignment.name }, 'description': { 'en-US': self.assignment.description }, }, 'objectType': 'Activity' } self.expected_xapi_assignment_question = { 'id': "https://localhost:8888/app/course/" + self.course.uuid + "/assignment/" + self.assignment.uuid + "/question", 'definition': { 'type': 'http://adlnet.gov/expapi/activities/question', 'name': { 'en-US': self.assignment.name }, 'description': { 'en-US': self.assignment.description }, }, 'objectType': 'Activity' } self.expected_xapi_answer1_attempt = { 'id': "https://localhost:8888/app/course/" + self.course.uuid + "/assignment/" + self.assignment.uuid + "/question/attempt/" + self.answer1.attempt_uuid, 'definition': { 'type': 'http://adlnet.gov/expapi/activities/attempt', 'extensions': { 'http://id.tincanapi.com/extension/attempt': { 'duration': "PT05M00S", 'startedAtTime': self.answer1.attempt_started.replace( tzinfo=pytz.utc).isoformat(), 'endedAtTime': self.answer1.attempt_ended.replace( tzinfo=pytz.utc).isoformat(), } } }, 'objectType': 'Activity' } self.expected_xapi_answer1 = { 'id': "https://localhost:8888/app/course/" + self.course.uuid + "/assignment/" + self.assignment.uuid + "/answer/" + self.answer1.uuid, 'definition': { 'type': 'http://id.tincanapi.com/activitytype/solution', 'extensions': { 'http://id.tincanapi.com/extension/isDraft': False } }, 'objectType': 'Activity' } self.expected_xapi_answer2_attempt = { 'id': "https://localhost:8888/app/course/" + self.course.uuid + "/assignment/" + self.assignment.uuid + "/question/attempt/" + self.answer2.attempt_uuid, 'definition': { 'type': 'http://adlnet.gov/expapi/activities/attempt', 'extensions': { 'http://id.tincanapi.com/extension/attempt': { 'duration': "PT05M00S", 'startedAtTime': self.answer2.attempt_started.replace( tzinfo=pytz.utc).isoformat(), 'endedAtTime': self.answer2.attempt_ended.replace( tzinfo=pytz.utc).isoformat(), } } }, 'objectType': 'Activity' } self.expected_xapi_answer2 = { 'id': "https://localhost:8888/app/course/" + self.course.uuid + "/assignment/" + self.assignment.uuid + "/answer/" + self.answer2.uuid, 'definition': { 'type': 'http://id.tincanapi.com/activitytype/solution', 'extensions': { 'http://id.tincanapi.com/extension/isDraft': False } }, 'objectType': 'Activity' } self.expected_xapi_comparison_question = { 'id': "https://localhost:8888/app/course/" + self.course.uuid + "/assignment/" + self.assignment.uuid + "/comparison/question/1", 'definition': { 'type': 'http://adlnet.gov/expapi/activities/question', 'name': { 'en-US': "Assignment comparison #1" } }, 'objectType': 'Activity' }
class CoursesDuplicateComplexAPITests(ComPAIRAPITestCase): def setUp(self): super(CoursesDuplicateComplexAPITests, self).setUp() self.data = ComparisonTestData() self.course = self.data.get_course() # add start_date self.course.start_date = datetime.datetime.utcnow() self.course.end_date = datetime.datetime.utcnow() + datetime.timedelta( days=90) index = 0 for assignment in self.course.assignments: assignment.answer_start = self.course.start_date + datetime.timedelta( days=index) assignment.answer_end = self.course.start_date + datetime.timedelta( days=index + 7) if assignment.compare_start != None: assignment.compare_start = self.course.start_date + datetime.timedelta( days=index + 14) if assignment.compare_end != None: assignment.compare_end = self.course.start_date + datetime.timedelta( days=index + 21) index += 1 db.session.commit() self.url = '/api/courses/' + self.course.uuid + '/duplicate' self.date_delta = datetime.timedelta(days=180) self.expected = { 'name': 'duplicate course', 'year': 2015, 'term': 'Winter', 'sandbox': False, 'start_date': (self.course.start_date + self.date_delta).isoformat() + 'Z', 'end_date': (self.course.end_date + self.date_delta).isoformat() + 'Z', 'assignments': [] } for assignment in self.course.assignments: if not assignment.active: continue assignment_data = { 'id': assignment.uuid, 'name': assignment.name, 'answer_start': (assignment.answer_start + self.date_delta).isoformat() + 'Z', 'answer_end': (assignment.answer_end + self.date_delta).isoformat() + 'Z' } if assignment.compare_start != None: assignment_data['compare_start'] = ( assignment.compare_start + self.date_delta).isoformat() + 'Z' if assignment.compare_end != None: assignment_data['compare_end'] = ( assignment.compare_end + self.date_delta).isoformat() + 'Z' self.expected['assignments'].append(assignment_data) now = datetime.datetime.utcnow() self.valid_course = self.data.create_course() self.data.enrol_instructor(self.data.get_authorized_instructor(), self.valid_course) self.valid_assignment = self.data.create_assignment_in_answer_period( self.valid_course, self.data.get_authorized_instructor()) self.valid_start_date = now.isoformat() + 'Z' self.valid_end_date = (now + datetime.timedelta(days=90)).isoformat() + 'Z' self.invalid_end_date = (now - datetime.timedelta(days=1)).isoformat() + 'Z' self.valid_answer_start = now.isoformat() + 'Z' self.valid_answer_end = (now + datetime.timedelta(days=90)).isoformat() + 'Z' self.invalid_answer_start = ( now - datetime.timedelta(days=1)).isoformat() + 'Z' self.invalid_answer_end = ( now - datetime.timedelta(days=1)).isoformat() + 'Z' self.invalid_answer_end2 = ( now + datetime.timedelta(days=91)).isoformat() + 'Z' self.valid_compare_start = now.isoformat() + 'Z' self.valid_compare_end = ( now + datetime.timedelta(days=90)).isoformat() + 'Z' self.invalid_compare_start = ( now - datetime.timedelta(days=1)).isoformat() + 'Z' self.invalid_compare_end = ( now - datetime.timedelta(days=1)).isoformat() + 'Z' self.invalid_compare_end2 = ( now + datetime.timedelta(days=91)).isoformat() + 'Z' self.validation_url = '/api/courses/' + self.valid_course.uuid + '/duplicate' self.validate_expected_assignment = { 'id': self.valid_assignment.uuid, 'name': self.valid_assignment.name, 'answer_start': self.valid_answer_start, 'answer_end': self.valid_answer_end, 'compare_start': self.valid_compare_start, 'compare_end': self.valid_compare_end } self.validate_expected_course = { 'name': 'duplicate validation course', 'year': 2015, 'term': 'Winter', 'sandbox': False, 'start_date': self.valid_start_date, 'end_date': self.valid_end_date, 'assignments': [self.validate_expected_assignment] } def test_duplicate_course_complex(self): original_course = self.data.get_course() # test authorized user with self.login(self.data.get_authorized_instructor().username): # test valid expected rv = self.client.post(self.validation_url, data=json.dumps( self.validate_expected_course), content_type='application/json') self.assert200(rv) # test invalid course start/end dates invalid_expected = self.validate_expected_course.copy() invalid_expected['end_date'] = self.invalid_end_date rv = self.client.post(self.validation_url, data=json.dumps(invalid_expected), content_type='application/json') self.assert400(rv) self.assertEqual(rv.json["title"], "Course Not Saved") self.assertEqual( rv.json["message"], "Course end time must be after course start time.") # test invalid assignment answer start invalid_expected = self.validate_expected_course.copy() invalid_expected['assignments'][ 0] = self.validate_expected_assignment.copy() invalid_expected['assignments'][0]['answer_start'] = None rv = self.client.post(self.validation_url, data=json.dumps(invalid_expected), content_type='application/json') self.assert400(rv) self.assertEqual(rv.json["title"], "Course Not Saved") self.assertEqual( rv.json["message"], "No answer period start time provided for assignment " + self.valid_assignment.name + ".") # test invalid assignment answer end invalid_expected = self.validate_expected_course.copy() invalid_expected['assignments'][ 0] = self.validate_expected_assignment.copy() invalid_expected['assignments'][0]['answer_end'] = None rv = self.client.post(self.validation_url, data=json.dumps(invalid_expected), content_type='application/json') self.assert400(rv) self.assertEqual(rv.json["title"], "Course Not Saved") self.assertEqual( rv.json["message"], "No answer period end time provided for assignment " + self.valid_assignment.name + ".") invalid_expected = self.validate_expected_course.copy() invalid_expected['assignments'][ 0] = self.validate_expected_assignment.copy() invalid_expected['assignments'][0][ 'answer_end'] = self.invalid_answer_end rv = self.client.post(self.validation_url, data=json.dumps(invalid_expected), content_type='application/json') self.assert400(rv) self.assertEqual(rv.json["title"], "Course Not Saved") self.assertEqual( rv.json["message"], "Answer period end time must be after the answer start time for assignment " + self.valid_assignment.name + ".") invalid_expected = self.validate_expected_course.copy() invalid_expected['assignments'][ 0] = self.validate_expected_assignment.copy() invalid_expected['assignments'][0][ 'answer_end'] = self.invalid_answer_end2 rv = self.client.post(self.validation_url, data=json.dumps(invalid_expected), content_type='application/json') self.assert400(rv) self.assertEqual(rv.json["title"], "Course Not Saved") self.assertEqual( rv.json["message"], "Answer period end time must be before the course end time for assignment " + self.valid_assignment.name + ".") # test invalid assignment compare start invalid_expected = self.validate_expected_course.copy() invalid_expected['assignments'][ 0] = self.validate_expected_assignment.copy() invalid_expected['assignments'][0]['compare_start'] = None rv = self.client.post(self.validation_url, data=json.dumps(invalid_expected), content_type='application/json') self.assert400(rv) self.assertEqual(rv.json["title"], "Course Not Saved") self.assertEqual( rv.json["message"], "No compare period start time provided for assignment " + self.valid_assignment.name + ".") invalid_expected = self.validate_expected_course.copy() invalid_expected['assignments'][ 0] = self.validate_expected_assignment.copy() invalid_expected['assignments'][0][ 'compare_start'] = self.invalid_compare_start rv = self.client.post(self.validation_url, data=json.dumps(invalid_expected), content_type='application/json') self.assert400(rv) self.assertEqual(rv.json["title"], "Course Not Saved") self.assertEqual( rv.json["message"], "Compare period start time must be after the answer start time for assignment " + self.valid_assignment.name + ".") # test invalid assignment compare end invalid_expected = self.validate_expected_course.copy() invalid_expected['assignments'][ 0] = self.validate_expected_assignment.copy() invalid_expected['assignments'][0]['compare_end'] = None rv = self.client.post(self.validation_url, data=json.dumps(invalid_expected), content_type='application/json') self.assert400(rv) self.assertEqual(rv.json["title"], "Course Not Saved") self.assertEqual( rv.json["message"], "No compare period end time provided for assignment " + self.valid_assignment.name + ".") invalid_expected = self.validate_expected_course.copy() invalid_expected['assignments'][ 0] = self.validate_expected_assignment.copy() invalid_expected['assignments'][0][ 'compare_end'] = self.invalid_compare_end rv = self.client.post(self.validation_url, data=json.dumps(invalid_expected), content_type='application/json') self.assert400(rv) self.assertEqual(rv.json["title"], "Course Not Saved") self.assertEqual( rv.json["message"], "Compare period end time must be after the compare start time for assignment " + self.valid_assignment.name + ".") invalid_expected = self.validate_expected_course.copy() invalid_expected['assignments'][ 0] = self.validate_expected_assignment.copy() invalid_expected['assignments'][0][ 'compare_end'] = self.invalid_compare_end2 rv = self.client.post(self.validation_url, data=json.dumps(invalid_expected), content_type='application/json') self.assert400(rv) self.assertEqual(rv.json["title"], "Course Not Saved") self.assertEqual( rv.json["message"], "Compare period end time must be before the course end time for assignment " + self.valid_assignment.name + ".") # test deep copy assignments rv = self.client.post(self.url, data=json.dumps(self.expected), content_type='application/json') self.assert200(rv) duplicate_course = Course.query.filter_by( uuid=rv.json['id']).first() self.assertIsNotNone(duplicate_course) # verify course duplicated correctly self.assertNotEqual(original_course.id, duplicate_course.id) self.assertNotEqual(original_course.uuid, duplicate_course.uuid) self.assertEqual(self.expected['name'], duplicate_course.name) self.assertEqual(self.expected['year'], duplicate_course.year) self.assertEqual(self.expected['term'], duplicate_course.term) self.assertEqual(self.expected['sandbox'], duplicate_course.sandbox) self.assertEqual(self.expected['start_date'].replace('Z', ''), rv.json['start_date'].replace('+00:00', '')) self.assertEqual(self.expected['end_date'].replace('Z', ''), rv.json['end_date'].replace('+00:00', '')) # verify instructor added to duplicate course user_course = UserCourse.query \ .filter_by( user_id=self.data.get_authorized_instructor().id, course_uuid=rv.json['id'] ) \ .one_or_none() self.assertIsNotNone(user_course) original_assignments = original_course.assignments.all() duplicate_assignments = duplicate_course.assignments.all() self.assertEqual(len(original_assignments), 3) self.assertEqual(len(original_assignments), len(duplicate_assignments)) for index, original_assignment in enumerate(original_assignments): duplicate_assignment = duplicate_assignments[index] self.assertNotEqual(original_assignment.id, duplicate_assignment.id) self.assertNotEqual(original_assignment.uuid, duplicate_assignment.uuid) self.assertEqual(duplicate_course.id, duplicate_assignment.course_id) self.assertEqual(original_assignment.name, duplicate_assignment.name) self.assertEqual(original_assignment.description, duplicate_assignment.description) self.assertEqual(original_assignment.number_of_comparisons, duplicate_assignment.number_of_comparisons) self.assertEqual(original_assignment.students_can_reply, duplicate_assignment.students_can_reply) self.assertEqual(original_assignment.enable_self_evaluation, duplicate_assignment.enable_self_evaluation) self.assertEqual(original_assignment.pairing_algorithm, duplicate_assignment.pairing_algorithm) self.assertEqual( original_assignment.answer_start, (duplicate_assignment.answer_start - self.date_delta)) self.assertEqual( original_assignment.answer_end, (duplicate_assignment.answer_end - self.date_delta)) if original_assignment.compare_start != None: self.assertEqual( original_assignment.compare_start, (duplicate_assignment.compare_start - self.date_delta)) else: self.assertIsNone(duplicate_assignment.compare_start) if original_assignment.compare_end != None: self.assertEqual( original_assignment.compare_end, (duplicate_assignment.compare_end - self.date_delta)) else: self.assertIsNone(duplicate_assignment.compare_end) self.assertEqual(len(original_assignment.criteria), 1) self.assertEqual(len(original_assignment.criteria), len(duplicate_assignment.criteria)) for index, original_criteria in enumerate( original_assignment.criteria): duplicate_criteria = duplicate_assignment.criteria[index] self.assertEqual(original_criteria.id, duplicate_criteria.id) original_comparison_examples = original_assignment.comparison_examples.all( ) duplicate_comparison_examples = duplicate_assignment.comparison_examples.all( ) if original_assignment.id in [1, 2]: self.assertEqual(len(original_comparison_examples), 1) else: self.assertEqual(len(original_comparison_examples), 0) self.assertEqual(len(original_comparison_examples), len(duplicate_comparison_examples)) for index, original_comparison_example in enumerate( original_comparison_examples): duplicate_comparison_example = duplicate_comparison_examples[ index] self.assertNotEqual(original_comparison_example.id, duplicate_comparison_example.id) self.assertNotEqual( original_comparison_example.answer1_id, duplicate_comparison_example.answer1_id) self.assertNotEqual( original_comparison_example.answer2_id, duplicate_comparison_example.answer2_id) self.assertEqual( duplicate_assignment.id, duplicate_comparison_example.assignment_id) original_answer1 = original_comparison_example.answer1 duplicate_answer1 = duplicate_comparison_example.answer1 self.assertNotEqual(original_answer1.id, duplicate_answer1.id) self.assertEqual(duplicate_assignment.id, duplicate_answer1.assignment_id) self.assertEqual(original_answer1.content, duplicate_answer1.content) self.assertEqual(original_answer1.practice, duplicate_answer1.practice) self.assertEqual(original_answer1.active, duplicate_answer1.active) self.assertEqual(original_answer1.draft, duplicate_answer1.draft) original_answer2 = original_comparison_example.answer2 duplicate_answer2 = duplicate_comparison_example.answer2 self.assertNotEqual(original_answer2.id, duplicate_answer2.id) self.assertEqual(duplicate_assignment.id, duplicate_answer2.assignment_id) self.assertEqual(original_answer2.content, duplicate_answer2.content) self.assertEqual(original_answer2.practice, original_answer2.practice) self.assertEqual(original_answer2.active, original_answer2.active) self.assertEqual(original_answer2.draft, original_answer2.draft)
def setUp(self): super(CoursesDuplicateComplexAPITests, self).setUp() self.data = ComparisonTestData() self.course = self.data.get_course() # add start_date self.course.start_date = datetime.datetime.utcnow() self.course.end_date = datetime.datetime.utcnow() + datetime.timedelta( days=90) index = 0 for assignment in self.course.assignments: assignment.answer_start = self.course.start_date + datetime.timedelta( days=index) assignment.answer_end = self.course.start_date + datetime.timedelta( days=index + 7) if assignment.compare_start != None: assignment.compare_start = self.course.start_date + datetime.timedelta( days=index + 14) if assignment.compare_end != None: assignment.compare_end = self.course.start_date + datetime.timedelta( days=index + 21) index += 1 db.session.commit() self.url = '/api/courses/' + self.course.uuid + '/duplicate' self.date_delta = datetime.timedelta(days=180) self.expected = { 'name': 'duplicate course', 'year': 2015, 'term': 'Winter', 'sandbox': False, 'start_date': (self.course.start_date + self.date_delta).isoformat() + 'Z', 'end_date': (self.course.end_date + self.date_delta).isoformat() + 'Z', 'assignments': [] } for assignment in self.course.assignments: if not assignment.active: continue assignment_data = { 'id': assignment.uuid, 'name': assignment.name, 'answer_start': (assignment.answer_start + self.date_delta).isoformat() + 'Z', 'answer_end': (assignment.answer_end + self.date_delta).isoformat() + 'Z' } if assignment.compare_start != None: assignment_data['compare_start'] = ( assignment.compare_start + self.date_delta).isoformat() + 'Z' if assignment.compare_end != None: assignment_data['compare_end'] = ( assignment.compare_end + self.date_delta).isoformat() + 'Z' self.expected['assignments'].append(assignment_data) now = datetime.datetime.utcnow() self.valid_course = self.data.create_course() self.data.enrol_instructor(self.data.get_authorized_instructor(), self.valid_course) self.valid_assignment = self.data.create_assignment_in_answer_period( self.valid_course, self.data.get_authorized_instructor()) self.valid_start_date = now.isoformat() + 'Z' self.valid_end_date = (now + datetime.timedelta(days=90)).isoformat() + 'Z' self.invalid_end_date = (now - datetime.timedelta(days=1)).isoformat() + 'Z' self.valid_answer_start = now.isoformat() + 'Z' self.valid_answer_end = (now + datetime.timedelta(days=90)).isoformat() + 'Z' self.invalid_answer_start = ( now - datetime.timedelta(days=1)).isoformat() + 'Z' self.invalid_answer_end = ( now - datetime.timedelta(days=1)).isoformat() + 'Z' self.invalid_answer_end2 = ( now + datetime.timedelta(days=91)).isoformat() + 'Z' self.valid_compare_start = now.isoformat() + 'Z' self.valid_compare_end = ( now + datetime.timedelta(days=90)).isoformat() + 'Z' self.invalid_compare_start = ( now - datetime.timedelta(days=1)).isoformat() + 'Z' self.invalid_compare_end = ( now - datetime.timedelta(days=1)).isoformat() + 'Z' self.invalid_compare_end2 = ( now + datetime.timedelta(days=91)).isoformat() + 'Z' self.validation_url = '/api/courses/' + self.valid_course.uuid + '/duplicate' self.validate_expected_assignment = { 'id': self.valid_assignment.uuid, 'name': self.valid_assignment.name, 'answer_start': self.valid_answer_start, 'answer_end': self.valid_answer_end, 'compare_start': self.valid_compare_start, 'compare_end': self.valid_compare_end } self.validate_expected_course = { 'name': 'duplicate validation course', 'year': 2015, 'term': 'Winter', 'sandbox': False, 'start_date': self.valid_start_date, 'end_date': self.valid_end_date, 'assignments': [self.validate_expected_assignment] }
class AlgorithmValidityTests(ComPAIRAPITestCase): SCORING_ALGORITHM = ScoringAlgorithm.elo PAIRING_ALGORITHM = PairingAlgorithm.adaptive_min_delta NUMBER_OF_ANSWERS = 100 REPORT_PATH = None WINNER_SELECTOR = WinnerSelector.always_correct CORRECT_RATE = 1.0 ACTUAL_GRADES = None def create_app(self): settings = test_app_settings.copy() settings['SQLALCHEMY_DATABASE_URI'] = 'sqlite:///'+self._sqlite_db() app = create_app(settings_override=settings) return app def setUp(self): # remove existing sqlite db if exists self._delete_sqlite_db() # get a new seed (needed for ConcurrentTestSuite so they don't produce the same results) random.seed() numpy.random.seed() super(AlgorithmValidityTests, self).setUp() self.data = ComparisonTestData() self.MAX_COMPARSIONS = (NUMBER_OF_ANSWERS - 1) * (NUMBER_OF_ANSWERS - 2) / 2 self.TOTAL_MAX_ROUNDS = 50 self.COMPARISONS_IN_ROUND = math.ceil(NUMBER_OF_ANSWERS / 2) # stop after lowest of total comparisons possible or 100 rounds worth of comparisons are complete self.TOTAL_MAX_COMPARISONS = min( self.COMPARISONS_IN_ROUND * self.TOTAL_MAX_ROUNDS, NUMBER_OF_ANSWERS * self.MAX_COMPARSIONS ) self.course = self.data.create_course() self.instructor = self.data.create_instructor() self.data.enrol_instructor(self.instructor, self.course) self.assignment = self.data.create_assignment_in_comparison_period( self.course, self.instructor, number_of_comparisons=self.MAX_COMPARSIONS, scoring_algorithm=AlgorithmValidityTests.SCORING_ALGORITHM, pairing_algorithm=AlgorithmValidityTests.PAIRING_ALGORITHM) self.students = [] self.answers = [] self.grade_by_answer_uuid = {} actual_grades = ACTUAL_GRADES if not actual_grades: actual_grades = numpy.random.normal(0.78, 0.1, self.NUMBER_OF_ANSWERS) for grade in actual_grades: student = self.data.create_normal_user() self.data.enrol_student(student, self.course) self.students.append(student) answer = self.data.create_answer(self.assignment, student, with_score=False) self.answers.append(answer) self.grade_by_answer_uuid[answer.uuid] = grade self.base_url = self._build_url(self.course.uuid, self.assignment.uuid) db.session.commit() def tearDown(self): self._delete_sqlite_db() def _sqlite_db(self): return 'test_comparison'+str(os.getpid())+'.db' def _delete_sqlite_db(self): file_path = os.path.join(os.getcwd(), 'compair', self._sqlite_db()) if os.path.isfile(file_path): try: os.remove(file_path) except Exception as e: print(e) def _decide_winner(self, answer1_uuid, answer2_uuid): answer1_grade = self.grade_by_answer_uuid[answer1_uuid] answer2_grade = self.grade_by_answer_uuid[answer2_uuid] if AlgorithmValidityTests.WINNER_SELECTOR == WinnerSelector.always_correct: return self.always_correct(answer1_grade, answer2_grade) elif AlgorithmValidityTests.WINNER_SELECTOR == WinnerSelector.guessing: return self.guessing() elif AlgorithmValidityTests.WINNER_SELECTOR == WinnerSelector.correct_with_error: return self.correct_with_error(answer1_grade, answer2_grade, AlgorithmValidityTests.CORRECT_RATE) elif AlgorithmValidityTests.WINNER_SELECTOR == WinnerSelector.closely_matched_errors: return self.closely_matched_errors(answer1_grade, answer2_grade, AlgorithmValidityTests.CORRECT_RATE) else: raise Exception() def always_correct(self, value1, value2): return self.correct_with_error(value1, value2, 1.0) def correct_with_error(self, value1, value2, correct_rate): if value1 == value2: return self.guessing() correct_answer = WinningAnswer.answer1 if value1 > value2 else WinningAnswer.answer2 incorrect_answer = WinningAnswer.answer1 if value1 < value2 else WinningAnswer.answer2 return correct_answer if random.random() <= correct_rate else incorrect_answer def guessing(self): return WinningAnswer.answer1 if random.random() <= 0.5 else WinningAnswer.answer2 def closely_matched_errors(self, value1, value2, sigma): # make the actual values of answers fuzzy (represents perceived value errors) fuzzy_value1 = numpy.random.normal(value1, sigma, 1)[0] fuzzy_value2 = numpy.random.normal(value2, sigma, 1)[0] # return the correct winner using fuzzy perceived values return self.always_correct(fuzzy_value1, fuzzy_value2) def _build_url(self, course_uuid, assignment_uuid, tail=""): url = '/api/courses/' + course_uuid + '/assignments/' + assignment_uuid + '/comparisons' + tail return url def _build_comparison_submit(self, winner, draft=False): submit = { 'comparison_criteria': [], 'draft': draft } for criterion in self.assignment.criteria: submit['comparison_criteria'].append({ 'criterion_id': criterion.uuid, 'winner': winner, 'content': None }) return submit def test_random_students_perform_comparisons(self): self.student_comparison_count = { student.id: 0 for student in self.students } comparison_count = 0 round_count = 0 results = [] while comparison_count < self.TOTAL_MAX_COMPARISONS: # select a random student to answer student = random.choice(self.students) with self.login(student.username): # perform selection algorithm rv = self.client.get(self.base_url) self.assert200(rv) winner = self._decide_winner(rv.json['comparison']['answer1_id'], rv.json['comparison']['answer2_id']) comparison_submit = self._build_comparison_submit(winner.value) rv = self.client.post(self.base_url, data=json.dumps(comparison_submit), content_type='application/json') self.assert200(rv) comparison_count += 1 # remove students who have completed all comparisons self.student_comparison_count[student.id] += 1 if self.student_comparison_count[student.id] >= self.MAX_COMPARSIONS: indexes = [i for i, s in enumerate(self.students) if student.id == s.id] del self.students[indexes[0]] if comparison_count % self.COMPARISONS_IN_ROUND == 0: round_count += 1 actual_grades = [] current_scores = [] for answer in self.answers: answer_score = AnswerScore.query.filter_by(answer_id=answer.id).first() if answer_score: current_scores.append(answer_score.score) actual_grades.append(self.grade_by_answer_uuid[answer.uuid]) r_value, pearsonr_p_value = pearsonr(actual_grades, current_scores) #rho, spearmanr_p_value = spearmanr(actual_grades, current_scores) #tau, kendalltau_p_value = kendalltau(actual_grades, current_scores) results.append(str(r_value)) #results.append(str(rho)) #results.append(str(tau)) #print("Round {} ----------- pearsonr={} value=={} spearmanr={} value=={} kendalltau={} value=={}".format( # round_count, r_value, pearsonr_p_value, rho, spearmanr_p_value, tau, kendalltau_p_value #)) if r_value >= ACCEPTABLE_CORRELATION: break with open(AlgorithmValidityTests.REPORT_PATH, "a") as csvfile: out = csv.writer(csvfile) out.writerow(results)
class AlgorithmValidityTests(ComPAIRAPITestCase): SCORING_ALGORITHM = ScoringAlgorithm.elo PAIRING_ALGORITHM = PairingAlgorithm.adaptive_min_delta NUMBER_OF_ANSWERS = 100 REPORT_PATH = None WINNER_SELECTOR = WinnerSelector.always_correct CORRECT_RATE = 1.0 ACTUAL_GRADES = None def create_app(self): settings = test_app_settings.copy() settings['SQLALCHEMY_DATABASE_URI'] = 'sqlite:///' + self._sqlite_db() app = create_app(settings_override=settings) return app def setUp(self): # remove existing sqlite db if exists self._delete_sqlite_db() # get a new seed (needed for ConcurrentTestSuite so they don't produce the same results) random.seed() numpy.random.seed() super(AlgorithmValidityTests, self).setUp() self.data = ComparisonTestData() self.MAX_COMPARSIONS = (NUMBER_OF_ANSWERS - 1) * (NUMBER_OF_ANSWERS - 2) / 2 self.TOTAL_MAX_ROUNDS = 50 self.COMPARISONS_IN_ROUND = math.ceil(NUMBER_OF_ANSWERS / 2) # stop after lowest of total comparisons possible or 100 rounds worth of comparisons are complete self.TOTAL_MAX_COMPARISONS = min( self.COMPARISONS_IN_ROUND * self.TOTAL_MAX_ROUNDS, NUMBER_OF_ANSWERS * self.MAX_COMPARSIONS) self.course = self.data.create_course() self.instructor = self.data.create_instructor() self.data.enrol_instructor(self.instructor, self.course) self.assignment = self.data.create_assignment_in_comparison_period( self.course, self.instructor, number_of_comparisons=self.MAX_COMPARSIONS, scoring_algorithm=AlgorithmValidityTests.SCORING_ALGORITHM, pairing_algorithm=AlgorithmValidityTests.PAIRING_ALGORITHM) self.students = [] self.answers = [] self.grade_by_answer_uuid = {} actual_grades = ACTUAL_GRADES if not actual_grades: actual_grades = numpy.random.normal(0.78, 0.1, self.NUMBER_OF_ANSWERS) for grade in actual_grades: student = self.data.create_normal_user() self.data.enrol_student(student, self.course) self.students.append(student) answer = self.data.create_answer(self.assignment, student, with_score=False) self.answers.append(answer) self.grade_by_answer_uuid[answer.uuid] = grade self.base_url = self._build_url(self.course.uuid, self.assignment.uuid) db.session.commit() def tearDown(self): self._delete_sqlite_db() def _sqlite_db(self): return 'test_comparison' + str(os.getpid()) + '.db' def _delete_sqlite_db(self): file_path = os.path.join(os.getcwd(), 'compair', self._sqlite_db()) if os.path.isfile(file_path): try: os.remove(file_path) except Exception as e: print(e) def _decide_winner(self, answer1_uuid, answer2_uuid): answer1_grade = self.grade_by_answer_uuid[answer1_uuid] answer2_grade = self.grade_by_answer_uuid[answer2_uuid] if AlgorithmValidityTests.WINNER_SELECTOR == WinnerSelector.always_correct: return self.always_correct(answer1_grade, answer2_grade) elif AlgorithmValidityTests.WINNER_SELECTOR == WinnerSelector.guessing: return self.guessing() elif AlgorithmValidityTests.WINNER_SELECTOR == WinnerSelector.correct_with_error: return self.correct_with_error(answer1_grade, answer2_grade, AlgorithmValidityTests.CORRECT_RATE) elif AlgorithmValidityTests.WINNER_SELECTOR == WinnerSelector.closely_matched_errors: return self.closely_matched_errors( answer1_grade, answer2_grade, AlgorithmValidityTests.CORRECT_RATE) else: raise Exception() def always_correct(self, value1, value2): return self.correct_with_error(value1, value2, 1.0) def correct_with_error(self, value1, value2, correct_rate): if value1 == value2: return self.guessing() correct_answer = WinningAnswer.answer1 if value1 > value2 else WinningAnswer.answer2 incorrect_answer = WinningAnswer.answer1 if value1 < value2 else WinningAnswer.answer2 return correct_answer if random.random( ) <= correct_rate else incorrect_answer def guessing(self): return WinningAnswer.answer1 if random.random( ) <= 0.5 else WinningAnswer.answer2 def closely_matched_errors(self, value1, value2, sigma): # make the actual values of answers fuzzy (represents perceived value errors) fuzzy_value1 = numpy.random.normal(value1, sigma, 1)[0] fuzzy_value2 = numpy.random.normal(value2, sigma, 1)[0] # return the correct winner using fuzzy perceived values return self.always_correct(fuzzy_value1, fuzzy_value2) def _build_url(self, course_uuid, assignment_uuid, tail=""): url = '/api/courses/' + course_uuid + '/assignments/' + assignment_uuid + '/comparisons' + tail return url def _build_comparison_submit(self, winner, draft=False): submit = {'comparison_criteria': [], 'draft': draft} for criterion in self.assignment.criteria: submit['comparison_criteria'].append({ 'criterion_id': criterion.uuid, 'winner': winner, 'content': None }) return submit def test_random_students_perform_comparisons(self): self.student_comparison_count = { student.id: 0 for student in self.students } comparison_count = 0 round_count = 0 results = [] while comparison_count < self.TOTAL_MAX_COMPARISONS: # select a random student to answer student = random.choice(self.students) with self.login(student.username): # perform selection algorithm rv = self.client.get(self.base_url) self.assert200(rv) winner = self._decide_winner( rv.json['comparison']['answer1_id'], rv.json['comparison']['answer2_id']) comparison_submit = self._build_comparison_submit(winner.value) rv = self.client.post(self.base_url, data=json.dumps(comparison_submit), content_type='application/json') self.assert200(rv) comparison_count += 1 # remove students who have completed all comparisons self.student_comparison_count[student.id] += 1 if self.student_comparison_count[ student.id] >= self.MAX_COMPARSIONS: indexes = [ i for i, s in enumerate(self.students) if student.id == s.id ] del self.students[indexes[0]] if comparison_count % self.COMPARISONS_IN_ROUND == 0: round_count += 1 actual_grades = [] current_scores = [] for answer in self.answers: answer_score = AnswerScore.query.filter_by( answer_id=answer.id).first() if answer_score: current_scores.append(answer_score.score) actual_grades.append( self.grade_by_answer_uuid[answer.uuid]) r_value, pearsonr_p_value = pearsonr(actual_grades, current_scores) #rho, spearmanr_p_value = spearmanr(actual_grades, current_scores) #tau, kendalltau_p_value = kendalltau(actual_grades, current_scores) results.append(str(r_value)) #results.append(str(rho)) #results.append(str(tau)) #print("Round {} ----------- pearsonr={} value=={} spearmanr={} value=={} kendalltau={} value=={}".format( # round_count, r_value, pearsonr_p_value, rho, spearmanr_p_value, tau, kendalltau_p_value #)) if r_value >= ACCEPTABLE_CORRELATION: break with open(AlgorithmValidityTests.REPORT_PATH, "a") as csvfile: out = csv.writer(csvfile) out.writerow(results)
def setUp(self): super(CoursesDuplicateComplexAPITests, self).setUp() self.data = ComparisonTestData() self.course = self.data.get_course() # add start_date self.course.start_date = datetime.datetime.utcnow() self.course.end_date = datetime.datetime.utcnow() + datetime.timedelta(days=90) index = 0 self.assertGreaterEqual(self.course.assignments.count(), 4) # make sure we have enough assignments to play with for assignment in self.course.assignments: assignment.answer_start = self.course.start_date + datetime.timedelta(days=index) assignment.answer_end = self.course.start_date + datetime.timedelta(days=index+7) if assignment.compare_start != None: assignment.compare_start = self.course.start_date + datetime.timedelta(days=index+14) if assignment.compare_end != None: assignment.compare_end = self.course.start_date + datetime.timedelta(days=index+21) # modify some assignments with self-evaluation if index % 2 == 0: assignment.enable_self_evaluation = True else: assignment.enable_self_evaluation = False if index % 4 == 0: assignment.self_eval_start = assignment.answer_end + datetime.timedelta(days=1) assignment.self_eval_end = assignment.self_eval_start + datetime.timedelta(days=1) assignment.self_eval_instructions = ''.join([choice(ascii_letters) for i in range(10)]) else: assignment.self_eval_start = None assignment.self_eval_end = None assignment.self_eval_instructions = None index+=1 db.session.commit() self.url = '/api/courses/' + self.course.uuid + '/duplicate' self.date_delta = datetime.timedelta(days=180) self.expected = { 'name': 'duplicate course', 'year': 2015, 'term': 'Winter', 'sandbox': False, 'start_date': (self.course.start_date + self.date_delta).isoformat() + 'Z', 'end_date': (self.course.end_date + self.date_delta).isoformat() + 'Z', 'assignments': [] } for assignment in self.course.assignments: if not assignment.active: continue assignment_data = { 'id': assignment.uuid, 'name': assignment.name, 'answer_start': (assignment.answer_start + self.date_delta).isoformat() + 'Z', 'answer_end': (assignment.answer_end + self.date_delta).isoformat() + 'Z', 'enable_self_evaluation': assignment.enable_self_evaluation, 'self_eval_start': (assignment.self_eval_start + self.date_delta).isoformat() + 'Z' if assignment.self_eval_start else assignment.self_eval_start, 'self_eval_end': (assignment.self_eval_end + self.date_delta).isoformat() + 'Z' if assignment.self_eval_end else assignment.self_eval_end, 'self_eval_instructions': assignment.self_eval_instructions, } if assignment.compare_start != None: assignment_data['compare_start'] = (assignment.compare_start + self.date_delta).isoformat() + 'Z' if assignment.compare_end != None: assignment_data['compare_end'] = (assignment.compare_end + self.date_delta).isoformat() + 'Z' self.expected['assignments'].append(assignment_data) now = datetime.datetime.utcnow() self.valid_course = self.data.create_course() self.data.enrol_instructor(self.data.get_authorized_instructor(), self.valid_course) self.valid_assignment = self.data.create_assignment_in_answer_period(self.valid_course, self.data.get_authorized_instructor()) self.valid_start_date = now.isoformat() + 'Z' self.valid_end_date = (now + datetime.timedelta(days=120)).isoformat() + 'Z' self.invalid_end_date = (now - datetime.timedelta(days=1)).isoformat() + 'Z' self.valid_answer_start = now.isoformat() + 'Z' self.valid_answer_end = (now + datetime.timedelta(days=90)).isoformat() + 'Z' self.invalid_answer_start = (now - datetime.timedelta(days=1)).isoformat() + 'Z' self.invalid_answer_end = (now - datetime.timedelta(days=1)).isoformat() + 'Z' self.invalid_answer_end2 = (now + datetime.timedelta(days=121)).isoformat() + 'Z' self.valid_compare_start = now.isoformat() + 'Z' self.valid_compare_end = (now + datetime.timedelta(days=90)).isoformat() + 'Z' self.invalid_compare_start = (now - datetime.timedelta(days=1)).isoformat() + 'Z' self.invalid_compare_end = (now - datetime.timedelta(days=1)).isoformat() + 'Z' self.invalid_compare_end2 = (now + datetime.timedelta(days=121)).isoformat() + 'Z' self.valid_self_eval_start = (now + datetime.timedelta(days=92)).isoformat() + 'Z' self.valid_self_eval_end = (now + datetime.timedelta(days=120)).isoformat() + 'Z' self.invalid_self_eval_start = now.isoformat() + 'Z' # = compare_start self.invalid_self_eval_start2 = (now + datetime.timedelta(days=90)).isoformat() + 'Z' # = answer_end self.invalid_self_eval_end = (now + datetime.timedelta(days=91)).isoformat() + 'Z' # < self_eval_start self.invalid_self_eval_end2 = (now + datetime.timedelta(days=121)).isoformat() + 'Z' # > course_end self.validation_url = '/api/courses/' + self.valid_course.uuid + '/duplicate' self.validate_expected_assignment = { 'id': self.valid_assignment.uuid, 'name': self.valid_assignment.name, 'answer_start': self.valid_answer_start, 'answer_end': self.valid_answer_end, 'compare_start': self.valid_compare_start, 'compare_end': self.valid_compare_end } self.validate_expected_course = { 'name': 'duplicate validation course', 'year': 2015, 'term': 'Winter', 'sandbox': False, 'start_date': self.valid_start_date, 'end_date': self.valid_end_date, 'assignments': [self.validate_expected_assignment] }
class AlgorithmValidityTests(ComPAIRAPITestCase): # def create_app(self): # settings = test_app_settings.copy() # settings['SQLALCHEMY_DATABASE_URI'] = 'sqlite:///'+self._sqlite_db() # app = create_app(settings_override=settings) # return app def setUp(self): if SKIP_VALIDITY_TEST: self.skipTest("scipy and numpy not installed. run `make deps`") # remove existing sqlite db if exists # self._delete_sqlite_db() # TODO: Modify conditions to be more fuzzy (closely_matched_errors with 0.05 correct rate) # Depends on results of research super(AlgorithmValidityTests, self).setUp() self.data = ComparisonTestData() self.ACCEPTABLE_CORRELATION = 0.8 self.NUMBER_OF_ANSWERS = 40 self.WINNER_SELECTOR = WinnerSelector.always_correct self.CORRECT_RATE = 1.0 self.MAX_COMPARSIONS = (self.NUMBER_OF_ANSWERS - 1) * (self.NUMBER_OF_ANSWERS - 2) / 2 self.TOTAL_MAX_ROUNDS = 6 # 3 comparisons per student self.COMPARISONS_IN_ROUND = math.ceil(self.NUMBER_OF_ANSWERS / 2) # stop after lowest of total comparisons possible or 100 rounds worth of comparisons are complete self.TOTAL_MAX_COMPARISONS = min( self.COMPARISONS_IN_ROUND * self.TOTAL_MAX_ROUNDS, self.NUMBER_OF_ANSWERS * self.MAX_COMPARSIONS ) self.course = self.data.create_course() self.instructor = self.data.create_instructor() self.data.enrol_instructor(self.instructor, self.course) self.assignment = self.data.create_assignment_in_comparison_period( self.course, self.instructor, number_of_comparisons=self.MAX_COMPARSIONS, scoring_algorithm=ScoringAlgorithm.elo, pairing_algorithm=PairingAlgorithm.adaptive_min_delta ) self.students = [] self.answers = [] self.grade_by_answer_uuid = {} actual_grades = numpy.random.normal(0.78, 0.1, self.NUMBER_OF_ANSWERS) for grade in actual_grades: student = self.data.create_normal_user() self.data.enrol_student(student, self.course) self.students.append(student) answer = self.data.create_answer(self.assignment, student, with_score=False) self.answers.append(answer) self.grade_by_answer_uuid[answer.uuid] = grade self.base_url = self._build_url(self.course.uuid, self.assignment.uuid) db.session.commit() # def tearDown(self): # self._delete_sqlite_db() # def _sqlite_db(self): # return 'test_comparison'+str(os.getpid())+'.db' # def _delete_sqlite_db(self): # file_path = os.path.join(os.getcwd(), 'compair', self._sqlite_db()) # if os.path.isfile(file_path): # try: # os.remove(file_path) # except Exception as e: # print(e) def _decide_winner(self, answer1_uuid, answer2_uuid): answer1_grade = self.grade_by_answer_uuid[answer1_uuid] answer2_grade = self.grade_by_answer_uuid[answer2_uuid] if self.WINNER_SELECTOR == WinnerSelector.always_correct: return self.always_correct(answer1_grade, answer2_grade) elif self.WINNER_SELECTOR == WinnerSelector.guessing: return self.guessing() elif self.WINNER_SELECTOR == WinnerSelector.correct_with_error: return self.correct_with_error(answer1_grade, answer2_grade, self.CORRECT_RATE) elif self.WINNER_SELECTOR == WinnerSelector.closely_matched_errors: return self.closely_matched_errors(answer1_grade, answer2_grade, self.CORRECT_RATE) else: raise Exception() def always_correct(self, value1, value2): return self.correct_with_error(value1, value2, 1.0) def correct_with_error(self, value1, value2, correct_rate): if value1 == value2: return self.guessing() correct_answer = WinningAnswer.answer1 if value1 > value2 else WinningAnswer.answer2 incorrect_answer = WinningAnswer.answer1 if value1 < value2 else WinningAnswer.answer2 return correct_answer if random.random() <= correct_rate else incorrect_answer def guessing(self): return WinningAnswer.answer1 if random.random() <= 0.5 else WinningAnswer.answer2 def closely_matched_errors(self, value1, value2, sigma): # make the actual values of answers fuzzy (represents perceived value errors) fuzzy_value1 = numpy.random.normal(value1, sigma, 1)[0] fuzzy_value2 = numpy.random.normal(value2, sigma, 1)[0] # return the correct winner using fuzzy perceived values return self.always_correct(fuzzy_value1, fuzzy_value2) def _build_url(self, course_uuid, assignment_uuid, tail=""): url = '/api/courses/' + course_uuid + '/assignments/' + assignment_uuid + '/comparisons' + tail return url def _build_comparison_submit(self, winner, draft=False): submit = { 'comparison_criteria': [], 'draft': draft } for criterion in self.assignment.criteria: submit['comparison_criteria'].append({ 'criterion_id': criterion.uuid, 'winner': winner, 'content': None }) return submit def test_random_students_perform_comparisons(self): self.student_comparison_count = { student.id: 0 for student in self.students } comparison_count = 0 round_count = 0 r_value = None while comparison_count < self.TOTAL_MAX_COMPARISONS: # select a random student to answer student = random.choice(self.students) with self.login(student.username): # perform selection algorithm rv = self.client.get(self.base_url) self.assert200(rv) winner = self._decide_winner(rv.json['comparison']['answer1_id'], rv.json['comparison']['answer2_id']) comparison_submit = self._build_comparison_submit(winner.value) rv = self.client.post(self.base_url, data=json.dumps(comparison_submit), content_type='application/json') self.assert200(rv) comparison_count += 1 # remove students who have completed all comparisons self.student_comparison_count[student.id] += 1 if self.student_comparison_count[student.id] >= self.MAX_COMPARSIONS: indexes = [i for i, s in enumerate(self.students) if student.id == s.id] del self.students[indexes[0]] if comparison_count % self.COMPARISONS_IN_ROUND == 0: round_count += 1 actual_grades = [] current_scores = [] for answer in self.answers: answer_score = AnswerScore.query.filter_by(answer_id=answer.id).first() if answer_score: current_scores.append(answer_score.score) actual_grades.append(self.grade_by_answer_uuid[answer.uuid]) r_value, pearsonr_p_value = pearsonr(actual_grades, current_scores) if r_value >= self.ACCEPTABLE_CORRELATION: break self.assertGreaterEqual(r_value, self.ACCEPTABLE_CORRELATION) self.assertLessEqual(round_count, self.TOTAL_MAX_ROUNDS)
class CoursesDuplicateComplexAPITests(ComPAIRAPITestCase): def setUp(self): super(CoursesDuplicateComplexAPITests, self).setUp() self.data = ComparisonTestData() self.course = self.data.get_course() # add start_date self.course.start_date = datetime.datetime.utcnow() self.course.end_date = datetime.datetime.utcnow() + datetime.timedelta(days=90) db.session.commit() self.url = '/api/courses/' + self.course.uuid + '/duplicate' self.date_delta = datetime.timedelta(days=180) self.expected = { 'name': 'duplicate course', 'year': 2015, 'term': 'Winter', 'start_date': (self.course.start_date + self.date_delta).isoformat() + 'Z', 'end_date': (self.course.end_date + self.date_delta).isoformat() + 'Z', 'assignments': [] } for assignment in self.course.assignments: if not assignment.active: continue assignment_data = { 'id': assignment.uuid, 'answer_start': (assignment.answer_start + self.date_delta).isoformat() + 'Z', 'answer_end': (assignment.answer_end + self.date_delta).isoformat() + 'Z' } if assignment.compare_start != None: assignment_data['compare_start'] = (assignment.compare_start + self.date_delta).isoformat() + 'Z' if assignment.compare_end != None: assignment_data['compare_end'] = (assignment.compare_end + self.date_delta).isoformat() + 'Z' self.expected['assignments'].append(assignment_data) def test_duplicate_course_complex(self): original_course = self.data.get_course() # test authorized user with self.login(self.data.get_authorized_instructor().username): rv = self.client.post(self.url, data=json.dumps(self.expected), content_type='application/json') self.assert200(rv) duplicate_course = Course.query.filter_by(uuid=rv.json['id']).first() self.assertIsNotNone(duplicate_course) # verify course duplicated correctly self.assertNotEqual(original_course.id, duplicate_course.id) self.assertNotEqual(original_course.uuid, duplicate_course.uuid) self.assertEqual(self.expected['name'], duplicate_course.name) self.assertEqual(self.expected['year'], duplicate_course.year) self.assertEqual(self.expected['term'], duplicate_course.term) self.assertEqual(self.expected['start_date'].replace('Z', ''), rv.json['start_date'].replace('+00:00', '')) self.assertEqual(self.expected['end_date'].replace('Z', ''), rv.json['end_date'].replace('+00:00', '')) self.assertEqual(original_course.description, duplicate_course.description) # verify instructor added to duplicate course user_course = UserCourse.query \ .filter_by( user_id=self.data.get_authorized_instructor().id, course_uuid=rv.json['id'] ) \ .one_or_none() self.assertIsNotNone(user_course) original_assignments = original_course.assignments.all() duplicate_assignments = duplicate_course.assignments.all() self.assertEqual(len(original_assignments), 3) self.assertEqual(len(original_assignments), len(duplicate_assignments)) for index, original_assignment in enumerate(original_assignments): duplicate_assignment = duplicate_assignments[index] self.assertNotEqual(original_assignment.id, duplicate_assignment.id) self.assertNotEqual(original_assignment.uuid, duplicate_assignment.uuid) self.assertEqual(duplicate_course.id, duplicate_assignment.course_id) self.assertEqual(original_assignment.name, duplicate_assignment.name) self.assertEqual(original_assignment.description, duplicate_assignment.description) self.assertEqual(original_assignment.number_of_comparisons, duplicate_assignment.number_of_comparisons) self.assertEqual(original_assignment.students_can_reply, duplicate_assignment.students_can_reply) self.assertEqual(original_assignment.enable_self_evaluation, duplicate_assignment.enable_self_evaluation) self.assertEqual(original_assignment.pairing_algorithm, duplicate_assignment.pairing_algorithm) self.assertEqual(original_assignment.answer_start, (duplicate_assignment.answer_start - self.date_delta)) self.assertEqual(original_assignment.answer_end, (duplicate_assignment.answer_end - self.date_delta)) if original_assignment.compare_start != None: self.assertEqual(original_assignment.compare_start, (duplicate_assignment.compare_start - self.date_delta)) else: self.assertIsNone(duplicate_assignment.compare_start) if original_assignment.compare_end != None: self.assertEqual(original_assignment.compare_end, (duplicate_assignment.compare_end - self.date_delta)) else: self.assertIsNone(duplicate_assignment.compare_end) self.assertEqual(len(original_assignment.criteria), 1) self.assertEqual(len(original_assignment.criteria), len(duplicate_assignment.criteria)) for index, original_criteria in enumerate(original_assignment.criteria): duplicate_criteria = duplicate_assignment.criteria[index] self.assertEqual(original_criteria.id, duplicate_criteria.id) original_comparison_examples = original_assignment.comparison_examples.all() duplicate_comparison_examples = duplicate_assignment.comparison_examples.all() if original_assignment.id in [1,2]: self.assertEqual(len(original_comparison_examples), 1) else: self.assertEqual(len(original_comparison_examples), 0) self.assertEqual(len(original_comparison_examples), len(duplicate_comparison_examples)) for index, original_comparison_example in enumerate(original_comparison_examples): duplicate_comparison_example = duplicate_comparison_examples[index] self.assertNotEqual(original_comparison_example.id, duplicate_comparison_example.id) self.assertNotEqual(original_comparison_example.answer1_id, duplicate_comparison_example.answer1_id) self.assertNotEqual(original_comparison_example.answer2_id, duplicate_comparison_example.answer2_id) self.assertEqual(duplicate_assignment.id, duplicate_comparison_example.assignment_id) original_answer1 = original_comparison_example.answer1 duplicate_answer1 = duplicate_comparison_example.answer1 self.assertNotEqual(original_answer1.id, duplicate_answer1.id) self.assertEqual(duplicate_assignment.id, duplicate_answer1.assignment_id) self.assertEqual(original_answer1.content, duplicate_answer1.content) self.assertEqual(original_answer1.practice, duplicate_answer1.practice) self.assertEqual(original_answer1.active, duplicate_answer1.active) self.assertEqual(original_answer1.draft, duplicate_answer1.draft) original_answer2 = original_comparison_example.answer2 duplicate_answer2 = duplicate_comparison_example.answer2 self.assertNotEqual(original_answer2.id, duplicate_answer2.id) self.assertEqual(duplicate_assignment.id, duplicate_answer2.assignment_id) self.assertEqual(original_answer2.content, duplicate_answer2.content) self.assertEqual(original_answer2.practice, original_answer2.practice) self.assertEqual(original_answer2.active, original_answer2.active) self.assertEqual(original_answer2.draft, original_answer2.draft)
def setUp(self): super(UsersCourseStatusAPITests, self).setUp() self.data = ComparisonTestData()
class UsersCourseStatusAPITests(ComPAIRAPITestCase): def setUp(self): super(UsersCourseStatusAPITests, self).setUp() self.data = ComparisonTestData() def _create_enough_answers_for_assignment(self, assignment): course = assignment.course for i in range(assignment.number_of_comparisons*2): student = self.data.create_normal_user() self.data.enrol_student(student, course) self.data.create_answer(assignment, student) def _submit_all_comparisons_for_assignment(self, assignment, user_id): submit_count = 0 for comparison_example in assignment.comparison_examples: comparisons = Comparison.create_new_comparison_set(assignment.id, user_id, False) self.assertEqual(comparisons[0].answer1_id, comparison_example.answer1_id) self.assertEqual(comparisons[0].answer2_id, comparison_example.answer2_id) for comparison in comparisons: comparison.completed = True comparison.winner_id = min([comparisons[0].answer1_id, comparisons[0].answer2_id]) db.session.add(comparison) submit_count += 1 db.session.commit() for i in range(assignment.number_of_comparisons): comparisons = Comparison.create_new_comparison_set(assignment.id, user_id, False) for comparison in comparisons: comparison.completed = True comparison.winner_id = min([comparisons[0].answer1_id, comparisons[0].answer2_id]) db.session.add(comparison) submit_count += 1 db.session.commit() Comparison.calculate_scores(assignment.id) return submit_count def test_get_course_list(self): # test login required course = self.data.get_course() url = '/api/users/courses/status?ids='+course.uuid rv = self.client.get(url) self.assert401(rv) with self.login(self.data.get_authorized_student().username): # ids missing rv = self.client.get('/api/users/courses/status') self.assert400(rv) # empty ids rv = self.client.get('/api/users/courses/status?ids=') self.assert400(rv) # empty course doesn't exists rv = self.client.get('/api/users/courses/status?ids=999') self.assert400(rv) # test authorized student rv = self.client.get(url) self.assert200(rv) self.assertEqual(1, len(rv.json['statuses'])) self.assertIn(course.uuid, rv.json['statuses']) # test course with no assignments course = self.data.create_course() self.data.enrol_student(self.data.get_authorized_student(), course) self.data.enrol_instructor(self.data.get_authorized_instructor(), course) url = '/api/users/courses/status?ids='+course.uuid rv = self.client.get(url) self.assert200(rv) self.assertEqual(1, len(rv.json['statuses'])) self.assertIn(course.uuid, rv.json['statuses']) self.assertEqual(0, rv.json['statuses'][course.uuid]['incomplete_assignments']) # test course with 2 assignment self.data.create_assignment(course, self.data.get_authorized_instructor(), datetime.datetime.now() + datetime.timedelta(days=7), datetime.datetime.now() + datetime.timedelta(days=14)) past_assignment = self.data.create_assignment(course, self.data.get_authorized_instructor(), datetime.datetime.now() - datetime.timedelta(days=28), datetime.datetime.now() - datetime.timedelta(days=21)) past_assignment.compare_start = datetime.datetime.now() - datetime.timedelta(days=14) past_assignment.compare_end = datetime.datetime.now() - datetime.timedelta(days=7) db.session.commit() rv = self.client.get(url) self.assert200(rv) self.assertEqual(1, len(rv.json['statuses'])) self.assertEqual(0, rv.json['statuses'][course.uuid]['incomplete_assignments']) # test course with 3 assignment # - 1 in answering period answering_assignment = self.data.create_assignment_in_answer_period(course, self.data.get_authorized_instructor()) rv = self.client.get(url) self.assert200(rv) self.assertEqual(1, len(rv.json['statuses'])) self.assertEqual(1, rv.json['statuses'][course.uuid]['incomplete_assignments']) # test course with 5 assignment # - 1 in answering period # - 1 in comparison_period # - 1 in comparison_period with self evaluation comparing_assignment = self.data.create_assignment_in_comparison_period(course, self.data.get_authorized_instructor()) comparing_assignment_self_eval = self.data.create_assignment_in_comparison_period(course, self.data.get_authorized_instructor()) comparing_assignment_self_eval.enable_self_evaluation = True db.session.commit() rv = self.client.get(url) self.assert200(rv) self.assertEqual(1, len(rv.json['statuses'])) self.assertEqual(3, rv.json['statuses'][course.uuid]['incomplete_assignments']) # test course with 5 assignment # - 1 in answering period (answered) # - 1 in comparison_period # - 1 in comparison_period with self evaluation self.data.create_answer(answering_assignment, self.data.get_authorized_student()) rv = self.client.get(url) self.assert200(rv) self.assertEqual(1, len(rv.json['statuses'])) self.assertEqual(2, rv.json['statuses'][course.uuid]['incomplete_assignments']) # test course with 5 assignment # - 1 in answering period (answered) # - 1 in comparison_period (fully compared) # - 1 in comparison_period with self evaluation self._create_enough_answers_for_assignment(comparing_assignment) self._submit_all_comparisons_for_assignment(comparing_assignment, self.data.get_authorized_student().id) rv = self.client.get(url) self.assert200(rv) self.assertEqual(1, len(rv.json['statuses'])) self.assertEqual(1, rv.json['statuses'][course.uuid]['incomplete_assignments']) # test course with 5 assignment # - 1 in answering period (answered) # - 1 in comparison_period (fully compared) # - 1 in comparison_period with self evaluation (fully compared, not self-evaulated) self._create_enough_answers_for_assignment(comparing_assignment_self_eval) self._submit_all_comparisons_for_assignment(comparing_assignment_self_eval, self.data.get_authorized_student().id) rv = self.client.get(url) self.assert200(rv) self.assertEqual(1, len(rv.json['statuses'])) self.assertEqual(1, rv.json['statuses'][course.uuid]['incomplete_assignments']) # test course with 5 assignment # - 1 in answering period (answered) # - 1 in comparison_period (fully compared) # - 1 in comparison_period with self evaluation (fully compared and self-evaulated) answer = self.data.create_answer(comparing_assignment_self_eval, self.data.get_authorized_student()) answer_comment = AnswerComment( user_id=self.data.get_authorized_student().id, answer_id=answer.id, comment_type=AnswerCommentType.self_evaluation, draft=False ) db.session.add(answer_comment) db.session.commit() rv = self.client.get(url) self.assert200(rv) self.assertEqual(1, len(rv.json['statuses'])) self.assertEqual(0, rv.json['statuses'][course.uuid]['incomplete_assignments'])
class CoursesDuplicateComplexAPITests(ComPAIRAPITestCase): def setUp(self): super(CoursesDuplicateComplexAPITests, self).setUp() self.data = ComparisonTestData() self.course = self.data.get_course() # add start_date self.course.start_date = datetime.datetime.utcnow() self.course.end_date = datetime.datetime.utcnow() + datetime.timedelta(days=90) index = 0 self.assertGreaterEqual(self.course.assignments.count(), 4) # make sure we have enough assignments to play with for assignment in self.course.assignments: assignment.answer_start = self.course.start_date + datetime.timedelta(days=index) assignment.answer_end = self.course.start_date + datetime.timedelta(days=index+7) if assignment.compare_start != None: assignment.compare_start = self.course.start_date + datetime.timedelta(days=index+14) if assignment.compare_end != None: assignment.compare_end = self.course.start_date + datetime.timedelta(days=index+21) # modify some assignments with self-evaluation if index % 2 == 0: assignment.enable_self_evaluation = True else: assignment.enable_self_evaluation = False if index % 4 == 0: assignment.self_eval_start = assignment.answer_end + datetime.timedelta(days=1) assignment.self_eval_end = assignment.self_eval_start + datetime.timedelta(days=1) assignment.self_eval_instructions = ''.join([choice(ascii_letters) for i in range(10)]) else: assignment.self_eval_start = None assignment.self_eval_end = None assignment.self_eval_instructions = None index+=1 db.session.commit() self.url = '/api/courses/' + self.course.uuid + '/duplicate' self.date_delta = datetime.timedelta(days=180) self.expected = { 'name': 'duplicate course', 'year': 2015, 'term': 'Winter', 'sandbox': False, 'start_date': (self.course.start_date + self.date_delta).isoformat() + 'Z', 'end_date': (self.course.end_date + self.date_delta).isoformat() + 'Z', 'assignments': [] } for assignment in self.course.assignments: if not assignment.active: continue assignment_data = { 'id': assignment.uuid, 'name': assignment.name, 'answer_start': (assignment.answer_start + self.date_delta).isoformat() + 'Z', 'answer_end': (assignment.answer_end + self.date_delta).isoformat() + 'Z', 'enable_self_evaluation': assignment.enable_self_evaluation, 'self_eval_start': (assignment.self_eval_start + self.date_delta).isoformat() + 'Z' if assignment.self_eval_start else assignment.self_eval_start, 'self_eval_end': (assignment.self_eval_end + self.date_delta).isoformat() + 'Z' if assignment.self_eval_end else assignment.self_eval_end, 'self_eval_instructions': assignment.self_eval_instructions, } if assignment.compare_start != None: assignment_data['compare_start'] = (assignment.compare_start + self.date_delta).isoformat() + 'Z' if assignment.compare_end != None: assignment_data['compare_end'] = (assignment.compare_end + self.date_delta).isoformat() + 'Z' self.expected['assignments'].append(assignment_data) now = datetime.datetime.utcnow() self.valid_course = self.data.create_course() self.data.enrol_instructor(self.data.get_authorized_instructor(), self.valid_course) self.valid_assignment = self.data.create_assignment_in_answer_period(self.valid_course, self.data.get_authorized_instructor()) self.valid_start_date = now.isoformat() + 'Z' self.valid_end_date = (now + datetime.timedelta(days=120)).isoformat() + 'Z' self.invalid_end_date = (now - datetime.timedelta(days=1)).isoformat() + 'Z' self.valid_answer_start = now.isoformat() + 'Z' self.valid_answer_end = (now + datetime.timedelta(days=90)).isoformat() + 'Z' self.invalid_answer_start = (now - datetime.timedelta(days=1)).isoformat() + 'Z' self.invalid_answer_end = (now - datetime.timedelta(days=1)).isoformat() + 'Z' self.invalid_answer_end2 = (now + datetime.timedelta(days=121)).isoformat() + 'Z' self.valid_compare_start = now.isoformat() + 'Z' self.valid_compare_end = (now + datetime.timedelta(days=90)).isoformat() + 'Z' self.invalid_compare_start = (now - datetime.timedelta(days=1)).isoformat() + 'Z' self.invalid_compare_end = (now - datetime.timedelta(days=1)).isoformat() + 'Z' self.invalid_compare_end2 = (now + datetime.timedelta(days=121)).isoformat() + 'Z' self.valid_self_eval_start = (now + datetime.timedelta(days=92)).isoformat() + 'Z' self.valid_self_eval_end = (now + datetime.timedelta(days=120)).isoformat() + 'Z' self.invalid_self_eval_start = now.isoformat() + 'Z' # = compare_start self.invalid_self_eval_start2 = (now + datetime.timedelta(days=90)).isoformat() + 'Z' # = answer_end self.invalid_self_eval_end = (now + datetime.timedelta(days=91)).isoformat() + 'Z' # < self_eval_start self.invalid_self_eval_end2 = (now + datetime.timedelta(days=121)).isoformat() + 'Z' # > course_end self.validation_url = '/api/courses/' + self.valid_course.uuid + '/duplicate' self.validate_expected_assignment = { 'id': self.valid_assignment.uuid, 'name': self.valid_assignment.name, 'answer_start': self.valid_answer_start, 'answer_end': self.valid_answer_end, 'compare_start': self.valid_compare_start, 'compare_end': self.valid_compare_end } self.validate_expected_course = { 'name': 'duplicate validation course', 'year': 2015, 'term': 'Winter', 'sandbox': False, 'start_date': self.valid_start_date, 'end_date': self.valid_end_date, 'assignments': [self.validate_expected_assignment] } def test_duplicate_course_complex(self): original_course = self.data.get_course() # test authorized user with self.login(self.data.get_authorized_instructor().username): # test valid expected rv = self.client.post(self.validation_url, data=json.dumps(self.validate_expected_course), content_type='application/json') self.assert200(rv) # test invalid course start/end dates invalid_expected = self.validate_expected_course.copy() invalid_expected['end_date'] = self.invalid_end_date rv = self.client.post(self.validation_url, data=json.dumps(invalid_expected), content_type='application/json') self.assert400(rv) self.assertEqual(rv.json["title"], "Course Not Saved") self.assertEqual(rv.json["message"], "Course end time must be after course start time.") # test invalid assignment answer start invalid_expected = self.validate_expected_course.copy() invalid_expected['assignments'][0] = self.validate_expected_assignment.copy() invalid_expected['assignments'][0]['answer_start'] = None rv = self.client.post(self.validation_url, data=json.dumps(invalid_expected), content_type='application/json') self.assert400(rv) self.assertEqual(rv.json["title"], "Course Not Saved") self.assertEqual(rv.json["message"], "No answer period start time provided for assignment "+self.valid_assignment.name+".") # test invalid assignment answer end invalid_expected = self.validate_expected_course.copy() invalid_expected['assignments'][0] = self.validate_expected_assignment.copy() invalid_expected['assignments'][0]['answer_end'] = None rv = self.client.post(self.validation_url, data=json.dumps(invalid_expected), content_type='application/json') self.assert400(rv) self.assertEqual(rv.json["title"], "Course Not Saved") self.assertEqual(rv.json["message"], "No answer period end time provided for assignment "+self.valid_assignment.name+".") invalid_expected = self.validate_expected_course.copy() invalid_expected['assignments'][0] = self.validate_expected_assignment.copy() invalid_expected['assignments'][0]['answer_end'] = self.invalid_answer_end rv = self.client.post(self.validation_url, data=json.dumps(invalid_expected), content_type='application/json') self.assert400(rv) self.assertEqual(rv.json["title"], "Course Not Saved") self.assertEqual(rv.json["message"], "Answer period end time must be after the answer start time for assignment "+self.valid_assignment.name+".") invalid_expected = self.validate_expected_course.copy() invalid_expected['assignments'][0] = self.validate_expected_assignment.copy() invalid_expected['assignments'][0]['answer_end'] = self.invalid_answer_end2 rv = self.client.post(self.validation_url, data=json.dumps(invalid_expected), content_type='application/json') self.assert400(rv) self.assertEqual(rv.json["title"], "Course Not Saved") self.assertEqual(rv.json["message"], "Answer period end time must be before the course end time for assignment "+self.valid_assignment.name+".") # test invalid assignment compare start invalid_expected = self.validate_expected_course.copy() invalid_expected['assignments'][0] = self.validate_expected_assignment.copy() invalid_expected['assignments'][0]['compare_start'] = None rv = self.client.post(self.validation_url, data=json.dumps(invalid_expected), content_type='application/json') self.assert400(rv) self.assertEqual(rv.json["title"], "Course Not Saved") self.assertEqual(rv.json["message"], "No compare period start time provided for assignment "+self.valid_assignment.name+".") invalid_expected = self.validate_expected_course.copy() invalid_expected['assignments'][0] = self.validate_expected_assignment.copy() invalid_expected['assignments'][0]['compare_start'] = self.invalid_compare_start rv = self.client.post(self.validation_url, data=json.dumps(invalid_expected), content_type='application/json') self.assert400(rv) self.assertEqual(rv.json["title"], "Course Not Saved") self.assertEqual(rv.json["message"], "Compare period start time must be after the answer start time for assignment "+self.valid_assignment.name+".") # test invalid assignment compare end invalid_expected = self.validate_expected_course.copy() invalid_expected['assignments'][0] = self.validate_expected_assignment.copy() invalid_expected['assignments'][0]['compare_end'] = None rv = self.client.post(self.validation_url, data=json.dumps(invalid_expected), content_type='application/json') self.assert400(rv) self.assertEqual(rv.json["title"], "Course Not Saved") self.assertEqual(rv.json["message"], "No compare period end time provided for assignment "+self.valid_assignment.name+".") invalid_expected = self.validate_expected_course.copy() invalid_expected['assignments'][0] = self.validate_expected_assignment.copy() invalid_expected['assignments'][0]['compare_end'] = self.invalid_compare_end rv = self.client.post(self.validation_url, data=json.dumps(invalid_expected), content_type='application/json') self.assert400(rv) self.assertEqual(rv.json["title"], "Course Not Saved") self.assertEqual(rv.json["message"], "Compare period end time must be after the compare start time for assignment "+self.valid_assignment.name+".") invalid_expected = self.validate_expected_course.copy() invalid_expected['assignments'][0] = self.validate_expected_assignment.copy() invalid_expected['assignments'][0]['compare_end'] = self.invalid_compare_end2 rv = self.client.post(self.validation_url, data=json.dumps(invalid_expected), content_type='application/json') self.assert400(rv) self.assertEqual(rv.json["title"], "Course Not Saved") self.assertEqual(rv.json["message"], "Compare period end time must be before the course end time for assignment "+self.valid_assignment.name+".") # test invalid assignment self-eval start invalid_expected = self.validate_expected_course.copy() invalid_expected['assignments'][0] = self.validate_expected_assignment.copy() invalid_expected['assignments'][0]['self_eval_start'] = None invalid_expected['assignments'][0]['self_eval_end'] = self.valid_self_eval_end rv = self.client.post(self.validation_url, data=json.dumps(invalid_expected), content_type='application/json') self.assert400(rv) self.assertEqual(rv.json["title"], "Course Not Saved") self.assertEqual(rv.json["message"], "No self-evaluation start time provided for assignment "+self.valid_assignment.name+".") invalid_expected = self.validate_expected_course.copy() invalid_expected['assignments'][0] = self.validate_expected_assignment.copy() invalid_expected['assignments'][0]['self_eval_start'] = self.invalid_self_eval_start invalid_expected['assignments'][0]['self_eval_end'] = self.valid_self_eval_end rv = self.client.post(self.validation_url, data=json.dumps(invalid_expected), content_type='application/json') self.assert400(rv) self.assertEqual(rv.json["title"], "Course Not Saved") self.assertEqual(rv.json["message"], "Self-evaluation start time must be after the compare start time for assignment "+self.valid_assignment.name+".") invalid_expected = self.validate_expected_course.copy() invalid_expected['assignments'][0] = self.validate_expected_assignment.copy() invalid_expected['assignments'][0]['compare_start'] = None invalid_expected['assignments'][0]['compare_end'] = None invalid_expected['assignments'][0]['self_eval_start'] = self.invalid_self_eval_start2 invalid_expected['assignments'][0]['self_eval_end'] = self.valid_self_eval_end rv = self.client.post(self.validation_url, data=json.dumps(invalid_expected), content_type='application/json') self.assert400(rv) self.assertEqual(rv.json["title"], "Course Not Saved") self.assertEqual(rv.json["message"], "Self-evaluation start time must be after the answer end time for assignment "+self.valid_assignment.name+".") # test invalid assignment self-eval end invalid_expected = self.validate_expected_course.copy() invalid_expected['assignments'][0] = self.validate_expected_assignment.copy() invalid_expected['assignments'][0]['self_eval_start'] = self.valid_self_eval_start invalid_expected['assignments'][0]['self_eval_end'] = None rv = self.client.post(self.validation_url, data=json.dumps(invalid_expected), content_type='application/json') self.assert400(rv) self.assertEqual(rv.json["title"], "Course Not Saved") self.assertEqual(rv.json["message"], "No self-evaluation end time provided for assignment "+self.valid_assignment.name+".") invalid_expected = self.validate_expected_course.copy() invalid_expected['assignments'][0] = self.validate_expected_assignment.copy() invalid_expected['assignments'][0]['self_eval_start'] = self.valid_self_eval_start invalid_expected['assignments'][0]['self_eval_end'] = self.invalid_self_eval_end rv = self.client.post(self.validation_url, data=json.dumps(invalid_expected), content_type='application/json') self.assert400(rv) self.assertEqual(rv.json["title"], "Course Not Saved") self.assertEqual(rv.json["message"], "Self-evaluation end time must be after the self-evaluation start time for assignment "+self.valid_assignment.name+".") invalid_expected = self.validate_expected_course.copy() invalid_expected['assignments'][0] = self.validate_expected_assignment.copy() invalid_expected['assignments'][0]['self_eval_start'] = self.valid_self_eval_start invalid_expected['assignments'][0]['self_eval_end'] = self.invalid_self_eval_end2 rv = self.client.post(self.validation_url, data=json.dumps(invalid_expected), content_type='application/json') self.assert400(rv) self.assertEqual(rv.json["title"], "Course Not Saved") self.assertEqual(rv.json["message"], "Self-evaluation end time must be before the course end time for assignment "+self.valid_assignment.name+".") # test deep copy assignments rv = self.client.post(self.url, data=json.dumps(self.expected), content_type='application/json') self.assert200(rv) duplicate_course = Course.query.filter_by(uuid=rv.json['id']).first() self.assertIsNotNone(duplicate_course) # verify course duplicated correctly self.assertNotEqual(original_course.id, duplicate_course.id) self.assertNotEqual(original_course.uuid, duplicate_course.uuid) self.assertEqual(self.expected['name'], duplicate_course.name) self.assertEqual(self.expected['year'], duplicate_course.year) self.assertEqual(self.expected['term'], duplicate_course.term) self.assertEqual(self.expected['sandbox'], duplicate_course.sandbox) self.assertEqual(self.expected['start_date'].replace('Z', ''), rv.json['start_date'].replace('+00:00', '')) self.assertEqual(self.expected['end_date'].replace('Z', ''), rv.json['end_date'].replace('+00:00', '')) # verify instructor added to duplicate course user_course = UserCourse.query \ .filter_by( user_id=self.data.get_authorized_instructor().id, course_uuid=rv.json['id'] ) \ .one_or_none() self.assertIsNotNone(user_course) original_assignments = original_course.assignments.all() duplicate_assignments = duplicate_course.assignments.all() self.assertEqual(len(original_assignments), 5) self.assertEqual(len(original_assignments), len(duplicate_assignments)) for index, original_assignment in enumerate(original_assignments): duplicate_assignment = duplicate_assignments[index] self.assertNotEqual(original_assignment.id, duplicate_assignment.id) self.assertNotEqual(original_assignment.uuid, duplicate_assignment.uuid) self.assertEqual(duplicate_course.id, duplicate_assignment.course_id) self.assertEqual(original_assignment.name, duplicate_assignment.name) self.assertEqual(original_assignment.description, duplicate_assignment.description) self.assertEqual(original_assignment.number_of_comparisons, duplicate_assignment.number_of_comparisons) self.assertEqual(original_assignment.students_can_reply, duplicate_assignment.students_can_reply) self.assertEqual(original_assignment.enable_self_evaluation, duplicate_assignment.enable_self_evaluation) self.assertEqual(original_assignment.pairing_algorithm, duplicate_assignment.pairing_algorithm) self.assertEqual(original_assignment.answer_grade_weight, duplicate_assignment.answer_grade_weight) self.assertEqual(original_assignment.comparison_grade_weight, duplicate_assignment.comparison_grade_weight) self.assertEqual(original_assignment.self_evaluation_grade_weight, duplicate_assignment.self_evaluation_grade_weight) self.assertEqual(original_assignment.enable_group_answers, duplicate_assignment.enable_group_answers) self.assertEqual(original_assignment.scoring_algorithm, duplicate_assignment.scoring_algorithm) self.assertEqual(original_assignment.peer_feedback_prompt, duplicate_assignment.peer_feedback_prompt) self.assertEqual(original_assignment.educators_can_compare, duplicate_assignment.educators_can_compare) self.assertEqual(original_assignment.rank_display_limit, duplicate_assignment.rank_display_limit) self.assertEqual(original_assignment.answer_start, (duplicate_assignment.answer_start - self.date_delta)) self.assertEqual(original_assignment.answer_end, (duplicate_assignment.answer_end - self.date_delta)) if original_assignment.compare_start != None: self.assertEqual(original_assignment.compare_start, (duplicate_assignment.compare_start - self.date_delta)) else: self.assertIsNone(duplicate_assignment.compare_start) if original_assignment.compare_end != None: self.assertEqual(original_assignment.compare_end, (duplicate_assignment.compare_end - self.date_delta)) else: self.assertIsNone(duplicate_assignment.compare_end) self.assertEqual(len(original_assignment.criteria), 1) self.assertEqual(len(original_assignment.criteria), len(duplicate_assignment.criteria)) for index, original_criteria in enumerate(original_assignment.criteria): duplicate_criteria = duplicate_assignment.criteria[index] self.assertEqual(original_criteria.id, duplicate_criteria.id) self.assertEqual(original_assignment.enable_self_evaluation, duplicate_assignment.enable_self_evaluation) if original_assignment.self_eval_start: self.assertEqual(original_assignment.self_eval_start, (duplicate_assignment.self_eval_start - self.date_delta)) else: self.assertIsNone(duplicate_assignment.self_eval_start) if original_assignment.self_eval_end: self.assertEqual(original_assignment.self_eval_end, (duplicate_assignment.self_eval_end - self.date_delta)) else: self.assertIsNone(duplicate_assignment.self_eval_end) self.assertEqual(original_assignment.self_eval_instructions, duplicate_assignment.self_eval_instructions) original_comparison_examples = original_assignment.comparison_examples.all() duplicate_comparison_examples = duplicate_assignment.comparison_examples.all() if original_assignment.id in [1,2,3,4]: self.assertEqual(len(original_comparison_examples), 1) else: self.assertEqual(len(original_comparison_examples), 0) self.assertEqual(len(original_comparison_examples), len(duplicate_comparison_examples)) for index, original_comparison_example in enumerate(original_comparison_examples): duplicate_comparison_example = duplicate_comparison_examples[index] self.assertNotEqual(original_comparison_example.id, duplicate_comparison_example.id) self.assertNotEqual(original_comparison_example.answer1_id, duplicate_comparison_example.answer1_id) self.assertNotEqual(original_comparison_example.answer2_id, duplicate_comparison_example.answer2_id) self.assertEqual(duplicate_assignment.id, duplicate_comparison_example.assignment_id) original_answer1 = original_comparison_example.answer1 duplicate_answer1 = duplicate_comparison_example.answer1 self.assertNotEqual(original_answer1.id, duplicate_answer1.id) self.assertEqual(duplicate_assignment.id, duplicate_answer1.assignment_id) self.assertEqual(original_answer1.content, duplicate_answer1.content) self.assertEqual(original_answer1.practice, duplicate_answer1.practice) self.assertEqual(original_answer1.active, duplicate_answer1.active) self.assertEqual(original_answer1.draft, duplicate_answer1.draft) original_answer2 = original_comparison_example.answer2 duplicate_answer2 = duplicate_comparison_example.answer2 self.assertNotEqual(original_answer2.id, duplicate_answer2.id) self.assertEqual(duplicate_assignment.id, duplicate_answer2.assignment_id) self.assertEqual(original_answer2.content, duplicate_answer2.content) self.assertEqual(original_answer2.practice, original_answer2.practice) self.assertEqual(original_answer2.active, original_answer2.active) self.assertEqual(original_answer2.draft, original_answer2.draft)
class ComparisonAPITests(ComPAIRAPITestCase): def setUp(self): super(ComparisonAPITests, self).setUp() self.data = ComparisonTestData() self.course = self.data.get_course() self.assignment = self.data.get_assignments()[0] self.base_url = self._build_url(self.course.uuid, self.assignment.uuid) self.lti_data = LTITestData() secondary_criterion = self.data.create_criterion( self.data.authorized_instructor) AssignmentCriterionFactory(criterion=secondary_criterion, assignment=self.assignment) db.session.commit() def _build_url(self, course_uuid, assignment_uuid, tail=""): url = '/api/courses/' + course_uuid + '/assignments/' + assignment_uuid + '/comparisons' + tail return url def _build_comparison_submit(self, winner, draft=False): submit = {'comparison_criteria': [], 'draft': draft} for criterion in self.assignment.criteria: submit['comparison_criteria'].append({ 'criterion_id': criterion.uuid, 'winner': winner, 'content': None }) return submit def test_get_answer_pair_access_control(self): # test login required rv = self.client.get(self.base_url) self.assert401(rv) # test deny access to unenroled users with self.login(self.data.get_unauthorized_student().username): rv = self.client.get(self.base_url) self.assert403(rv) with self.login(self.data.get_unauthorized_instructor().username): rv = self.client.get(self.base_url) self.assert403(rv) # enroled user from this point on with self.login(self.data.get_authorized_student().username): # test non-existent course rv = self.client.get( self._build_url("9993929", self.assignment.uuid)) self.assert404(rv) # test non-existent assignment rv = self.client.get(self._build_url(self.course.uuid, "23902390")) self.assert404(rv) # no comparisons has been entered yet, assignment is not in comparing period rv = self.client.get( self._build_url( self.course.uuid, self.data.get_assignment_in_answer_period().uuid)) self.assert403(rv) def test_submit_comparison_access_control(self): # test login required rv = self.client.post(self.base_url, data=json.dumps({}), content_type='application/json') self.assert401(rv) # establish expected data by first getting an answer pair with self.login(self.data.get_authorized_student().username): rv = self.client.get(self.base_url) self.assert200(rv) # expected_comparisons = rv.json comparison_submit = self._build_comparison_submit( WinningAnswer.answer1.value) # test deny access to unenroled users with self.login(self.data.get_unauthorized_student().username): rv = self.client.post(self.base_url, data=json.dumps(comparison_submit), content_type='application/json') self.assert403(rv) with self.login(self.data.get_unauthorized_instructor().username): rv = self.client.post(self.base_url, data=json.dumps(comparison_submit), content_type='application/json') self.assert403(rv) # authorized user from this point with self.login(self.data.get_authorized_student().username): # test non-existent course rv = self.client.post(self._build_url("9999999", self.assignment.uuid), data=json.dumps(comparison_submit), content_type='application/json') self.assert404(rv) # test non-existent assignment rv = self.client.post(self._build_url(self.course.uuid, "9999999"), data=json.dumps(comparison_submit), content_type='application/json') self.assert404(rv) # test reject missing criteria faulty_comparisons = copy.deepcopy(comparison_submit) faulty_comparisons['comparison_criteria'] = [] rv = self.client.post(self.base_url, data=json.dumps(faulty_comparisons), content_type='application/json') self.assert400(rv) # test reject missing course criteria id faulty_comparisons = copy.deepcopy(comparison_submit) del faulty_comparisons['comparison_criteria'][0]['criterion_id'] rv = self.client.post(self.base_url, data=json.dumps(faulty_comparisons), content_type='application/json') self.assert400(rv) # test invalid criterion id faulty_comparisons = copy.deepcopy(comparison_submit) faulty_comparisons['comparison_criteria'][0][ 'criterion_id'] = 3930230 rv = self.client.post(self.base_url, data=json.dumps(faulty_comparisons), content_type='application/json') self.assert400(rv) # test invalid winner faulty_comparisons = copy.deepcopy(comparison_submit) faulty_comparisons['comparison_criteria'][0]['winner'] = "2382301" rv = self.client.post(self.base_url, data=json.dumps(faulty_comparisons), content_type='application/json') self.assert400(rv) # test past grace period self.assignment.compare_start = datetime.datetime.utcnow( ) - datetime.timedelta(days=7) self.assignment.compare_end = datetime.datetime.utcnow( ) - datetime.timedelta(minutes=2) db.session.add(self.assignment) db.session.commit() ok_comparisons = copy.deepcopy(comparison_submit) rv = self.client.post(self.base_url, data=json.dumps(ok_comparisons), content_type='application/json') self.assert403(rv) self.assertEqual("Comparison Not Saved", rv.json['title']) self.assertEqual( "Sorry, the comparison deadline has passed. No comparisons can be done after the deadline.", rv.json['message']) # test within grace period self.assignment.compare_start = datetime.datetime.utcnow( ) - datetime.timedelta(days=7) self.assignment.compare_end = datetime.datetime.utcnow( ) - datetime.timedelta(seconds=15) db.session.add(self.assignment) db.session.commit() ok_comparisons = copy.deepcopy(comparison_submit) rv = self.client.post(self.base_url, data=json.dumps(ok_comparisons), content_type='application/json') self.assert200(rv) self.assignment.educators_can_compare = False db.session.commit() # instructors can access with self.login(self.data.get_authorized_instructor().username): rv = self.client.get(self.base_url) self.assert403(rv) self.assignment.educators_can_compare = True db.session.commit() rv = self.client.get(self.base_url) self.assert200(rv) # expected_comparisons = rv.json comparison_submit = self._build_comparison_submit( WinningAnswer.answer1.value) ok_comparisons = copy.deepcopy(comparison_submit) rv = self.client.post(self.base_url, data=json.dumps(ok_comparisons), content_type='application/json') self.assert200(rv) self.assignment.educators_can_compare = False db.session.commit() # ta can access with self.login(self.data.get_authorized_ta().username): rv = self.client.get(self.base_url) self.assert403(rv) self.assignment.educators_can_compare = True db.session.commit() rv = self.client.get(self.base_url) self.assert200(rv) # expected_comparisons = rv.json comparison_submit = self._build_comparison_submit( WinningAnswer.answer1.value) ok_comparisons = copy.deepcopy(comparison_submit) rv = self.client.post(self.base_url, data=json.dumps(ok_comparisons), content_type='application/json') self.assert200(rv) @mock.patch('compair.tasks.lti_outcomes.update_lti_course_grades.run') @mock.patch('compair.tasks.lti_outcomes.update_lti_assignment_grades.run') def test_get_and_submit_comparison(self, mocked_update_assignment_grades_run, mocked_update_course_grades_run): lti_consumer = self.lti_data.lti_consumer (lti_user_resource_link1, lti_user_resource_link2 ) = self.lti_data.setup_student_user_resource_links( self.data.get_authorized_student(), self.course, self.assignment) users = [ self.data.get_authorized_student(), self.data.get_authorized_instructor(), self.data.get_authorized_ta() ] for user in users: max_comparisons = 0 other_student_answers = 0 valid_answer_uuids = set() for answer in self.data.get_student_answers(): if answer.assignment.id == self.assignment.id and answer.user_id != user.id: other_student_answers += 1 valid_answer_uuids.add(answer.uuid) max_comparisons = int(other_student_answers * (other_student_answers - 1) / 2) if user.id == self.data.get_authorized_student().id: for comparison_example in self.data.comparisons_examples: if comparison_example.assignment_id == self.assignment.id: max_comparisons += 1 valid_answer_uuids.add(comparison_example.answer1_uuid) valid_answer_uuids.add(comparison_example.answer2_uuid) with self.login(user.username): if user.id in [ self.data.get_authorized_instructor().id, self.data.get_authorized_ta().id ]: self.assignment.educators_can_compare = False db.session.commit() # cannot compare answers unless educators_can_compare is set for assignment rv = self.client.get(self.base_url) self.assert403(rv) self.assignment.educators_can_compare = True db.session.commit() current = 0 while current < max_comparisons: current += 1 if user.id == self.data.get_authorized_student().id: course_grade = CourseGrade.get_user_course_grade( self.course, user).grade assignment_grade = AssignmentGrade.get_user_assignment_grade( self.assignment, user).grade # establish expected data by first getting an answer pair rv = self.client.get(self.base_url) self.assert200(rv) actual_answer1_uuid = rv.json['comparison']['answer1_id'] actual_answer2_uuid = rv.json['comparison']['answer2_id'] self.assertIn(actual_answer1_uuid, valid_answer_uuids) self.assertIn(actual_answer2_uuid, valid_answer_uuids) self.assertNotEqual(actual_answer1_uuid, actual_answer2_uuid) self.assertTrue(rv.json['new_pair']) self.assertEqual(rv.json['current'], current) # fetch again rv = self.client.get(self.base_url) self.assert200(rv) expected_comparison = rv.json['comparison'] self.assertEqual(actual_answer1_uuid, rv.json['comparison']['answer1_id']) self.assertEqual(actual_answer2_uuid, rv.json['comparison']['answer2_id']) self.assertFalse(rv.json['new_pair']) self.assertEqual(rv.json['current'], current) # test draft post comparison_submit = self._build_comparison_submit( WinningAnswer.answer1.value, True) rv = self.client.post(self.base_url, data=json.dumps(comparison_submit), content_type='application/json') self.assert200(rv) actual_comparison = rv.json['comparison'] self._validate_comparison_submit(comparison_submit, actual_comparison, expected_comparison) # test draft post (no winner) comparison_submit = self._build_comparison_submit(None) rv = self.client.post(self.base_url, data=json.dumps(comparison_submit), content_type='application/json') self.assert200(rv) actual_comparison = rv.json['comparison'] self._validate_comparison_submit(comparison_submit, actual_comparison, expected_comparison) # test normal post comparison_submit = self._build_comparison_submit( WinningAnswer.answer1.value) rv = self.client.post(self.base_url, data=json.dumps(comparison_submit), content_type='application/json') self.assert200(rv) actual_comparison = rv.json['comparison'] self._validate_comparison_submit(comparison_submit, actual_comparison, expected_comparison) # grades should increase for every comparison if user.id == self.data.get_authorized_student().id: new_course_grade = CourseGrade.get_user_course_grade( self.course, user) new_assignment_grade = AssignmentGrade.get_user_assignment_grade( self.assignment, user) self.assertGreater(new_course_grade.grade, course_grade) self.assertGreater(new_assignment_grade.grade, assignment_grade) mocked_update_course_grades_run.assert_called_once_with( lti_consumer.id, [(lti_user_resource_link1.lis_result_sourcedid, new_course_grade.id)]) mocked_update_course_grades_run.reset_mock() mocked_update_assignment_grades_run.assert_called_once_with( lti_consumer.id, [(lti_user_resource_link2.lis_result_sourcedid, new_assignment_grade.id)]) mocked_update_assignment_grades_run.reset_mock() else: new_course_grade = CourseGrade.get_user_course_grade( self.course, user) new_assignment_grade = AssignmentGrade.get_user_assignment_grade( self.assignment, user) self.assertIsNone(new_course_grade) self.assertIsNone(new_assignment_grade) mocked_update_assignment_grades_run.assert_not_called() mocked_update_course_grades_run.assert_not_called() # Resubmit of same comparison should fail rv = self.client.post(self.base_url, data=json.dumps(comparison_submit), content_type='application/json') self.assert400(rv) # all answers has been compared by the user, errors out when trying to get another pair rv = self.client.get(self.base_url) self.assert400(rv) def _validate_comparison_submit(self, comparison_submit, actual_comparison, expected_comparison): self.assertEqual( len(actual_comparison['comparison_criteria']), len(comparison_submit['comparison_criteria']), "The number of comparisons saved does not match the number sent") self.assertEqual( expected_comparison['answer1_id'], actual_comparison['answer1_id'], "Expected and actual comparison answer1 id did not match") self.assertEqual( expected_comparison['answer2_id'], actual_comparison['answer2_id'], "Expected and actual comparison answer2 id did not match") for actual_comparison_criterion in actual_comparison[ 'comparison_criteria']: found_comparison = False for expected_comparison_criterion in comparison_submit[ 'comparison_criteria']: if expected_comparison_criterion[ 'criterion_id'] != actual_comparison_criterion[ 'criterion_id']: continue self.assertEqual(expected_comparison_criterion['winner'], actual_comparison_criterion['winner'], "Expected and actual winner did not match.") found_comparison = True self.assertTrue( found_comparison, "Actual comparison received contains a comparison that was not sent." ) def _submit_all_possible_comparisons_for_user(self, user_id): example_winner_ids = [] example_loser_ids = [] for comparison_example in self.data.comparisons_examples: if comparison_example.assignment_id == self.assignment.id: comparison = Comparison.create_new_comparison( self.assignment.id, user_id, False) self.assertEqual(comparison.answer1_id, comparison_example.answer1_id) self.assertEqual(comparison.answer2_id, comparison_example.answer2_id) min_id = min([comparison.answer1_id, comparison.answer2_id]) max_id = max([comparison.answer1_id, comparison.answer2_id]) example_winner_ids.append(min_id) example_loser_ids.append(max_id) comparison.completed = True comparison.winner = WinningAnswer.answer1 if comparison.answer1_id < comparison.answer2_id else WinningAnswer.answer2 for comparison_criterion in comparison.comparison_criteria: comparison_criterion.winner = comparison.winner db.session.add(comparison) db.session.commit() # self.login(username) # calculate number of comparisons to do before user has compared all the pairs it can num_eligible_answers = 0 # need to minus one to exclude the logged in user's own answer for answer in self.data.get_student_answers(): if answer.assignment_id == self.assignment.id and answer.user_id != user_id: num_eligible_answers += 1 # n(n-1)/2 possible pairs before all answers have been compared num_possible_comparisons = int(num_eligible_answers * (num_eligible_answers - 1) / 2) winner_ids = [] loser_ids = [] for i in range(num_possible_comparisons): comparison = Comparison.create_new_comparison( self.assignment.id, user_id, False) min_id = min([comparison.answer1_id, comparison.answer2_id]) max_id = max([comparison.answer1_id, comparison.answer2_id]) winner_ids.append(min_id) loser_ids.append(max_id) comparison.completed = True comparison.winner = WinningAnswer.answer1 if comparison.answer1_id < comparison.answer2_id else WinningAnswer.answer2 for comparison_criterion in comparison.comparison_criteria: comparison_criterion.winner = comparison.winner db.session.add(comparison) db.session.commit() Comparison.calculate_scores(self.assignment.id) return { 'comparisons': { 'winners': winner_ids, 'losers': loser_ids }, 'comparison_examples': { 'winners': example_winner_ids, 'losers': example_loser_ids } } @mock.patch('random.shuffle') def test_score_calculation(self, mock_shuffle): """ This is just a rough check on whether score calculations are correct. Answers that has more wins should have the highest scores. """ # Make sure all answers are compared first comparisons_auth = self._submit_all_possible_comparisons_for_user( self.data.get_authorized_student().id) comparisons_secondary = self._submit_all_possible_comparisons_for_user( self.data.get_secondary_authorized_student().id) loser_ids = comparisons_auth['comparisons'][ 'losers'] + comparisons_secondary['comparisons']['losers'] winner_ids = comparisons_auth['comparisons'][ 'winners'] + comparisons_secondary['comparisons']['winners'] # Count the number of wins each answer has had num_wins_by_id = {} for loser_id in loser_ids: num_wins_by_id[loser_id] = num_wins_by_id.setdefault(loser_id, 0) for winner_id in winner_ids: num_wins = num_wins_by_id.setdefault(winner_id, 0) num_wins_by_id[winner_id] = num_wins + 1 # Get the actual score calculated for each answer answers = self.data.get_student_answers() answer_scores = {} for answer in answers: if answer.assignment.id == self.assignment.id: answer_scores[answer.id] = answer.score.score # Check that ranking by score and by wins match, this only works for low number of # comparisons sorted_expect_ranking = sorted(num_wins_by_id.items(), key=operator.itemgetter(0)) sorted_expect_ranking = sorted(sorted_expect_ranking, key=operator.itemgetter(1)) expected_ranking_by_wins = [ answer_id for (answer_id, wins) in sorted_expect_ranking ] sorted_actual_ranking = sorted(answer_scores.items(), key=operator.itemgetter(1)) actual_ranking_by_scores = [ answer_id for (answer_id, score) in sorted_actual_ranking ] self.assertSequenceEqual(actual_ranking_by_scores, expected_ranking_by_wins) def test_comparison_count_matched_pairing(self): # Make sure all answers are compared first answer_ids = self._submit_all_possible_comparisons_for_user( self.data.get_authorized_student().id) answer_ids2 = self._submit_all_possible_comparisons_for_user( self.data.get_secondary_authorized_student().id) compared_ids = \ answer_ids['comparisons']['winners'] + answer_ids2['comparisons']['winners'] + \ answer_ids['comparisons']['losers'] + answer_ids2['comparisons']['losers'] + \ answer_ids['comparison_examples']['winners'] + answer_ids2['comparison_examples']['winners'] + \ answer_ids['comparison_examples']['losers'] + answer_ids2['comparison_examples']['losers'] # Just a simple test for now, make sure that answers with the smaller number of # comparisons are matched up with each other # Count number of comparisons done for each answer num_comp_by_id = {} for answer_id in compared_ids: num_comp = num_comp_by_id.setdefault(answer_id, 0) num_comp_by_id[answer_id] = num_comp + 1 comp_groups = {} for answerId in num_comp_by_id: count = num_comp_by_id[answerId] comp_groups.setdefault(count, []) comp_groups[count].append(answerId) counts = sorted(comp_groups) # get the answerIds with the lowest count of comparisons possible_answer_ids = comp_groups[counts[0]] if len(possible_answer_ids) < 2: # if the lowest count group does not have enough to create a pair - add the next group possible_answer_ids += comp_groups[counts[1]] # Check that the 2 answers with 1 win gets returned with self.login( self.data.get_authorized_student_with_no_answers().username): rv = self.client.get(self.base_url) self.assert200(rv) answer1 = Answer.query.filter_by( uuid=rv.json['comparison']['answer1_id']).first() answer2 = Answer.query.filter_by( uuid=rv.json['comparison']['answer2_id']).first() self.assertIsNotNone(answer1) self.assertIsNotNone(answer2) self.assertIn(answer1.id, possible_answer_ids) self.assertIn(answer2.id, possible_answer_ids) def test_comparison_winners(self): # disable current criteria for assignment_criterion in self.assignment.assignment_criteria: assignment_criterion.active = False # test 1 criterion: answer1, answer2 (draw not possible) criterion = self.data.create_criterion(self.data.authorized_instructor) AssignmentCriterionFactory(criterion=criterion, assignment=self.assignment, weight=100) student = self.data.create_user(SystemRole.student) self.data.enrol_student(student, self.course) db.session.commit() # test winner = answer1 with self.login(student.username): rv = self.client.get(self.base_url) self.assert200(rv) comparison_submit = self._build_comparison_submit( WinningAnswer.answer1.value) rv = self.client.post(self.base_url, data=json.dumps(comparison_submit), content_type='application/json') self.assert200(rv) actual_comparison = rv.json['comparison'] self.assertEqual(actual_comparison['winner'], WinningAnswer.answer1.value) self.assertEqual(len(actual_comparison['comparison_criteria']), 1) self.assertEqual( actual_comparison['comparison_criteria'][0]['winner'], WinningAnswer.answer1.value) # test winner = answer2 rv = self.client.get(self.base_url) self.assert200(rv) comparison_submit = self._build_comparison_submit( WinningAnswer.answer2.value) rv = self.client.post(self.base_url, data=json.dumps(comparison_submit), content_type='application/json') self.assert200(rv) actual_comparison = rv.json['comparison'] self.assertEqual(actual_comparison['winner'], WinningAnswer.answer2.value) self.assertEqual(len(actual_comparison['comparison_criteria']), 1) self.assertEqual( actual_comparison['comparison_criteria'][0]['winner'], WinningAnswer.answer2.value) # test 2 criterion: answer1, answer2, draw for assignment_criterion in self.assignment.assignment_criteria: assignment_criterion.active = False criterion1 = self.data.create_criterion( self.data.authorized_instructor) criterion2 = self.data.create_criterion( self.data.authorized_instructor) AssignmentCriterionFactory(criterion=criterion1, assignment=self.assignment, weight=100) AssignmentCriterionFactory(criterion=criterion2, assignment=self.assignment, weight=100) student = self.data.create_user(SystemRole.student) self.data.enrol_student(student, self.course) db.session.commit() # test winner = answer1 with self.login(student.username): rv = self.client.get(self.base_url) self.assert200(rv) comparison_submit = self._build_comparison_submit( WinningAnswer.answer1.value) rv = self.client.post(self.base_url, data=json.dumps(comparison_submit), content_type='application/json') self.assert200(rv) actual_comparison = rv.json['comparison'] self.assertEqual(actual_comparison['winner'], WinningAnswer.answer1.value) self.assertEqual(len(actual_comparison['comparison_criteria']), 2) self.assertEqual( actual_comparison['comparison_criteria'][0]['winner'], WinningAnswer.answer1.value) self.assertEqual( actual_comparison['comparison_criteria'][1]['winner'], WinningAnswer.answer1.value) # test winner = answer2 rv = self.client.get(self.base_url) self.assert200(rv) comparison_submit = self._build_comparison_submit( WinningAnswer.answer2.value) rv = self.client.post(self.base_url, data=json.dumps(comparison_submit), content_type='application/json') self.assert200(rv) actual_comparison = rv.json['comparison'] self.assertEqual(actual_comparison['winner'], WinningAnswer.answer2.value) self.assertEqual(len(actual_comparison['comparison_criteria']), 2) self.assertEqual( actual_comparison['comparison_criteria'][0]['winner'], WinningAnswer.answer2.value) self.assertEqual( actual_comparison['comparison_criteria'][1]['winner'], WinningAnswer.answer2.value) # test winner = draw rv = self.client.get(self.base_url) self.assert200(rv) comparison_submit = self._build_comparison_submit( WinningAnswer.answer1.value) comparison_submit['comparison_criteria'][1][ 'winner'] = WinningAnswer.answer2.value rv = self.client.post(self.base_url, data=json.dumps(comparison_submit), content_type='application/json') self.assert200(rv) actual_comparison = rv.json['comparison'] self.assertEqual(actual_comparison['winner'], WinningAnswer.draw.value) self.assertEqual(len(actual_comparison['comparison_criteria']), 2) self.assertEqual( actual_comparison['comparison_criteria'][0]['winner'], WinningAnswer.answer1.value) self.assertEqual( actual_comparison['comparison_criteria'][1]['winner'], WinningAnswer.answer2.value) # test 3 criterion: answer1, answer2, draw (with with different weights) for assignment_criterion in self.assignment.assignment_criteria: assignment_criterion.active = False criterion1 = self.data.create_criterion( self.data.authorized_instructor) criterion2 = self.data.create_criterion( self.data.authorized_instructor) criterion3 = self.data.create_criterion( self.data.authorized_instructor) AssignmentCriterionFactory(criterion=criterion1, assignment=self.assignment, weight=200) AssignmentCriterionFactory(criterion=criterion2, assignment=self.assignment, weight=100) AssignmentCriterionFactory(criterion=criterion3, assignment=self.assignment, weight=100) student = self.data.create_user(SystemRole.student) self.data.enrol_student(student, self.course) db.session.commit() # test winner = answer1 with self.login(student.username): rv = self.client.get(self.base_url) self.assert200(rv) comparison_submit = self._build_comparison_submit( WinningAnswer.answer1.value) comparison_submit['comparison_criteria'][1][ 'winner'] = WinningAnswer.answer2.value rv = self.client.post(self.base_url, data=json.dumps(comparison_submit), content_type='application/json') self.assert200(rv) actual_comparison = rv.json['comparison'] self.assertEqual(actual_comparison['winner'], WinningAnswer.answer1.value) self.assertEqual(len(actual_comparison['comparison_criteria']), 3) self.assertEqual( actual_comparison['comparison_criteria'][0]['winner'], WinningAnswer.answer1.value) self.assertEqual( actual_comparison['comparison_criteria'][1]['winner'], WinningAnswer.answer2.value) self.assertEqual( actual_comparison['comparison_criteria'][2]['winner'], WinningAnswer.answer1.value) # test winner = answer2 rv = self.client.get(self.base_url) self.assert200(rv) comparison_submit = self._build_comparison_submit( WinningAnswer.answer2.value) comparison_submit['comparison_criteria'][1][ 'winner'] = WinningAnswer.answer1.value rv = self.client.post(self.base_url, data=json.dumps(comparison_submit), content_type='application/json') self.assert200(rv) actual_comparison = rv.json['comparison'] self.assertEqual(actual_comparison['winner'], WinningAnswer.answer2.value) self.assertEqual(len(actual_comparison['comparison_criteria']), 3) self.assertEqual( actual_comparison['comparison_criteria'][0]['winner'], WinningAnswer.answer2.value) self.assertEqual( actual_comparison['comparison_criteria'][1]['winner'], WinningAnswer.answer1.value) self.assertEqual( actual_comparison['comparison_criteria'][2]['winner'], WinningAnswer.answer2.value) # test winner = draw rv = self.client.get(self.base_url) self.assert200(rv) comparison_submit = self._build_comparison_submit( WinningAnswer.answer1.value) comparison_submit['comparison_criteria'][1][ 'winner'] = WinningAnswer.answer2.value comparison_submit['comparison_criteria'][2][ 'winner'] = WinningAnswer.answer2.value rv = self.client.post(self.base_url, data=json.dumps(comparison_submit), content_type='application/json') self.assert200(rv) actual_comparison = rv.json['comparison'] self.assertEqual(actual_comparison['winner'], WinningAnswer.draw.value) self.assertEqual(len(actual_comparison['comparison_criteria']), 3) self.assertEqual( actual_comparison['comparison_criteria'][0]['winner'], WinningAnswer.answer1.value) self.assertEqual( actual_comparison['comparison_criteria'][1]['winner'], WinningAnswer.answer2.value) self.assertEqual( actual_comparison['comparison_criteria'][2]['winner'], WinningAnswer.answer2.value)