def test2_3papers_4reviewers_1custom_load(self, test_util): ''' Reviewer 0 has a custom_load of 1 and cannot be the second reviewer of paper 1 :param test_util: :return: ''' score_matrix = np.array([[10.67801, 3, 0], [0, 10.67801, 0], [0, 0, 10.67801], [0, 0, 10.67801]]) num_papers = 3 num_reviewers = 4 num_reviews_per_paper = 2 reviewer_max_papers = 2 params = Params({ Params.NUM_PAPERS: num_papers, Params.NUM_REVIEWERS: num_reviewers, Params.NUM_REVIEWS_NEEDED_PER_PAPER: num_reviews_per_paper, Params.REVIEWER_MAX_PAPERS: reviewer_max_papers, Params.CUSTOM_LOAD_CONFIG: { Params.CUSTOM_LOAD_MAP: { 0: 1, 1: 2 } }, Params.SCORES_CONFIG: { Params.SCORES_SPEC: { 'affinity': { 'weight': 1, 'default': 0 } }, Params.SCORE_TYPE: Params.MATRIX_SCORE, Params.SCORE_MATRIX: score_matrix } }) test_util.set_test_params(params) test_util.build_conference() match = Match(test_util.client, test_util.get_conference().get_config_note()) match.compute_match() conference = test_util.get_conference() assert conference.get_config_note_status() == Configuration.STATUS_COMPLETE, \ "Failure: Config status is {} expected {}".format(conference.get_config_note_status(), Configuration.STATUS_COMPLETE) assignment_edges = conference.get_assignment_edges() assert len(assignment_edges) == num_reviews_per_paper * len(conference.get_paper_notes()), "Number of assignment edges {} is incorrect. Should be". \ format(len(assignment_edges), num_reviews_per_paper * len(conference.get_paper_notes())) aggregate_score_edges = conference.get_aggregate_score_edges() assert len(aggregate_score_edges) == num_reviewers * num_papers paper_ids = conference.get_paper_note_ids() reviewers = conference.reviewers c = AssignmentChecker(conference) assert c.is_paper_assigned_to_reviewer(paper_ids[0], reviewers[0]) assert c.is_paper_assigned_to_reviewer(paper_ids[0], reviewers[1]) assert c.is_paper_assigned_to_reviewer(paper_ids[1], reviewers[1]) assert c.is_paper_assigned_to_reviewer(paper_ids[1], reviewers[2]) assert c.is_paper_assigned_to_reviewer(paper_ids[2], reviewers[2]) assert c.is_paper_assigned_to_reviewer(paper_ids[2], reviewers[3])
def test3_3papers_4reviewers_1conflict (self, test_util): ''' Paper 0 conflicts with Reviewer 0 so this cannot be in the solution. :param test_util: :return: ''' score_matrix = np.array([ [10.67801, 0, 0], [0, 10.67801, 0], [0, 0, 10.67801], [0, 0, 10.67801] ]) num_papers = 3 num_reviewers = 4 num_reviews_per_paper = 2 reviewer_max_papers = 2 params = Params({Params.NUM_PAPERS: num_papers, Params.NUM_REVIEWERS: num_reviewers, Params.NUM_REVIEWS_NEEDED_PER_PAPER: num_reviews_per_paper, Params.REVIEWER_MAX_PAPERS: reviewer_max_papers, Params.CONFLICTS_CONFIG: {0: [0]}, Params.SCORES_CONFIG: {Params.SCORES_SPEC: {'affinity': {'weight': 1, 'default': 0}}, Params.SCORE_TYPE: Params.MATRIX_SCORE, Params.SCORE_MATRIX: score_matrix } }) test_util.set_test_params(params) test_util.build_conference() match = Match(test_util.client, test_util.get_conference().get_config_note()) match.compute_match() conference = test_util.get_conference() assert conference.get_config_note_status() == Configuration.STATUS_COMPLETE, \ "Failure: Config status is {} expected {}".format(conference.get_config_note_status(), Configuration.STATUS_COMPLETE) assignment_edges = conference.get_assignment_edges() assert len(assignment_edges) == num_reviews_per_paper * len(conference.get_paper_notes()), "Number of assignment edges {} is incorrect. Should be". \ format(len(assignment_edges), num_reviews_per_paper * len(conference.get_paper_notes())) aggregate_score_edges = conference.get_aggregate_score_edges() assert len(aggregate_score_edges) == num_reviewers * num_papers reviewers = conference.reviewers papers = conference.get_paper_notes() paper_reviewer_data = match.paper_reviewer_data self.check_aggregate_score_edges(test_util.client,reviewers,papers,conference,paper_reviewer_data) # Validate that the assignment edges are correct # reviewer-1 -> paper-1 assert conference.get_assignment_edge(papers[1].id, reviewers[1]) != None # 2 -> 2 assert conference.get_assignment_edge(papers[2].id, reviewers[2]) != None # 3 -> 2 assert conference.get_assignment_edge(papers[2].id, reviewers[3]) != None # !reviewer-0 -> paper-0 try: conference.get_assignment_edge(papers[0].id, reviewers[0]) except NotFoundError: assert True
def test_aggregate_score_with_missing_score (self, test_util): ''' Is the same as test4 above but it deletes all the edges created from the 0's in the score_matrix which sets up the test case. This will test the matcher's ability to use a default when no score is present in the configuration. :param test_util: :return: ''' score_matrix = np.array([ [10.67801, 0, 0], [0, 10.67801, 0], [0, 0, 10.67801], [0, 0, 10.67801] ]) num_papers = 3 num_reviewers = 4 num_reviews_per_paper = 2 reviewer_max_papers = 2 params = Params({Params.NUM_PAPERS: num_papers, Params.NUM_REVIEWERS: num_reviewers, Params.NUM_REVIEWS_NEEDED_PER_PAPER: num_reviews_per_paper, Params.REVIEWER_MAX_PAPERS: reviewer_max_papers, Params.SCORES_CONFIG: {Params.SCORES_SPEC: {'affinity': {'weight': 1, 'default': 0}}, Params.SCORE_TYPE: Params.MATRIX_SCORE, Params.OMIT_ZERO_SCORE_EDGES: True, Params.SCORE_MATRIX: score_matrix } }) test_util.set_test_params(params) test_util.build_conference() match = Match(test_util.client, test_util.get_conference().get_config_note()) match.compute_match() conference = test_util.get_conference() assert conference.get_config_note_status() == Configuration.STATUS_COMPLETE, \ "Failure: Config status is {} expected {}".format(conference.get_config_note_status(), Configuration.STATUS_COMPLETE) assignment_edges = conference.get_assignment_edges() assert len(assignment_edges) == num_reviews_per_paper * len(conference.get_paper_notes()), "Number of assignment edges {} is incorrect. Should be". \ format(len(assignment_edges), num_reviews_per_paper * len(conference.get_paper_notes())) aggregate_score_edges = conference.get_aggregate_score_edges() assert len(aggregate_score_edges) == num_reviewers * num_papers reviewers = conference.reviewers papers = conference.get_paper_notes() paper_reviewer_data = match.paper_reviewer_data self.check_aggregate_score_edges(test_util.client,reviewers,papers,conference,paper_reviewer_data) # Validate that the assignment edges are correct # reviewer-1 -> paper-1 assert conference.get_assignment_edge(papers[1].id, reviewers[1]) != None # 2 -> 2 assert conference.get_assignment_edge(papers[2].id, reviewers[2]) != None # 3 -> 2 assert conference.get_assignment_edge(papers[2].id, reviewers[3]) != None # !reviewer-0 -> paper-0 assert conference.get_assignment_edge(papers[0].id, reviewers[0]) != None
def test3_5papers_5reviewers_custom_loads(self, test_util): ''' Sanity check to make sure the correct solution is found :param test_util: :return: ''' score_matrix = np.array([[10.67801, 0, 0, 0, 0], [0, 0, 10.67801, 0, 0], [0, 0, 0, 0, 10.67801], [0, 0, 0, 10.67801, 0], [0, 10.67801, 0, 0, 0]]) num_papers = 5 num_reviewers = 5 num_reviews_per_paper = 1 reviewer_max_papers = 2 params = Params({ Params.NUM_PAPERS: num_papers, Params.NUM_REVIEWERS: num_reviewers, Params.NUM_REVIEWS_NEEDED_PER_PAPER: num_reviews_per_paper, Params.REVIEWER_MAX_PAPERS: reviewer_max_papers, Params.SCORES_CONFIG: { Params.SCORES_SPEC: { 'affinity': { 'weight': 1, 'default': 0 } }, Params.SCORE_TYPE: Params.MATRIX_SCORE, Params.SCORE_MATRIX: score_matrix } }) test_util.set_test_params(params) test_util.build_conference() match = Match(test_util.client, test_util.get_conference().get_config_note()) match.compute_match() conference = test_util.get_conference() assert conference.get_config_note_status() == Configuration.STATUS_COMPLETE, \ "Failure: Config status is {} expected {}".format(conference.get_config_note_status(), Configuration.STATUS_COMPLETE) assignment_edges = conference.get_assignment_edges() assert len(assignment_edges) == num_reviews_per_paper * len(conference.get_paper_notes()), "Number of assignment edges {} is incorrect. Should be". \ format(len(assignment_edges), num_reviews_per_paper * len(conference.get_paper_notes())) paper_ids = conference.get_paper_note_ids() reviewers = conference.reviewers c = AssignmentChecker(conference) assert c.is_paper_assigned_to_reviewer(paper_ids[0], reviewers[0]) assert c.is_paper_assigned_to_reviewer(paper_ids[1], reviewers[4]) assert c.is_paper_assigned_to_reviewer(paper_ids[2], reviewers[1]) assert c.is_paper_assigned_to_reviewer(paper_ids[3], reviewers[3]) assert c.is_paper_assigned_to_reviewer(paper_ids[4], reviewers[2])
def test_aggregate_with_missing_scores (self, test_util): ''' Three scores are used but no user will provide them. This will test the matcher's ability to use a default when no score is present in the configuration. :param test_util: :return: ''' num_papers = 3 num_reviewers = 4 num_reviews_per_paper = 2 reviewer_max_papers = 2 params = Params({Params.NUM_PAPERS: num_papers, Params.NUM_REVIEWERS: num_reviewers, Params.NUM_REVIEWS_NEEDED_PER_PAPER: num_reviews_per_paper, Params.REVIEWER_MAX_PAPERS: reviewer_max_papers, Params.SCORES_CONFIG: { Params.SCORES_SPEC: { 'affinity': {'weight': 1, 'default': 0}, 'recommendation': {'weight': 1, 'default': 0}, 'bid': {'weight': 1, 'default': 0.3, 'translate_fn': self.bid_translate_fn} }, Params.SCORE_TYPE: Params.FIXED_SCORE, Params.OMIT_ZERO_SCORE_EDGES: True, Params.FIXED_SCORE_VALUE: {'affinity': 0, 'recommendation': 0, 'bid': 'low'} } }) test_util.set_test_params(params) test_util.build_conference() match = Match(test_util.client, test_util.get_conference().get_config_note()) match.compute_match() conference = test_util.get_conference() assert conference.get_config_note_status() == Configuration.STATUS_COMPLETE, \ "Failure: Config status is {} expected {}".format(conference.get_config_note_status(), Configuration.STATUS_COMPLETE) assignment_edges = conference.get_assignment_edges() assert len(assignment_edges) == num_reviews_per_paper * len(conference.get_paper_notes()), "Number of assignment edges {} is incorrect. Should be". \ format(len(assignment_edges), num_reviews_per_paper * len(conference.get_paper_notes())) aggregate_score_edges = conference.get_aggregate_score_edges() assert len(aggregate_score_edges) == num_reviewers * num_papers reviewers = conference.reviewers papers = conference.get_paper_notes() paper_reviewer_data = match.paper_reviewer_data self.check_aggregate_score_edges(test_util.client,reviewers,papers,conference,paper_reviewer_data)
def test1_10papers_7reviewers (self, test_util): ''' Tests 10 papers each requiring 2 reviews. 7 users each capable of giving 3 reviews. Validates that the output aggregate score edges are correct with respect to the paper-reviewer scores input ''' num_papers = 10 num_reviewers = 7 num_reviews_per_paper = 2 params = Params({Params.NUM_PAPERS: num_papers, Params.NUM_REVIEWERS: num_reviewers, Params.NUM_REVIEWS_NEEDED_PER_PAPER: num_reviews_per_paper, Params.REVIEWER_MAX_PAPERS: 3, Params.SCORES_CONFIG: { Params.SCORE_TYPE: Params.INCREMENTAL_SCORE, Params.SCORE_INCREMENT: 0.01, Params.SCORES_SPEC: {'affinity': {'weight': 1, 'default': 0}}} }) test_util.set_test_params(params) test_util.build_conference() match = Match(test_util.client, test_util.get_conference().get_config_note()) match.compute_match() conference = test_util.get_conference() assert conference.get_config_note_status() == Configuration.STATUS_COMPLETE, \ "Failure: Config status is {} expected {}".format(conference.get_config_note_status(), Configuration.STATUS_COMPLETE) assignment_edges = conference.get_assignment_edges() assert len(assignment_edges) == num_reviews_per_paper * len(conference.get_paper_notes()), "Number of assignment edges {} is incorrect. Should be". \ format(len(assignment_edges), num_reviews_per_paper * len(conference.get_paper_notes())) aggregate_score_edges = conference.get_aggregate_score_edges() assert len(aggregate_score_edges) == num_reviewers * num_papers # Verify for every paper P and reviewer R that there is an aggregate score edge with a weight set to # the matcher's cost_func applied to the score edges for P and R * the weights. # Its not safe to just compare the edges to the cost_matrix because that's what they were built from. Going back to the # score edges will be closer to the source of the data that forms the cost. reviewers = conference.reviewers papers = conference.get_paper_notes() # enc = Encoder(config=test_util.get_conference().get_config_note().content) paper_reviewer_data = match.paper_reviewer_data self.check_aggregate_score_edges(test_util.client, reviewers, papers, conference, paper_reviewer_data)
def test4_5papers_5reviewers_custom_loads(self, test_util): ''' Reviewer 0 can only do 1 paper, but reviewer 1 can do 3, so result should make use of these :param test_util: :return: ''' score_matrix = np.array([[10.67801, 0, 0, 0, 0], [0, 0, 10.67801, 0, 0], [0, 0, 0, 0, 10.67801], [0, 0, 0, 10.67801, 0], [0, 10.67801, 0, 0, 0]]) num_papers = 5 num_reviewers = 5 num_reviews_per_paper = 2 reviewer_max_papers = 2 params = Params({ Params.NUM_PAPERS: num_papers, Params.NUM_REVIEWERS: num_reviewers, Params.NUM_REVIEWS_NEEDED_PER_PAPER: num_reviews_per_paper, Params.REVIEWER_MAX_PAPERS: reviewer_max_papers, Params.CUSTOM_LOAD_CONFIG: { Params.CUSTOM_LOAD_MAP: { 0: 1, 1: 3 } }, Params.SCORES_CONFIG: { Params.SCORES_SPEC: { 'affinity': { 'weight': 1, 'default': 0 } }, Params.SCORE_TYPE: Params.MATRIX_SCORE, Params.SCORE_MATRIX: score_matrix } }) test_util.set_test_params(params) test_util.build_conference() match = Match(test_util.client, test_util.get_conference().get_config_note()) match.compute_match() conference = test_util.get_conference() assert conference.get_config_note_status() == Configuration.STATUS_COMPLETE, \ "Failure: Config status is {} expected {}".format(conference.get_config_note_status(), Configuration.STATUS_COMPLETE) assignment_edges = conference.get_assignment_edges() assert len(assignment_edges) == num_reviews_per_paper * len(conference.get_paper_notes()), "Number of assignment edges {} is incorrect. Should be". \ format(len(assignment_edges), num_reviews_per_paper * len(conference.get_paper_notes())) aggregate_score_edges = conference.get_aggregate_score_edges() assert len(aggregate_score_edges) == num_reviewers * num_papers reviewers = conference.reviewers assert len(conference.get_assignment_edges_by_reviewer( reviewers[0])) == 1 assert len(conference.get_assignment_edges_by_reviewer( reviewers[1])) == 3 assert len(conference.get_assignment_edges_by_reviewer( reviewers[2])) == 2 assert len(conference.get_assignment_edges_by_reviewer( reviewers[3])) == 2 assert len(conference.get_assignment_edges_by_reviewer( reviewers[4])) == 2