def test2_3papers_4reviewers_1custom_load(self, test_util):
'''
Reviewer 0 has a custom_load of 1 and cannot be the second reviewer of paper 1
:param test_util:
:return:
'''
score_matrix = np.array([[10.67801, 3, 0], [0, 10.67801, 0],
[0, 0, 10.67801], [0, 0, 10.67801]])
num_papers = 3
num_reviewers = 4
num_reviews_per_paper = 2
reviewer_max_papers = 2
params = Params({
Params.NUM_PAPERS: num_papers,
Params.NUM_REVIEWERS: num_reviewers,
Params.NUM_REVIEWS_NEEDED_PER_PAPER: num_reviews_per_paper,
Params.REVIEWER_MAX_PAPERS: reviewer_max_papers,
Params.CUSTOM_LOAD_CONFIG: {
Params.CUSTOM_LOAD_MAP: {
0: 1,
1: 2
}
},
Params.SCORES_CONFIG: {
Params.SCORES_SPEC: {
'affinity': {
'weight': 1,
'default': 0
}
},
Params.SCORE_TYPE: Params.MATRIX_SCORE,
Params.SCORE_MATRIX: score_matrix
}
})
test_util.set_test_params(params)
test_util.build_conference()
match = Match(test_util.client,
test_util.get_conference().get_config_note())
match.compute_match()
conference = test_util.get_conference()
assert conference.get_config_note_status() == Configuration.STATUS_COMPLETE, \
"Failure: Config status is {} expected {}".format(conference.get_config_note_status(), Configuration.STATUS_COMPLETE)
assignment_edges = conference.get_assignment_edges()
assert len(assignment_edges) == num_reviews_per_paper * len(conference.get_paper_notes()), "Number of assignment edges {} is incorrect. Should be". \
format(len(assignment_edges), num_reviews_per_paper * len(conference.get_paper_notes()))
aggregate_score_edges = conference.get_aggregate_score_edges()
assert len(aggregate_score_edges) == num_reviewers * num_papers
paper_ids = conference.get_paper_note_ids()
reviewers = conference.reviewers
c = AssignmentChecker(conference)
assert c.is_paper_assigned_to_reviewer(paper_ids[0], reviewers[0])
assert c.is_paper_assigned_to_reviewer(paper_ids[0], reviewers[1])
assert c.is_paper_assigned_to_reviewer(paper_ids[1], reviewers[1])
assert c.is_paper_assigned_to_reviewer(paper_ids[1], reviewers[2])
assert c.is_paper_assigned_to_reviewer(paper_ids[2], reviewers[2])
assert c.is_paper_assigned_to_reviewer(paper_ids[2], reviewers[3])
def test3_3papers_4reviewers_1conflict (self, test_util):
'''
Paper 0 conflicts with Reviewer 0 so this cannot be in the solution.
:param test_util:
:return:
'''
score_matrix = np.array([
[10.67801, 0, 0],
[0, 10.67801, 0],
[0, 0, 10.67801],
[0, 0, 10.67801]
])
num_papers = 3
num_reviewers = 4
num_reviews_per_paper = 2
reviewer_max_papers = 2
params = Params({Params.NUM_PAPERS: num_papers,
Params.NUM_REVIEWERS: num_reviewers,
Params.NUM_REVIEWS_NEEDED_PER_PAPER: num_reviews_per_paper,
Params.REVIEWER_MAX_PAPERS: reviewer_max_papers,
Params.CONFLICTS_CONFIG: {0: [0]},
Params.SCORES_CONFIG: {Params.SCORES_SPEC: {'affinity': {'weight': 1, 'default': 0}},
Params.SCORE_TYPE: Params.MATRIX_SCORE,
Params.SCORE_MATRIX: score_matrix
}
})
test_util.set_test_params(params)
test_util.build_conference()
match = Match(test_util.client, test_util.get_conference().get_config_note())
match.compute_match()
conference = test_util.get_conference()
assert conference.get_config_note_status() == Configuration.STATUS_COMPLETE, \
"Failure: Config status is {} expected {}".format(conference.get_config_note_status(), Configuration.STATUS_COMPLETE)
assignment_edges = conference.get_assignment_edges()
assert len(assignment_edges) == num_reviews_per_paper * len(conference.get_paper_notes()), "Number of assignment edges {} is incorrect. Should be". \
format(len(assignment_edges), num_reviews_per_paper * len(conference.get_paper_notes()))
aggregate_score_edges = conference.get_aggregate_score_edges()
assert len(aggregate_score_edges) == num_reviewers * num_papers
reviewers = conference.reviewers
papers = conference.get_paper_notes()
paper_reviewer_data = match.paper_reviewer_data
self.check_aggregate_score_edges(test_util.client,reviewers,papers,conference,paper_reviewer_data)
# Validate that the assignment edges are correct
# reviewer-1 -> paper-1
assert conference.get_assignment_edge(papers[1].id, reviewers[1]) != None
# 2 -> 2
assert conference.get_assignment_edge(papers[2].id, reviewers[2]) != None
# 3 -> 2
assert conference.get_assignment_edge(papers[2].id, reviewers[3]) != None
# !reviewer-0 -> paper-0
try:
conference.get_assignment_edge(papers[0].id, reviewers[0])
except NotFoundError:
assert True
def test_aggregate_score_with_missing_score (self, test_util):
'''
Is the same as test4 above but it deletes all the edges created from the 0's in the score_matrix which sets up the test case.
This will test the matcher's ability to use a default when no score is present in the configuration.
:param test_util:
:return:
'''
score_matrix = np.array([
[10.67801, 0, 0],
[0, 10.67801, 0],
[0, 0, 10.67801],
[0, 0, 10.67801]
])
num_papers = 3
num_reviewers = 4
num_reviews_per_paper = 2
reviewer_max_papers = 2
params = Params({Params.NUM_PAPERS: num_papers,
Params.NUM_REVIEWERS: num_reviewers,
Params.NUM_REVIEWS_NEEDED_PER_PAPER: num_reviews_per_paper,
Params.REVIEWER_MAX_PAPERS: reviewer_max_papers,
Params.SCORES_CONFIG: {Params.SCORES_SPEC: {'affinity': {'weight': 1, 'default': 0}},
Params.SCORE_TYPE: Params.MATRIX_SCORE,
Params.OMIT_ZERO_SCORE_EDGES: True,
Params.SCORE_MATRIX: score_matrix
}
})
test_util.set_test_params(params)
test_util.build_conference()
match = Match(test_util.client, test_util.get_conference().get_config_note())
match.compute_match()
conference = test_util.get_conference()
assert conference.get_config_note_status() == Configuration.STATUS_COMPLETE, \
"Failure: Config status is {} expected {}".format(conference.get_config_note_status(), Configuration.STATUS_COMPLETE)
assignment_edges = conference.get_assignment_edges()
assert len(assignment_edges) == num_reviews_per_paper * len(conference.get_paper_notes()), "Number of assignment edges {} is incorrect. Should be". \
format(len(assignment_edges), num_reviews_per_paper * len(conference.get_paper_notes()))
aggregate_score_edges = conference.get_aggregate_score_edges()
assert len(aggregate_score_edges) == num_reviewers * num_papers
reviewers = conference.reviewers
papers = conference.get_paper_notes()
paper_reviewer_data = match.paper_reviewer_data
self.check_aggregate_score_edges(test_util.client,reviewers,papers,conference,paper_reviewer_data)
# Validate that the assignment edges are correct
# reviewer-1 -> paper-1
assert conference.get_assignment_edge(papers[1].id, reviewers[1]) != None
# 2 -> 2
assert conference.get_assignment_edge(papers[2].id, reviewers[2]) != None
# 3 -> 2
assert conference.get_assignment_edge(papers[2].id, reviewers[3]) != None
# !reviewer-0 -> paper-0
assert conference.get_assignment_edge(papers[0].id, reviewers[0]) != None
def test3_5papers_5reviewers_custom_loads(self, test_util):
'''
Sanity check to make sure the correct solution is found
:param test_util:
:return:
'''
score_matrix = np.array([[10.67801, 0, 0, 0,
0], [0, 0, 10.67801, 0, 0],
[0, 0, 0, 0,
10.67801], [0, 0, 0, 10.67801, 0],
[0, 10.67801, 0, 0, 0]])
num_papers = 5
num_reviewers = 5
num_reviews_per_paper = 1
reviewer_max_papers = 2
params = Params({
Params.NUM_PAPERS: num_papers,
Params.NUM_REVIEWERS: num_reviewers,
Params.NUM_REVIEWS_NEEDED_PER_PAPER: num_reviews_per_paper,
Params.REVIEWER_MAX_PAPERS: reviewer_max_papers,
Params.SCORES_CONFIG: {
Params.SCORES_SPEC: {
'affinity': {
'weight': 1,
'default': 0
}
},
Params.SCORE_TYPE: Params.MATRIX_SCORE,
Params.SCORE_MATRIX: score_matrix
}
})
test_util.set_test_params(params)
test_util.build_conference()
match = Match(test_util.client,
test_util.get_conference().get_config_note())
match.compute_match()
conference = test_util.get_conference()
assert conference.get_config_note_status() == Configuration.STATUS_COMPLETE, \
"Failure: Config status is {} expected {}".format(conference.get_config_note_status(), Configuration.STATUS_COMPLETE)
assignment_edges = conference.get_assignment_edges()
assert len(assignment_edges) == num_reviews_per_paper * len(conference.get_paper_notes()), "Number of assignment edges {} is incorrect. Should be". \
format(len(assignment_edges), num_reviews_per_paper * len(conference.get_paper_notes()))
paper_ids = conference.get_paper_note_ids()
reviewers = conference.reviewers
c = AssignmentChecker(conference)
assert c.is_paper_assigned_to_reviewer(paper_ids[0], reviewers[0])
assert c.is_paper_assigned_to_reviewer(paper_ids[1], reviewers[4])
assert c.is_paper_assigned_to_reviewer(paper_ids[2], reviewers[1])
assert c.is_paper_assigned_to_reviewer(paper_ids[3], reviewers[3])
assert c.is_paper_assigned_to_reviewer(paper_ids[4], reviewers[2])
def test_aggregate_with_missing_scores (self, test_util):
'''
Three scores are used but no user will provide them.
This will test the matcher's ability to use a default when no score is present in the configuration.
:param test_util:
:return:
'''
num_papers = 3
num_reviewers = 4
num_reviews_per_paper = 2
reviewer_max_papers = 2
params = Params({Params.NUM_PAPERS: num_papers,
Params.NUM_REVIEWERS: num_reviewers,
Params.NUM_REVIEWS_NEEDED_PER_PAPER: num_reviews_per_paper,
Params.REVIEWER_MAX_PAPERS: reviewer_max_papers,
Params.SCORES_CONFIG: {
Params.SCORES_SPEC: {
'affinity': {'weight': 1, 'default': 0},
'recommendation': {'weight': 1, 'default': 0},
'bid': {'weight': 1, 'default': 0.3, 'translate_fn': self.bid_translate_fn}
},
Params.SCORE_TYPE: Params.FIXED_SCORE,
Params.OMIT_ZERO_SCORE_EDGES: True,
Params.FIXED_SCORE_VALUE: {'affinity': 0, 'recommendation': 0, 'bid': 'low'}
}
})
test_util.set_test_params(params)
test_util.build_conference()
match = Match(test_util.client, test_util.get_conference().get_config_note())
match.compute_match()
conference = test_util.get_conference()
assert conference.get_config_note_status() == Configuration.STATUS_COMPLETE, \
"Failure: Config status is {} expected {}".format(conference.get_config_note_status(), Configuration.STATUS_COMPLETE)
assignment_edges = conference.get_assignment_edges()
assert len(assignment_edges) == num_reviews_per_paper * len(conference.get_paper_notes()), "Number of assignment edges {} is incorrect. Should be". \
format(len(assignment_edges), num_reviews_per_paper * len(conference.get_paper_notes()))
aggregate_score_edges = conference.get_aggregate_score_edges()
assert len(aggregate_score_edges) == num_reviewers * num_papers
reviewers = conference.reviewers
papers = conference.get_paper_notes()
paper_reviewer_data = match.paper_reviewer_data
self.check_aggregate_score_edges(test_util.client,reviewers,papers,conference,paper_reviewer_data)
def test1_10papers_7reviewers (self, test_util):
'''
Tests 10 papers each requiring 2 reviews. 7 users each capable of giving 3 reviews.
Validates that the output aggregate score edges are correct with respect to the paper-reviewer scores input
'''
num_papers = 10
num_reviewers = 7
num_reviews_per_paper = 2
params = Params({Params.NUM_PAPERS: num_papers,
Params.NUM_REVIEWERS: num_reviewers,
Params.NUM_REVIEWS_NEEDED_PER_PAPER: num_reviews_per_paper,
Params.REVIEWER_MAX_PAPERS: 3,
Params.SCORES_CONFIG: { Params.SCORE_TYPE: Params.INCREMENTAL_SCORE,
Params.SCORE_INCREMENT: 0.01,
Params.SCORES_SPEC: {'affinity': {'weight': 1, 'default': 0}}}
})
test_util.set_test_params(params)
test_util.build_conference()
match = Match(test_util.client, test_util.get_conference().get_config_note())
match.compute_match()
conference = test_util.get_conference()
assert conference.get_config_note_status() == Configuration.STATUS_COMPLETE, \
"Failure: Config status is {} expected {}".format(conference.get_config_note_status(), Configuration.STATUS_COMPLETE)
assignment_edges = conference.get_assignment_edges()
assert len(assignment_edges) == num_reviews_per_paper * len(conference.get_paper_notes()), "Number of assignment edges {} is incorrect. Should be". \
format(len(assignment_edges), num_reviews_per_paper * len(conference.get_paper_notes()))
aggregate_score_edges = conference.get_aggregate_score_edges()
assert len(aggregate_score_edges) == num_reviewers * num_papers
# Verify for every paper P and reviewer R that there is an aggregate score edge with a weight set to
# the matcher's cost_func applied to the score edges for P and R * the weights.
# Its not safe to just compare the edges to the cost_matrix because that's what they were built from. Going back to the
# score edges will be closer to the source of the data that forms the cost.
reviewers = conference.reviewers
papers = conference.get_paper_notes()
# enc = Encoder(config=test_util.get_conference().get_config_note().content)
paper_reviewer_data = match.paper_reviewer_data
self.check_aggregate_score_edges(test_util.client, reviewers, papers, conference, paper_reviewer_data)
def test4_5papers_5reviewers_custom_loads(self, test_util):
'''
Reviewer 0 can only do 1 paper, but reviewer 1 can do 3, so result should make use of these
:param test_util:
:return:
'''
score_matrix = np.array([[10.67801, 0, 0, 0,
0], [0, 0, 10.67801, 0, 0],
[0, 0, 0, 0,
10.67801], [0, 0, 0, 10.67801, 0],
[0, 10.67801, 0, 0, 0]])
num_papers = 5
num_reviewers = 5
num_reviews_per_paper = 2
reviewer_max_papers = 2
params = Params({
Params.NUM_PAPERS: num_papers,
Params.NUM_REVIEWERS: num_reviewers,
Params.NUM_REVIEWS_NEEDED_PER_PAPER: num_reviews_per_paper,
Params.REVIEWER_MAX_PAPERS: reviewer_max_papers,
Params.CUSTOM_LOAD_CONFIG: {
Params.CUSTOM_LOAD_MAP: {
0: 1,
1: 3
}
},
Params.SCORES_CONFIG: {
Params.SCORES_SPEC: {
'affinity': {
'weight': 1,
'default': 0
}
},
Params.SCORE_TYPE: Params.MATRIX_SCORE,
Params.SCORE_MATRIX: score_matrix
}
})
test_util.set_test_params(params)
test_util.build_conference()
match = Match(test_util.client,
test_util.get_conference().get_config_note())
match.compute_match()
conference = test_util.get_conference()
assert conference.get_config_note_status() == Configuration.STATUS_COMPLETE, \
"Failure: Config status is {} expected {}".format(conference.get_config_note_status(), Configuration.STATUS_COMPLETE)
assignment_edges = conference.get_assignment_edges()
assert len(assignment_edges) == num_reviews_per_paper * len(conference.get_paper_notes()), "Number of assignment edges {} is incorrect. Should be". \
format(len(assignment_edges), num_reviews_per_paper * len(conference.get_paper_notes()))
aggregate_score_edges = conference.get_aggregate_score_edges()
assert len(aggregate_score_edges) == num_reviewers * num_papers
reviewers = conference.reviewers
assert len(conference.get_assignment_edges_by_reviewer(
reviewers[0])) == 1
assert len(conference.get_assignment_edges_by_reviewer(
reviewers[1])) == 3
assert len(conference.get_assignment_edges_by_reviewer(
reviewers[2])) == 2
assert len(conference.get_assignment_edges_by_reviewer(
reviewers[3])) == 2
assert len(conference.get_assignment_edges_by_reviewer(
reviewers[4])) == 2
def match():
app.logger.debug("POST /match")
res = {}
matcher = None
try:
token = request.headers.get('Authorization')
if not token:
raise NoTokenException('No Authorization token in headers')
# N.B. If the token is invalid, it succeeds using a guest
client = get_client(token=token)
params = request.json
config_note_id = params['configNoteId']
# TODO move status setting out of Match and into this ConfigStatus class
# config_status = ConfigStatus(client,config_note_id)
app.logger.debug("Request to assign reviewers for configId: " +
config_note_id)
# If the client was constructed with a bad token, the failure happens here
config_note = client.get_note(config_note_id)
# If the configuration is already running a matching task, do not allow another until the
# running task is complete
if config_note.content[
Configuration.STATUS] == Configuration.STATUS_RUNNING:
raise AlreadyRunningException(
'There is already a running matching task for config ' +
config_note_id)
elif config_note.content[
Configuration.STATUS] == Configuration.STATUS_COMPLETE:
raise AlreadyCompleteException(
'Cannot run matcher on a completed configuration ' +
config_note_id)
matcher = Match(client, config_note, app.logger)
# runs the match task in a separate thread
matcher.run()
except openreview.OpenReviewException as e:
app.logger.error('OpenReview-py error:', exc_info=True)
# this exception type has args which is a tuple containing a list containing a dict where the type key indicates what went wrong
err_type = e.args[0][0]['type']
status = 500
if err_type.lower() == 'not found':
status = 404
elif err_type.lower() == 'forbidden':
status = 403
else:
err_type = str(e)
res['error'] = err_type
if matcher:
matcher.set_status(Configuration.STATUS_ERROR, "Error: " + str(e))
return jsonify(res), status
except (NoTokenException, BadTokenException, AlreadyRunningException) as e:
app.logger.error('OpenReview-matcher error:', exc_info=True)
res['error'] = str(e)
if matcher:
matcher.set_status(Configuration.STATUS_ERROR, "Error: " + str(e))
return jsonify(res), 400
except Exception as e:
app.logger.error('OpenReview-matcher error:', exc_info=True)
res['error'] = str(e)
# TODO Shouldn't need to rely on existence of a matcher in order to set config status- Use ConfigStatus object instead.
if matcher:
matcher.set_status(Configuration.STATUS_ERROR, "Error: " + str(e))
return jsonify(res), 500
else:
app.logger.debug("POST returns " + str(res))
return jsonify(res)