def _build_subject_area_scores(self, submissions):
'''
Create subject area scores between all users in the match group and all given submissions
'''
invitation = self._create_edge_invitation(
self._get_edge_invitation_id('Subject_Areas_Score'),
extendable_readers=True)
edges = []
user_subject_areas = list(
openreview.tools.iterget_notes(
self.client, invitation=self.conference.get_registration_id()))
for note in submissions:
note_subject_areas = note.content['subject_areas']
paper_note_id = note.id
for subject_area_note in user_subject_areas:
profile_id = subject_area_note.signatures[0]
if profile_id in self.match_group.members:
subject_areas = subject_area_note.content['subject_areas']
score = _jaccard_similarity(note_subject_areas,
subject_areas)
edges.append(
openreview.Edge(invitation=invitation.id,
head=paper_note_id,
tail=profile_id,
weight=float(score),
readers=[self.conference.id],
writers=[self.conference.id],
signatures=[self.conference.id]))
openreview.tools.post_bulk_edges(client=self.conference.client,
edges=edges)
return invitation
def _build_scores(self, score_invitation_id, score_file):
'''
Given a csv file with affinity scores, create score edges
'''
invitation = self._create_edge_invitation(score_invitation_id,
extendable_readers=True)
edges = []
with open(score_file) as file_handle:
for row in csv.reader(file_handle):
paper_note_id = row[0]
profile_id = row[1]
score = row[2]
edges.append(
openreview.Edge(invitation=invitation.id,
head=paper_note_id,
tail=profile_id,
weight=float(score),
readers=[self.conference.id, profile_id],
writers=[self.conference.id],
signatures=[self.conference.id]))
openreview.tools.post_bulk_edges(client=self.conference.client,
edges=edges)
return invitation
def create_reviewer_edge(client,
conference,
name,
note,
reviewer,
label=None,
weight=None):
conference_id = conference.id
return client.post_edge(
openreview.Edge(
invitation=f'{conference.id}/Reviewers/-/{name}',
readers=[
conference_id,
conference.get_senior_area_chairs_id(number=note.number),
conference.get_area_chairs_id(number=note.number), reviewer
],
nonreaders=[conference.get_authors_id(number=note.number)],
writers=[
conference_id,
conference.get_senior_area_chairs_id(number=note.number),
conference.get_area_chairs_id(number=note.number)
],
signatures=[conference_id],
head=note.id,
tail=reviewer,
label=label,
weight=weight))
def add_config_custom_loads(self):
'''
The supply deduction > 0 indicates that some reviewers cannot do the default minimum number of reviews and
we want to cycle through the reviewers lowering their loads until the supply deduction is met.
:return:
'''
loads = {
reviewer: self.params.reviewer_max_papers
for reviewer in self.reviewers
}
if self.params.custom_load_supply_deduction:
self.reduce_reviewer_loads(
loads, self.params.custom_load_supply_deduction)
elif self.params.custom_load_map != {}:
self.set_reviewer_loads(loads, self.params.custom_load_map)
# build custom_load edge for those reviewers that are different from the default max
edges = []
for rev, load in loads.items():
if load != self.params.reviewer_max_papers:
edge = openreview.Edge(
invitation=self.conf_ids.CUSTOM_LOAD_INV_ID,
label=self.config_title,
head=self.conf_ids.CONF_ID,
tail=rev,
weight=load,
readers=[self.conf_ids.CONF_ID],
writers=[self.conf_ids.CONF_ID],
signatures=[self.conf_ids.CONF_ID])
edges.append(edge)
openreview.tools.post_bulk_edges(self.client, edges)
def _build_conflicts(self, submissions, user_profiles):
'''
Create conflict edges between the given Notes and Profiles
'''
invitation = self._create_edge_invitation(
self.conference.get_conflict_score_id(self.match_group.id),
extendable_readers=True)
authorids_profiles = {}
for submission in submissions:
edges = []
for profile in user_profiles:
authorids = submission.content['authorids']
if submission.details and submission.details.get('original'):
authorids = submission.details['original']['content'][
'authorids']
if submission.number not in authorids_profiles:
profiles = _get_profiles(self.client, authorids)
authorids_profiles[submission.number] = profiles
author_profiles = authorids_profiles[submission.number]
conflicts = openreview.tools.get_conflicts(
author_profiles, profile)
if conflicts:
edges.append(
openreview.Edge(
invitation=invitation.id,
head=submission.id,
tail=profile.id,
weight=-1,
label=_conflict_label(conflicts),
readers=[self.conference.id, profile.id],
writers=[self.conference.id],
signatures=[self.conference.id]))
openreview.tools.post_bulk_edges(client=self.client, edges=edges)
return invitation
def _build_edge(self, invitation, forum_id, reviewer, score, label, number):
'''
Helper function for constructing an openreview.Edge object.
Readers, nonreaders, writers, and signatures are automatically filled based on the invitaiton.
'''
return openreview.Edge(
head = forum_id,
tail = reviewer,
weight = score,
label = label,
invitation = invitation.id,
readers = self._get_values(invitation, number, 'readers', forum_id, reviewer),
nonreaders = self._get_values(invitation, number, 'nonreaders'),
writers = self._get_values(invitation, number, 'writers'),
signatures = [self.venue_id])
def create_constraint_edges(self, constraints, val):
constraint_edge_inv = self.conf_ids.CONSTRAINTS_INV_ID
edges = []
for paper_ix, reviewers_list in constraints.items():
p = self.paper_notes[paper_ix]
for reviewer_ix in reviewers_list:
r = self.reviewers[reviewer_ix]
e = openreview.Edge(head=p.id,
tail=r,
label=self.config_title,
weight=val,
invitation=constraint_edge_inv,
readers=[self.conf_ids.CONF_ID],
writers=[self.conf_ids.CONF_ID],
signatures=[r])
edges.append(e)
openreview.tools.post_bulk_edges(self.client, edges)
def _build_tpms_scores(self, tpms_score_file, submissions, user_profiles):
'''
Create tpms score edges given a csv file with scores, papers, and profiles.
'''
# pylint: disable=too-many-locals
invitation = self._create_edge_invitation(
self._get_edge_invitation_id('TPMS_Score'),
extendable_readers=True)
submissions_per_number = {note.number: note for note in submissions}
profiles_by_email = {}
for profile in user_profiles:
for email in profile.content['emails']:
profiles_by_email[email] = profile
edges = []
with open(tpms_score_file) as file_handle:
for row in csv.reader(file_handle):
number = int(row[0])
if number in submissions_per_number:
paper_note_id = submissions_per_number[number].id
profile = profiles_by_email.get(row[1])
if profile:
profile_id = profile.id
else:
profile_id = row[1]
score = row[2]
edges.append(
openreview.Edge(
invitation=invitation.id,
head=paper_note_id,
tail=profile_id,
weight=float(score),
readers=[self.conference.id, profile.id],
writers=[self.conference.id],
signatures=[self.conference.id]))
openreview.tools.post_bulk_edges(client=self.client, edges=edges)
return invitation
def test_save_bulk(self, client):
builder = openreview.conference.ConferenceBuilder(client)
assert builder, 'builder is None'
builder.set_conference_id('NIPS.cc/2020/Workshop/MLITS')
builder.set_submission_stage(public=True)
conference = builder.get_result()
# Edge invitation
inv1 = openreview.Invitation(
id=conference.id + '/-/affinity',
reply={
'content': {
'head': {
'type': 'Note'
},
'tail': {
'type': 'Profile',
},
'label': {
'value-radio':
['Very High', 'High', 'Neutral', 'Low', 'Very Low']
},
'weight': {
'value-regex': r'[0-9]+\.[0-9]'
}
}
})
inv1 = client.post_invitation(inv1)
# Sample note
note = openreview.Note(
invitation=conference.get_submission_id(),
readers=['everyone'],
writers=[conference.id],
signatures=['~Super_User1'],
content={
'title': 'Paper title',
'abstract': 'This is an abstract',
'authorids': ['*****@*****.**'],
'authors': ['Test User'],
'pdf': '/pdf/22234qweoiuweroi22234qweoiuweroi12345678.pdf'
})
note = client.post_note(note)
# Edges
edges = []
for p in range(1000):
edge = openreview.Edge(head=note.id,
tail='~Super_User1',
label='High',
weight=0.5,
invitation=inv1.id,
readers=['everyone'],
writers=[conference.id],
signatures=['~Super_User1'])
edges.append(edge)
openreview.tools.post_bulk_edges(client, edges)
them = list(openreview.tools.iterget_edges(client, invitation=inv1.id))
assert len(edges) == len(them)
for i in ids:
if i in confirmations:
confirmation = confirmations[i]
if confirmation:
review_capacity = int(
confirmation.content.get('emergency_review_count', '0'))
if review_capacity:
total_review_capacity += review_capacity
edge = openreview.Edge(
head='thecvf.com/ECCV/2020/Conference/Emergency_Reviewers',
tail=profile.id,
invitation=
'thecvf.com/ECCV/2020/Conference/Emergency_Reviewers/-/Custom_Max_Papers',
readers=[
'thecvf.com/ECCV/2020/Conference',
'thecvf.com/ECCV/2020/Conference/Area_Chairs',
profile.id
],
writers=['thecvf.com/ECCV/2020/Conference'],
signatures=['thecvf.com/ECCV/2020/Conference'],
weight=review_capacity)
emergency_load_edges.append(edge)
emergency_reviewer_group.members.append(profile.id)
x = client.post_group(emergency_reviewer_group)
print('Created updated emergency reviewers group\n')
print('Posting {0} edges'.format(len(emergency_load_edges)))
posted_edges = openreview.tools.post_bulk_edges(client,
emergency_load_edges)
def test_setup_matching_with_recommendations(self, client, test_client, helpers):
builder = openreview.conference.ConferenceBuilder(client)
assert builder, 'builder is None'
builder = openreview.conference.ConferenceBuilder(client)
builder.set_conference_id('auai.org/UAI/2019/Conference')
builder.set_conference_name('Conference on Uncertainty in Artificial Intelligence')
builder.set_conference_short_name('UAI 2019')
builder.set_homepage_header({
'title': 'UAI 2019',
'subtitle': 'Conference on Uncertainty in Artificial Intelligence',
'deadline': 'Abstract Submission Deadline: 11:59 pm Samoa Standard Time, March 4, 2019, Full Submission Deadline: 11:59 pm Samoa Standard Time, March 8, 2019',
'date': 'July 22 - July 25, 2019',
'website': 'http://auai.org/uai2019/',
'location': 'Tel Aviv, Israel',
'instructions': '''<p><strong>Important Information about Anonymity:</strong><br>
When you post a submission to UAI 2019, please provide the real names and email addresses of authors in the submission form below (but NOT in the manuscript).
The <em>original</em> record of your submission will be private, and will contain your real name(s).
The PDF in your submission should not contain the names of the authors. </p>
<p><strong>Conflict of Interest:</strong><br>
Please make sure that your current and previous affiliations listed on your OpenReview <a href=\"/profile\">profile page</a> is up-to-date to avoid conflict of interest.</p>
<p><strong>Questions or Concerns:</strong><br> Please contact the UAI 2019 Program chairs at <a href=\"mailto:[email protected]\">[email protected]</a>.
<br>Please contact the OpenReview support team at <a href=\"mailto:[email protected]\">[email protected]</a> with any OpenReview related questions or concerns.
</p>'''
})
print ('Homepage header set')
builder.set_conference_area_chairs_name('Senior_Program_Committee')
builder.set_conference_reviewers_name('Program_Committee')
builder.set_override_homepage(True)
now = datetime.datetime.utcnow()
builder.set_submission_stage(due_date = now + datetime.timedelta(minutes = 40), double_blind= True, subject_areas=[
"Algorithms: Approximate Inference",
"Algorithms: Belief Propagation",
"Algorithms: Distributed and Parallel",
"Algorithms: Exact Inference",
])
conference = builder.get_result()
assert conference, 'conference is None'
blinded_notes = list(conference.get_submissions())
## Open reviewer recommendations
now = datetime.datetime.utcnow()
conference.open_recommendations(due_date = now + datetime.timedelta(minutes = 40))
## Recommend reviewers
ac1_client = helpers.get_user('*****@*****.**')
ac1_client.post_edge(openreview.Edge(invitation = conference.get_recommendation_id(),
readers = ['auai.org/UAI/2019/Conference', '~AreaChair_One1'],
signatures = ['~AreaChair_One1'],
writers = ['~AreaChair_One1'],
head = blinded_notes[0].id,
tail = '~Reviewer_One1',
weight = 1
))
ac1_client.post_edge(openreview.Edge(invitation = conference.get_recommendation_id(),
readers = ['auai.org/UAI/2019/Conference', '~AreaChair_One1'],
signatures = ['~AreaChair_One1'],
writers = ['~AreaChair_One1'],
head = blinded_notes[1].id,
tail = '*****@*****.**',
weight = 2
))
ac1_client.post_edge(openreview.Edge(invitation = conference.get_recommendation_id(),
readers = ['auai.org/UAI/2019/Conference', '~AreaChair_One1'],
signatures = ['~AreaChair_One1'],
writers = ['~AreaChair_One1'],
head = blinded_notes[1].id,
tail = '*****@*****.**',
weight = 3
))
# Set up reviewer matching
conference.setup_matching(tpms_score_file=os.path.join(os.path.dirname(__file__), 'data/reviewer_tpms_scores.csv'))
print(conference.get_reviewers_id())
invitation = client.get_invitation(id='auai.org/UAI/2019/Conference/Program_Committee/-/Assignment_Configuration')
assert invitation
assert 'scores_specification' in invitation.reply['content']
assert 'auai.org/UAI/2019/Conference/Program_Committee/-/Bid' in invitation.reply['content']['scores_specification']['default']
assert 'auai.org/UAI/2019/Conference/Program_Committee/-/TPMS_Score' in invitation.reply['content']['scores_specification']['default']
assert 'auai.org/UAI/2019/Conference/Program_Committee/-/Subject_Areas_Score' in invitation.reply['content']['scores_specification']['default']
assert 'auai.org/UAI/2019/Conference/-/Recommendation' in invitation.reply['content']['scores_specification']['default']
assert client.get_invitation(id='auai.org/UAI/2019/Conference/Program_Committee/-/Custom_Load')
assert client.get_invitation(id='auai.org/UAI/2019/Conference/Program_Committee/-/Conflict')
# Set up ac matching
conference.setup_matching(
is_area_chair=True,
tpms_score_file=os.path.join(os.path.dirname(__file__), 'data/ac_tpms_scores.csv'))
assert client.get_invitation(id='auai.org/UAI/2019/Conference/Senior_Program_Committee/-/Custom_Load')
assert client.get_invitation(id='auai.org/UAI/2019/Conference/Senior_Program_Committee/-/Conflict')
bids = client.get_edges(invitation = conference.get_bid_id(conference.get_area_chairs_id()))
assert bids
assert 3 == len(bids)
bids = client.get_edges(invitation = conference.get_bid_id(conference.get_reviewers_id()))
assert bids
assert 3 == len(bids)
recommendations = client.get_edges(invitation = conference.get_recommendation_id())
assert recommendations
assert 3 == len(recommendations)
reviewer_custom_loads = client.get_edges(
invitation='auai.org/UAI/2019/Conference/Program_Committee/-/Custom_Load')
assert not reviewer_custom_loads
ac_custom_loads = client.get_edges(
invitation='auai.org/UAI/2019/Conference/Senior_Program_Committee/-/Custom_Load')
assert not ac_custom_loads
reviewer_conflicts = client.get_edges(
invitation='auai.org/UAI/2019/Conference/Program_Committee/-/Conflict')
assert 1 == len(reviewer_conflicts)
ac_conflicts = client.get_edges(
invitation='auai.org/UAI/2019/Conference/Senior_Program_Committee/-/Conflict')
assert 2 == len(ac_conflicts)
ac1_conflicts = client.get_edges(
invitation='auai.org/UAI/2019/Conference/Senior_Program_Committee/-/Conflict', tail='~AreaChair_One1')
assert ac1_conflicts
assert len(ac1_conflicts)
assert ac1_conflicts[0].label == 'Institutional (level 1)'
r1_conflicts = client.get_edges(
invitation='auai.org/UAI/2019/Conference/Program_Committee/-/Conflict', tail='~Reviewer_One1')
assert r1_conflicts
assert len(r1_conflicts)
assert r1_conflicts[0].label == 'Institutional (level 1)'
ac2_conflicts = client.get_edges(
invitation='auai.org/UAI/2019/Conference/Senior_Program_Committee/-/Conflict', tail='*****@*****.**')
assert ac2_conflicts
assert len(ac2_conflicts)
assert ac2_conflicts[0].label == 'Institutional (level 1)'
submissions = conference.get_submissions()
assert submissions
assert 3 == len(submissions)
reviewer_tpms_scores = client.get_edges(
invitation='auai.org/UAI/2019/Conference/Program_Committee/-/TPMS_Score')
assert 9 == len(reviewer_tpms_scores)
ac_tpms_scores = client.get_edges(
invitation='auai.org/UAI/2019/Conference/Senior_Program_Committee/-/TPMS_Score')
assert 6 == len(ac_tpms_scores)
r3_s0_tpms_scores = client.get_edges(
invitation='auai.org/UAI/2019/Conference/Program_Committee/-/TPMS_Score',
tail='*****@*****.**',
head=submissions[0].id)
assert r3_s0_tpms_scores
assert 1 == len(r3_s0_tpms_scores)
assert r3_s0_tpms_scores[0].weight == 0.21
r3_s1_tpms_scores = client.get_edges(
invitation='auai.org/UAI/2019/Conference/Program_Committee/-/TPMS_Score',
tail='*****@*****.**',
head=submissions[1].id)
assert r3_s1_tpms_scores
assert 1 == len(r3_s1_tpms_scores)
assert r3_s1_tpms_scores[0].weight == 0.31
r3_s2_tpms_scores = client.get_edges(
invitation='auai.org/UAI/2019/Conference/Program_Committee/-/TPMS_Score',
tail='*****@*****.**',
head=submissions[2].id)
assert r3_s2_tpms_scores
assert 1 == len(r3_s2_tpms_scores)
assert r3_s2_tpms_scores[0].weight == 0.51
def build_conflicts(match_group, submissions):
edges = []
invitation = _create_edge_invitation(
conference.get_invitation_id('Conflict', prefix=match_group.id),
match_group)
user_profiles = openreview.matching._get_profiles(client,
match_group.members)
user_profiles_info = [
get_profile_info(profile) for profile in user_profiles
]
for submission in submissions:
author_domains = set()
author_emails = set()
author_relations = set()
author_publications = set()
author_profiles = openreview.matching._get_profiles(
client, submission.details['original']['content']['authorids'])
for author in author_profiles:
author_info = get_profile_info(author)
author_domains.update(author_info['domains'])
author_emails.update(author_info['emails'])
author_relations.update(author_info['relations'])
author_publications.update(author_info['publications'])
for user_info in user_profiles_info:
conflicts = set()
conflicts.update(author_domains.intersection(user_info['domains']))
conflicts.update(author_relations.intersection(
user_info['emails']))
conflicts.update(author_emails.intersection(
user_info['relations']))
conflicts.update(author_emails.intersection(user_info['emails']))
conflicts.update(
author_publications.intersection(user_info['publications']))
if conflicts:
edges.append(
openreview.Edge(
invitation=invitation.id,
head=submission.id,
tail=user_info['id'],
weight=-1,
label='Conflict',
readers=_get_edge_readers(tail=user_info['id']),
writers=[conference.id],
signatures=[conference.id]))
openreview.tools.post_bulk_edges(client, edges=edges)
# Perform sanity check
edges_posted = client.get_edges_count(invitation=invitation.id)
if edges_posted < len(edges):
raise openreview.OpenReviewException(
'Failed during bulk post of Conflict edges! Scores found: {0}, Edges posted: {1}'
.format(len(edges), edges_posted))
return edges
def test_integration_no_solution_due_to_conflicts(openreview_context):
'''
Basic integration test. Makes use of the OpenReview Builder
'''
openreview_client = openreview_context['openreview_client']
test_client = openreview_context['test_client']
conference_id = 'ICLR.cca/2020/Conference'
num_reviewers = 3
num_papers = 1
reviews_per_paper = 3
max_papers = 1
min_papers = 1
alternates = 0
conference = clean_start_conference(openreview_client, conference_id,
num_reviewers, num_papers,
reviews_per_paper)
reviewers_id = conference.get_reviewers_id()
config = {
'title':
'integration-test',
'user_demand':
str(reviews_per_paper),
'max_papers':
str(max_papers),
'min_papers':
str(min_papers),
'alternates':
str(alternates),
'config_invitation':
'{}/-/Assignment_Configuration'.format(reviewers_id),
'paper_invitation':
conference.get_blind_submission_id(),
'assignment_invitation':
conference.get_paper_assignment_id(reviewers_id),
'deployed_assignment_invitation':
conference.get_paper_assignment_id(reviewers_id, deployed=True),
'invite_assignment_invitation':
conference.get_paper_assignment_id(reviewers_id, invite=True),
'aggregate_score_invitation':
'{}/-/Aggregate_Score'.format(reviewers_id),
'conflicts_invitation':
conference.get_conflict_score_id(reviewers_id),
'custom_max_papers_invitation':
'{}/-/Custom_Max_Papers'.format(reviewers_id),
'match_group':
reviewers_id,
'scores_specification': {
conference.get_affinity_score_id(reviewers_id): {
'weight': 1.0,
'default': 0.0
}
},
'status':
'Initialized',
'solver':
'MinMax'
}
config_note = openreview.Note(
**{
'invitation': '{}/-/Assignment_Configuration'.format(reviewers_id),
'readers': [conference.get_id()],
'writers': [conference.get_id()],
'signatures': [conference.get_id()],
'content': config
})
for reviewer in openreview_client.get_group(reviewers_id).members:
openreview_client.post_edge(
openreview.Edge(
invitation=conference.get_conflict_score_id(reviewers_id),
label="Personal",
weight=-1,
head=conference.get_submissions()[-1].id,
tail=reviewer,
signatures=[conference.id],
readers=[conference.id, reviewer],
writers=[conference.id],
))
config_note = openreview_client.post_note(config_note)
assert config_note
response = test_client.post('/match',
data=json.dumps(
{'configNoteId': config_note.id}),
content_type='application/json',
headers=openreview_client.headers)
assert response.status_code == 200
matcher_status = wait_for_status(openreview_client, config_note.id)
assert matcher_status.content['status'] == 'No Solution'
assert matcher_status.content[
'error_message'] == 'Solver could not find a solution. Adjust your parameters'
paper_assignment_edges = openreview_client.get_edges(
label='integration-test',
invitation=conference.get_paper_assignment_id(
conference.get_reviewers_id()))
assert len(paper_assignment_edges) == 0
def test_setup_matching(self, client, test_client, helpers):
builder = openreview.conference.ConferenceBuilder(client)
assert builder, 'builder is None'
builder = openreview.conference.ConferenceBuilder(client)
builder.set_conference_id('auai.org/UAI/2019/Conference')
builder.set_conference_name('Conference on Uncertainty in Artificial Intelligence')
builder.set_conference_short_name('UAI 2019')
builder.set_homepage_header({
'title': 'UAI 2019',
'subtitle': 'Conference on Uncertainty in Artificial Intelligence',
'deadline': 'Abstract Submission Deadline: 11:59 pm Samoa Standard Time, March 4, 2019, Full Submission Deadline: 11:59 pm Samoa Standard Time, March 8, 2019',
'date': 'July 22 - July 25, 2019',
'website': 'http://auai.org/uai2019/',
'location': 'Tel Aviv, Israel',
'instructions': '''<p><strong>Important Information about Anonymity:</strong><br>
When you post a submission to UAI 2019, please provide the real names and email addresses of authors in the submission form below (but NOT in the manuscript).
The <em>original</em> record of your submission will be private, and will contain your real name(s).
The PDF in your submission should not contain the names of the authors. </p>
<p><strong>Conflict of Interest:</strong><br>
Please make sure that your current and previous affiliations listed on your OpenReview <a href=\"/profile\">profile page</a> is up-to-date to avoid conflict of interest.</p>
<p><strong>Questions or Concerns:</strong><br> Please contact the UAI 2019 Program chairs at <a href=\"mailto:[email protected]\">[email protected]</a>.
<br>Please contact the OpenReview support team at <a href=\"mailto:[email protected]\">[email protected]</a> with any OpenReview related questions or concerns.
</p>'''
})
print ('Homepage header set')
builder.set_conference_area_chairs_name('Senior_Program_Committee')
builder.set_conference_reviewers_name('Program_Committee')
builder.set_override_homepage(True)
now = datetime.datetime.utcnow()
builder.set_submission_stage(due_date = now + datetime.timedelta(minutes = 40), double_blind= True, subject_areas=[
"Algorithms: Approximate Inference",
"Algorithms: Belief Propagation",
"Algorithms: Distributed and Parallel",
"Algorithms: Exact Inference",
])
builder.set_bid_stage(due_date = now + datetime.timedelta(minutes = 40), request_count = 50)
conference = builder.get_result()
assert conference, 'conference is None'
## Set committee
conference.set_program_chairs(['*****@*****.**', '*****@*****.**'])
conference.set_area_chairs(['*****@*****.**', '*****@*****.**'])
conference.set_reviewers(['*****@*****.**', '*****@*****.**', '*****@*****.**'])
## Paper 1
note_1 = openreview.Note(invitation = conference.get_submission_id(),
readers = ['~Test_User1', '*****@*****.**', '*****@*****.**'],
writers = [conference.id, '~Test_User1', '*****@*****.**', '*****@*****.**'],
signatures = ['~Test_User1'],
content = {
'title': 'Paper title 1',
'abstract': 'This is an abstract',
'authorids': ['*****@*****.**', '*****@*****.**'],
'authors': ['Test User', 'Author 1'],
'subject_areas': [
'Algorithms: Approximate Inference',
'Algorithms: Belief Propagation'
]
}
)
url = test_client.put_pdf(os.path.join(os.path.dirname(__file__), 'data/paper.pdf'))
note_1.content['pdf'] = url
note_1 = test_client.post_note(note_1)
## Paper 2
note_2 = openreview.Note(invitation = conference.get_submission_id(),
readers = ['~Test_User1', '*****@*****.**', '*****@*****.**'],
writers = [conference.id, '~Test_User1', '*****@*****.**', '*****@*****.**'],
signatures = ['~Test_User1'],
content = {
'title': 'Paper title 2',
'abstract': 'This is an abstract',
'authorids': ['*****@*****.**', '*****@*****.**'],
'authors': ['Test User', 'Author 2'],
'subject_areas': [
'Algorithms: Approximate Inference',
'Algorithms: Exact Inference'
]
}
)
url = test_client.put_pdf(os.path.join(os.path.dirname(__file__), 'data/paper.pdf'))
note_2.content['pdf'] = url
note_2 = test_client.post_note(note_2)
## Paper 3
note_3 = openreview.Note(invitation = conference.get_submission_id(),
readers = ['~Test_User1', '*****@*****.**', '*****@*****.**'],
writers = [conference.id, '~Test_User1', '*****@*****.**', '*****@*****.**'],
signatures = ['~Test_User1'],
content = {
'title': 'Paper title 3',
'abstract': 'This is an abstract',
'authorids': ['*****@*****.**', '*****@*****.**'],
'authors': ['Test User', 'Author 3'],
'subject_areas': [
'Algorithms: Distributed and Parallel',
'Algorithms: Exact Inference'
]
}
)
url = test_client.put_pdf(os.path.join(os.path.dirname(__file__), 'data/paper.pdf'))
note_3.content['pdf'] = url
note_3 = test_client.post_note(note_3)
## Create blind submissions
builder.set_submission_stage(due_date = now, double_blind= True, subject_areas=[
"Algorithms: Approximate Inference",
"Algorithms: Belief Propagation",
"Algorithms: Distributed and Parallel",
"Algorithms: Exact Inference",
])
conference = builder.get_result()
blinded_notes = conference.create_blind_submissions()
conference.set_authors()
ac1_client = helpers.create_user('*****@*****.**', 'AreaChair', 'One')
ac1_client.post_edge(openreview.Edge(invitation = conference.get_bid_id(conference.get_area_chairs_id()),
readers = ['auai.org/UAI/2019/Conference', '~AreaChair_One1'],
writers = ['auai.org/UAI/2019/Conference', '~AreaChair_One1'],
signatures = ['~AreaChair_One1'],
head = blinded_notes[0].id,
tail = '~AreaChair_One1',
label = 'High'
))
ac1_client.post_edge(openreview.Edge(invitation = conference.get_bid_id(conference.get_area_chairs_id()),
readers = ['auai.org/UAI/2019/Conference', '~AreaChair_One1'],
writers = ['auai.org/UAI/2019/Conference', '~AreaChair_One1'],
signatures = ['~AreaChair_One1'],
head = blinded_notes[1].id,
tail = '~AreaChair_One1',
label = 'Low'
))
ac1_client.post_edge(openreview.Edge(invitation = conference.get_bid_id(conference.get_area_chairs_id()),
readers = ['auai.org/UAI/2019/Conference', '~AreaChair_One1'],
writers = ['auai.org/UAI/2019/Conference', '~AreaChair_One1'],
signatures = ['~AreaChair_One1'],
head = blinded_notes[2].id,
tail = '~AreaChair_One1',
label = 'Very Low'
))
r1_client = helpers.create_user('*****@*****.**', 'Reviewer', 'One')
r1_client.post_edge(openreview.Edge(invitation = conference.get_bid_id(conference.get_reviewers_id()),
readers = ['auai.org/UAI/2019/Conference', '~Reviewer_One1'],
writers = ['auai.org/UAI/2019/Conference', '~Reviewer_One1'],
signatures = ['~Reviewer_One1'],
head = blinded_notes[0].id,
tail = '~Reviewer_One1',
label = 'Neutral'
))
r1_client.post_edge(openreview.Edge(invitation = conference.get_bid_id(conference.get_reviewers_id()),
readers = ['auai.org/UAI/2019/Conference', '~Reviewer_One1'],
writers = ['auai.org/UAI/2019/Conference', '~Reviewer_One1'],
signatures = ['~Reviewer_One1'],
head = blinded_notes[1].id,
tail = '~Reviewer_One1',
label = 'Very High'
))
r1_client.post_edge(openreview.Edge(invitation = conference.get_bid_id(conference.get_reviewers_id()),
readers = ['auai.org/UAI/2019/Conference', '~Reviewer_One1'],
writers = ['auai.org/UAI/2019/Conference', '~Reviewer_One1'],
signatures = ['~Reviewer_One1'],
head = blinded_notes[2].id,
tail = '~Reviewer_One1',
label = 'Low'
))
# Set up reviewer matching
conference.setup_matching()
invitation = client.get_invitation(id='auai.org/UAI/2019/Conference/Program_Committee/-/Assignment_Configuration')
assert invitation
assert 'scores_specification' in invitation.reply['content']
assert 'auai.org/UAI/2019/Conference/Program_Committee/-/Bid' in invitation.reply['content']['scores_specification']['default']
assert client.get_invitation(id='auai.org/UAI/2019/Conference/Program_Committee/-/Custom_Load')
assert client.get_invitation(id='auai.org/UAI/2019/Conference/Program_Committee/-/Conflict')
assert client.get_invitation(id='auai.org/UAI/2019/Conference/Program_Committee/-/Aggregate_Score')
assert client.get_invitation(id='auai.org/UAI/2019/Conference/Program_Committee/-/Paper_Assignment')
# Set up AC matching
conference.setup_matching(is_area_chair=True)
invitation = client.get_invitation(id='auai.org/UAI/2019/Conference/Senior_Program_Committee/-/Assignment_Configuration')
assert invitation
assert 'scores_specification' in invitation.reply['content']
assert 'auai.org/UAI/2019/Conference/Senior_Program_Committee/-/Bid' in invitation.reply['content']['scores_specification']['default']
assert client.get_invitation(id='auai.org/UAI/2019/Conference/Senior_Program_Committee/-/Custom_Load')
assert client.get_invitation(id='auai.org/UAI/2019/Conference/Senior_Program_Committee/-/Conflict')
assert client.get_invitation(id='auai.org/UAI/2019/Conference/Senior_Program_Committee/-/Aggregate_Score')
assert client.get_invitation(id='auai.org/UAI/2019/Conference/Senior_Program_Committee/-/Paper_Assignment')
bids = client.get_edges(invitation = conference.get_bid_id(conference.get_area_chairs_id()))
assert bids
assert 3 == len(bids)
bids = client.get_edges(invitation = conference.get_bid_id(conference.get_reviewers_id()))
assert bids
assert 3 == len(bids)
reviewer_custom_loads = client.get_edges(
invitation='auai.org/UAI/2019/Conference/Program_Committee/-/Custom_Load')
assert not reviewer_custom_loads
ac_custom_loads = client.get_edges(
invitation='auai.org/UAI/2019/Conference/Senior_Program_Committee/-/Custom_Load')
assert not ac_custom_loads
reviewer_conflicts = client.get_edges(
invitation='auai.org/UAI/2019/Conference/Program_Committee/-/Conflict')
assert 1 == len(reviewer_conflicts)
ac_conflicts = client.get_edges(
invitation='auai.org/UAI/2019/Conference/Senior_Program_Committee/-/Conflict')
assert 2 == len(ac_conflicts)
ac1_conflicts = client.get_edges(
invitation='auai.org/UAI/2019/Conference/Senior_Program_Committee/-/Conflict',
tail='~AreaChair_One1')
assert ac1_conflicts
assert len(ac1_conflicts)
assert ac1_conflicts[0].label == 'Institutional (level 1)'
r1_conflicts = client.get_edges(
invitation='auai.org/UAI/2019/Conference/Program_Committee/-/Conflict',
tail='~Reviewer_One1')
assert r1_conflicts
assert len(r1_conflicts)
assert r1_conflicts[0].label == 'Institutional (level 1)'
ac2_conflicts = client.get_edges(
invitation='auai.org/UAI/2019/Conference/Senior_Program_Committee/-/Conflict',
tail='*****@*****.**')
assert ac2_conflicts
assert len(ac2_conflicts)
assert ac2_conflicts[0].label == 'Institutional (level 1)'
confirmation = confirmations[i]
if not custom_load and (i in reduced_loads):
custom_load = reduced_loads[i]
emergency_review_count = int(confirmation.content.get('emergency_review_count', '0')) if confirmation else 0
if custom_load:
review_capacity = int(custom_load.content['reviewer_load']) - emergency_review_count
else:
review_capacity = 7 - emergency_review_count
if review_capacity != 7:
edge = openreview.Edge(
head='thecvf.com/ECCV/2020/Conference/Reviewers',
tail=profile.id,
invitation='thecvf.com/ECCV/2020/Conference/Reviewers/-/Custom_Load',
readers=[
'thecvf.com/ECCV/2020/Conference',
'thecvf.com/ECCV/2020/Conference/Area_Chairs',
profile.id],
writers=['thecvf.com/ECCV/2020/Conference'],
signatures=['thecvf.com/ECCV/2020/Conference'],
weight=review_capacity
)
custom_load_edges.append(edge)
print ('Posting {0} edges'.format(len(custom_load_edges)))
posted_edges = openreview.tools.post_bulk_edges(client, custom_load_edges)
print ('Posted {0} edges'.format(len(posted_edges)))
print ('Users with no profiles:', users_without_profiles)
custom_load = reduced_loads[i]
if not max3_accepted and (i in max3_accepted_users):
max3_accepted = max3_accepted_users[i]
emergency_review_count = int(
confirmation.content.get(
'emergency_review_count')) if confirmation else 0
if custom_load:
review_capacity = int(custom_load) - emergency_review_count
edge = openreview.Edge(
head='ICLR.cc/2021/Conference/Reviewers',
tail=profile.id,
invitation=
'ICLR.cc/2021/Conference/Reviewers/-/Max_Load_Papers',
readers=[
'ICLR.cc/2021/Conference',
'ICLR.cc/2021/Conference/Area_Chairs', profile.id
],
writers=['ICLR.cc/2021/Conference'],
signatures=['ICLR.cc/2021/Conference'],
weight=int(custom_load))
max_load_edges.append(edge)
elif max3_accepted:
review_capacity = 3 - emergency_review_count
edge = openreview.Edge(
head='ICLR.cc/2021/Conference/Reviewers',
tail=profile.id,
invitation=
'ICLR.cc/2021/Conference/Reviewers/-/Max_Load_Papers',
readers=[
'ICLR.cc/2021/Conference',
def test_redeploy_assigments(self, conference, client, test_client,
helpers):
pc_client = openreview.Client(username='******', password='******')
blinded_notes = list(conference.get_submissions())
#Reviewer assignments
pc_client.post_edge(
openreview.Edge(invitation=conference.get_paper_assignment_id(
conference.get_reviewers_id()),
readers=[conference.id, '*****@*****.**'],
writers=[conference.id],
signatures=[conference.id],
head=blinded_notes[0].id,
tail='*****@*****.**',
label='rev-matching-new',
weight=0.98))
pc_client.post_edge(
openreview.Edge(invitation=conference.get_paper_assignment_id(
conference.get_reviewers_id()),
readers=[conference.id, '*****@*****.**'],
writers=[conference.id],
signatures=[conference.id],
head=blinded_notes[1].id,
tail='*****@*****.**',
label='rev-matching-new',
weight=0.98))
pc_client.post_edge(
openreview.Edge(invitation=conference.get_paper_assignment_id(
conference.get_reviewers_id()),
readers=[conference.id, '*****@*****.**'],
writers=[conference.id],
signatures=[conference.id],
head=blinded_notes[2].id,
tail='*****@*****.**',
label='rev-matching-new',
weight=0.98))
edges = client.get_edges(
invitation=
'auai.org/UAI/2019/Conference/Program_Committee/-/Paper_Assignment',
label='rev-matching-new')
assert 3 == len(edges)
conference.set_assignments(assignment_title='rev-matching-new',
overwrite=True)
revs_paper0 = client.get_group(conference.get_id() +
'/Paper{x}/Reviewers'.format(
x=blinded_notes[0].number))
assert ['*****@*****.**'] == revs_paper0.members
revs_paper1 = client.get_group(conference.get_id() +
'/Paper{x}/Reviewers'.format(
x=blinded_notes[1].number))
assert ['~Reviewer_One1'] == revs_paper1.members
revs_paper2 = client.get_group(conference.get_id() +
'/Paper{x}/Reviewers'.format(
x=blinded_notes[2].number))
assert ['*****@*****.**'] == revs_paper2.members
now = datetime.datetime.now()
conference.set_review_stage(openreview.ReviewStage(start_date=now))
invitation = client.get_invitation(
id='auai.org/UAI/2019/Conference/-/Official_Review')
assert invitation
with pytest.raises(openreview.OpenReviewException,
match=r'Review stage has started.'):
conference.set_assignments(assignment_title='rev-matching-new2',
overwrite=True)
def test_setup_matching_with_recommendations(self, conference, client,
test_client, helpers):
blinded_notes = list(conference.get_submissions())
## Open reviewer recommendations
now = datetime.datetime.utcnow()
conference.open_recommendations(assignment_title='',
due_date=now +
datetime.timedelta(minutes=40))
## Recommend reviewers
ac1_client = helpers.get_user('*****@*****.**')
ac1_client.post_edge(
openreview.Edge(
invitation=conference.get_recommendation_id(),
readers=['auai.org/UAI/2019/Conference', '~AreaChair_One1'],
signatures=['~AreaChair_One1'],
writers=['~AreaChair_One1'],
head=blinded_notes[0].id,
tail='~Reviewer_One1',
weight=1))
ac1_client.post_edge(
openreview.Edge(
invitation=conference.get_recommendation_id(),
readers=['auai.org/UAI/2019/Conference', '~AreaChair_One1'],
signatures=['~AreaChair_One1'],
writers=['~AreaChair_One1'],
head=blinded_notes[1].id,
tail='*****@*****.**',
weight=2))
ac1_client.post_edge(
openreview.Edge(
invitation=conference.get_recommendation_id(),
readers=['auai.org/UAI/2019/Conference', '~AreaChair_One1'],
signatures=['~AreaChair_One1'],
writers=['~AreaChair_One1'],
head=blinded_notes[1].id,
tail='*****@*****.**',
weight=3))
# Set up reviewer matching
conference.setup_matching(tpms_score_file=os.path.join(
os.path.dirname(__file__), 'data/reviewer_tpms_scores.csv'),
build_conflicts=True)
print(conference.get_reviewers_id())
invitation = client.get_invitation(
id=
'auai.org/UAI/2019/Conference/Program_Committee/-/Assignment_Configuration'
)
assert invitation
assert 'scores_specification' in invitation.reply['content']
assert 'auai.org/UAI/2019/Conference/Program_Committee/-/Bid' in invitation.reply[
'content']['scores_specification']['default']
assert 'auai.org/UAI/2019/Conference/Program_Committee/-/TPMS_Score' in invitation.reply[
'content']['scores_specification']['default']
assert 'auai.org/UAI/2019/Conference/Program_Committee/-/Subject_Areas_Score' in invitation.reply[
'content']['scores_specification']['default']
assert client.get_invitation(
id=
'auai.org/UAI/2019/Conference/Program_Committee/-/Custom_Max_Papers'
)
assert client.get_invitation(
id='auai.org/UAI/2019/Conference/Program_Committee/-/Conflict')
# Set up ac matching
conference.setup_matching(is_area_chair=True,
tpms_score_file=os.path.join(
os.path.dirname(__file__),
'data/ac_tpms_scores.csv'),
build_conflicts=True)
invitation = client.get_invitation(
id=
'auai.org/UAI/2019/Conference/Senior_Program_Committee/-/Assignment_Configuration'
)
assert invitation
assert 'scores_specification' in invitation.reply['content']
assert 'auai.org/UAI/2019/Conference/Senior_Program_Committee/-/Bid' in invitation.reply[
'content']['scores_specification']['default']
assert 'auai.org/UAI/2019/Conference/Senior_Program_Committee/-/TPMS_Score' in invitation.reply[
'content']['scores_specification']['default']
assert 'auai.org/UAI/2019/Conference/Senior_Program_Committee/-/Subject_Areas_Score' in invitation.reply[
'content']['scores_specification']['default']
assert 'auai.org/UAI/2019/Conference/Program_Committee/-/Recommendation' in invitation.reply[
'content']['scores_specification']['default']
assert client.get_invitation(
id=
'auai.org/UAI/2019/Conference/Senior_Program_Committee/-/Custom_Max_Papers'
)
assert client.get_invitation(
id=
'auai.org/UAI/2019/Conference/Senior_Program_Committee/-/Conflict')
bids = client.get_edges(
invitation=conference.get_bid_id(conference.get_area_chairs_id()))
assert bids
assert 3 == len(bids)
bids = client.get_edges(
invitation=conference.get_bid_id(conference.get_reviewers_id()))
assert bids
assert 3 == len(bids)
recommendations = client.get_edges(
invitation=conference.get_recommendation_id())
assert recommendations
assert 3 == len(recommendations)
reviewer_custom_loads = client.get_edges(
invitation=
'auai.org/UAI/2019/Conference/Program_Committee/-/Custom_Max_Papers'
)
assert not reviewer_custom_loads
ac_custom_loads = client.get_edges(
invitation=
'auai.org/UAI/2019/Conference/Senior_Program_Committee/-/Custom_Max_Papers'
)
assert not ac_custom_loads
reviewer_conflicts = client.get_edges(
invitation=
'auai.org/UAI/2019/Conference/Program_Committee/-/Conflict')
assert 1 == len(reviewer_conflicts)
ac_conflicts = client.get_edges(
invitation=
'auai.org/UAI/2019/Conference/Senior_Program_Committee/-/Conflict')
assert 2 == len(ac_conflicts)
ac1_conflicts = client.get_edges(
invitation=
'auai.org/UAI/2019/Conference/Senior_Program_Committee/-/Conflict',
tail='~AreaChair_One1')
assert ac1_conflicts
assert len(ac1_conflicts)
assert ac1_conflicts[0].label == 'Institutional (level 1)'
r1_conflicts = client.get_edges(
invitation=
'auai.org/UAI/2019/Conference/Program_Committee/-/Conflict',
tail='~Reviewer_One1')
assert r1_conflicts
assert len(r1_conflicts)
assert r1_conflicts[0].label == 'Institutional (level 1)'
ac2_conflicts = client.get_edges(
invitation=
'auai.org/UAI/2019/Conference/Senior_Program_Committee/-/Conflict',
tail='*****@*****.**')
assert ac2_conflicts
assert len(ac2_conflicts)
assert ac2_conflicts[0].label == 'Institutional (level 1)'
submissions = conference.get_submissions()
assert submissions
assert 3 == len(submissions)
reviewer_tpms_scores = client.get_edges(
invitation=
'auai.org/UAI/2019/Conference/Program_Committee/-/TPMS_Score')
assert 9 == len(reviewer_tpms_scores)
ac_tpms_scores = client.get_edges(
invitation=
'auai.org/UAI/2019/Conference/Senior_Program_Committee/-/TPMS_Score'
)
assert 6 == len(ac_tpms_scores)
r3_s0_tpms_scores = client.get_edges(
invitation=
'auai.org/UAI/2019/Conference/Program_Committee/-/TPMS_Score',
tail='*****@*****.**',
head=submissions[0].id)
assert r3_s0_tpms_scores
assert 1 == len(r3_s0_tpms_scores)
assert r3_s0_tpms_scores[0].weight == 0.21
r3_s1_tpms_scores = client.get_edges(
invitation=
'auai.org/UAI/2019/Conference/Program_Committee/-/TPMS_Score',
tail='*****@*****.**',
head=submissions[1].id)
assert r3_s1_tpms_scores
assert 1 == len(r3_s1_tpms_scores)
assert r3_s1_tpms_scores[0].weight == 0.31
r3_s2_tpms_scores = client.get_edges(
invitation=
'auai.org/UAI/2019/Conference/Program_Committee/-/TPMS_Score',
tail='*****@*****.**',
head=submissions[2].id)
assert r3_s2_tpms_scores
assert 1 == len(r3_s2_tpms_scores)
assert r3_s2_tpms_scores[0].weight == 0.51
secondary_ac_demands = []
total_review_demand = 0
print('Accumulating Custom_User_Demand edges')
for paper_num, sub in tqdm(map_submissions.items()):
weight = 0
if paper_num not in set_reject_papers:
weight = 1
total_review_demand += 1
secondary_ac_demands.append(
openreview.Edge(
head=sub.id,
tail='thecvf.com/ECCV/2020/Conference/Secondary_Area_Chairs',
weight=weight,
invitation=
'thecvf.com/ECCV/2020/Conference/Secondary_Area_Chairs/-/Custom_User_Demands',
readers=[
'thecvf.com/ECCV/2020/Conference',
],
writers=['thecvf.com/ECCV/2020/Conference'],
signatures=['thecvf.com/ECCV/2020/Conference']))
print('Custom_User_Demands: Posting {0} edges'.format(
len(secondary_ac_demands)))
posted_edges = openreview.tools.post_bulk_edges(client,
secondary_ac_demands)
print('Custom_User_Demands: Posted {0} edges\n'.format(len(posted_edges)))
conflicts = []
print(
def test_set_assigments(self, conference, client, test_client, helpers):
pc_client = helpers.create_user('*****@*****.**', 'TestPC', 'UAI')
blinded_notes = list(conference.get_submissions())
edges = client.get_edges(
invitation=
'auai.org/UAI/2019/Conference/Senior_Program_Committee/-/Paper_Assignment',
label='ac-matching')
assert 0 == len(edges)
#Reviewer assignments
pc_client.post_edge(
openreview.Edge(invitation=conference.get_paper_assignment_id(
conference.get_reviewers_id()),
readers=[conference.id, '*****@*****.**'],
writers=[conference.id],
signatures=[conference.id],
head=blinded_notes[0].id,
tail='*****@*****.**',
label='rev-matching',
weight=0.98))
pc_client.post_edge(
openreview.Edge(invitation=conference.get_paper_assignment_id(
conference.get_reviewers_id()),
readers=[conference.id, '*****@*****.**'],
writers=[conference.id],
signatures=[conference.id],
head=blinded_notes[0].id,
tail='*****@*****.**',
label='rev-matching',
weight=0.87))
pc_client.post_edge(
openreview.Edge(invitation=conference.get_paper_assignment_id(
conference.get_reviewers_id()),
readers=[conference.id, '*****@*****.**'],
writers=[conference.id],
signatures=[conference.id],
head=blinded_notes[1].id,
tail='*****@*****.**',
label='rev-matching',
weight=0.87))
pc_client.post_edge(
openreview.Edge(invitation=conference.get_paper_assignment_id(
conference.get_reviewers_id()),
readers=[conference.id, '*****@*****.**'],
writers=[conference.id],
signatures=[conference.id],
head=blinded_notes[1].id,
tail='*****@*****.**',
label='rev-matching',
weight=0.94))
pc_client.post_edge(
openreview.Edge(invitation=conference.get_paper_assignment_id(
conference.get_reviewers_id()),
readers=[conference.id, '*****@*****.**'],
writers=[conference.id],
signatures=[conference.id],
head=blinded_notes[2].id,
tail='*****@*****.**',
label='rev-matching',
weight=0.94))
pc_client.post_edge(
openreview.Edge(invitation=conference.get_paper_assignment_id(
conference.get_reviewers_id()),
readers=[conference.id, '*****@*****.**'],
writers=[conference.id],
signatures=[conference.id],
head=blinded_notes[2].id,
tail='*****@*****.**',
label='rev-matching',
weight=0.98))
edges = client.get_edges(
invitation=
'auai.org/UAI/2019/Conference/Program_Committee/-/Paper_Assignment',
label='rev-matching')
assert 6 == len(edges)
conference.set_assignments(assignment_title='rev-matching')
revs_paper0 = client.get_group(conference.get_id() +
'/Paper{x}/Reviewers'.format(
x=blinded_notes[0].number))
assert 2 == len(revs_paper0.members)
revs_paper1 = client.get_group(conference.get_id() +
'/Paper{x}/Reviewers'.format(
x=blinded_notes[1].number))
assert 2 == len(revs_paper1.members)
revs_paper2 = client.get_group(conference.get_id() +
'/Paper{x}/Reviewers'.format(
x=blinded_notes[2].number))
assert 2 == len(revs_paper2.members)
def test_setup_matching(self, conference, client, test_client, helpers):
## Set committee
conference.set_program_chairs(['*****@*****.**', '*****@*****.**'])
conference.set_area_chairs(['*****@*****.**', '*****@*****.**'])
conference.set_reviewers(['*****@*****.**', '*****@*****.**', '*****@*****.**'])
## Paper 1
note_1 = openreview.Note(
invitation=conference.get_submission_id(),
readers=['~Test_User1', '*****@*****.**', '*****@*****.**'],
writers=[
conference.id, '~Test_User1', '*****@*****.**', '*****@*****.**'
],
signatures=['~Test_User1'],
content={
'title':
'Paper title 1',
'abstract':
'This is an abstract',
'authorids': ['*****@*****.**', '*****@*****.**'],
'authors': ['Test User', 'Author 1'],
'subject_areas': [
'Algorithms: Approximate Inference',
'Algorithms: Belief Propagation'
]
})
url = test_client.put_attachment(
os.path.join(os.path.dirname(__file__), 'data/paper.pdf'),
conference.get_submission_id(), 'pdf')
note_1.content['pdf'] = url
note_1 = test_client.post_note(note_1)
## Paper 2
note_2 = openreview.Note(
invitation=conference.get_submission_id(),
readers=['~Test_User1', '*****@*****.**', '*****@*****.**'],
writers=[
conference.id, '~Test_User1', '*****@*****.**', '*****@*****.**'
],
signatures=['~Test_User1'],
content={
'title':
'Paper title 2',
'abstract':
'This is an abstract',
'authorids': ['*****@*****.**', '*****@*****.**'],
'authors': ['Test User', 'Author 2'],
'subject_areas': [
'Algorithms: Approximate Inference',
'Algorithms: Exact Inference'
]
})
url = test_client.put_attachment(
os.path.join(os.path.dirname(__file__), 'data/paper.pdf'),
conference.get_submission_id(), 'pdf')
note_2.content['pdf'] = url
note_2 = test_client.post_note(note_2)
## Paper 3
note_3 = openreview.Note(
invitation=conference.get_submission_id(),
readers=['~Test_User1', '*****@*****.**', '*****@*****.**'],
writers=[
conference.id, '~Test_User1', '*****@*****.**', '*****@*****.**'
],
signatures=['~Test_User1'],
content={
'title':
'Paper title 3',
'abstract':
'This is an abstract',
'authorids': ['*****@*****.**', '*****@*****.**'],
'authors': ['Test User', 'Author 3'],
'subject_areas': [
'Algorithms: Distributed and Parallel',
'Algorithms: Exact Inference'
]
})
url = test_client.put_attachment(
os.path.join(os.path.dirname(__file__), 'data/paper.pdf'),
conference.get_submission_id(), 'pdf')
note_3.content['pdf'] = url
note_3 = test_client.post_note(note_3)
## Create blind submissions
conference.setup_post_submission_stage(force=True)
blinded_notes = conference.get_submissions()
ac1_client = helpers.create_user('*****@*****.**', 'AreaChair', 'One')
ac1_client.post_edge(
openreview.Edge(
invitation=conference.get_bid_id(
conference.get_area_chairs_id()),
readers=['auai.org/UAI/2019/Conference', '~AreaChair_One1'],
writers=['auai.org/UAI/2019/Conference', '~AreaChair_One1'],
signatures=['~AreaChair_One1'],
head=blinded_notes[0].id,
tail='~AreaChair_One1',
label='High'))
ac1_client.post_edge(
openreview.Edge(
invitation=conference.get_bid_id(
conference.get_area_chairs_id()),
readers=['auai.org/UAI/2019/Conference', '~AreaChair_One1'],
writers=['auai.org/UAI/2019/Conference', '~AreaChair_One1'],
signatures=['~AreaChair_One1'],
head=blinded_notes[1].id,
tail='~AreaChair_One1',
label='Low'))
ac1_client.post_edge(
openreview.Edge(
invitation=conference.get_bid_id(
conference.get_area_chairs_id()),
readers=['auai.org/UAI/2019/Conference', '~AreaChair_One1'],
writers=['auai.org/UAI/2019/Conference', '~AreaChair_One1'],
signatures=['~AreaChair_One1'],
head=blinded_notes[2].id,
tail='~AreaChair_One1',
label='Very Low'))
r1_client = helpers.create_user('*****@*****.**', 'Reviewer', 'One')
r1_client.post_edge(
openreview.Edge(
invitation=conference.get_bid_id(
conference.get_reviewers_id()),
readers=['auai.org/UAI/2019/Conference', '~Reviewer_One1'],
writers=['auai.org/UAI/2019/Conference', '~Reviewer_One1'],
signatures=['~Reviewer_One1'],
head=blinded_notes[0].id,
tail='~Reviewer_One1',
label='Neutral'))
r1_client.post_edge(
openreview.Edge(
invitation=conference.get_bid_id(
conference.get_reviewers_id()),
readers=['auai.org/UAI/2019/Conference', '~Reviewer_One1'],
writers=['auai.org/UAI/2019/Conference', '~Reviewer_One1'],
signatures=['~Reviewer_One1'],
head=blinded_notes[1].id,
tail='~Reviewer_One1',
label='Very High'))
r1_client.post_edge(
openreview.Edge(
invitation=conference.get_bid_id(
conference.get_reviewers_id()),
readers=['auai.org/UAI/2019/Conference', '~Reviewer_One1'],
writers=['auai.org/UAI/2019/Conference', '~Reviewer_One1'],
signatures=['~Reviewer_One1'],
head=blinded_notes[2].id,
tail='~Reviewer_One1',
label='Low'))
# Set up reviewer matching
conference.setup_matching(build_conflicts=True)
invitation = client.get_invitation(
id=
'auai.org/UAI/2019/Conference/Program_Committee/-/Assignment_Configuration'
)
assert invitation
assert 'scores_specification' in invitation.reply['content']
assert 'auai.org/UAI/2019/Conference/Program_Committee/-/Bid' in invitation.reply[
'content']['scores_specification']['default']
assert client.get_invitation(
id=
'auai.org/UAI/2019/Conference/Program_Committee/-/Custom_Max_Papers'
)
assert client.get_invitation(
id='auai.org/UAI/2019/Conference/Program_Committee/-/Conflict')
assert client.get_invitation(
id=
'auai.org/UAI/2019/Conference/Program_Committee/-/Aggregate_Score')
assert client.get_invitation(
id=
'auai.org/UAI/2019/Conference/Program_Committee/-/Paper_Assignment'
)
# Set up AC matching
conference.setup_matching(is_area_chair=True, build_conflicts=True)
invitation = client.get_invitation(
id=
'auai.org/UAI/2019/Conference/Senior_Program_Committee/-/Assignment_Configuration'
)
assert invitation
assert 'scores_specification' in invitation.reply['content']
assert 'auai.org/UAI/2019/Conference/Senior_Program_Committee/-/Bid' in invitation.reply[
'content']['scores_specification']['default']
assert client.get_invitation(
id=
'auai.org/UAI/2019/Conference/Senior_Program_Committee/-/Custom_Max_Papers'
)
assert client.get_invitation(
id=
'auai.org/UAI/2019/Conference/Senior_Program_Committee/-/Conflict')
assert client.get_invitation(
id=
'auai.org/UAI/2019/Conference/Senior_Program_Committee/-/Aggregate_Score'
)
assert client.get_invitation(
id=
'auai.org/UAI/2019/Conference/Senior_Program_Committee/-/Paper_Assignment'
)
bids = client.get_edges(
invitation=conference.get_bid_id(conference.get_area_chairs_id()))
assert bids
assert 3 == len(bids)
bids = client.get_edges(
invitation=conference.get_bid_id(conference.get_reviewers_id()))
assert bids
assert 3 == len(bids)
reviewer_custom_loads = client.get_edges(
invitation=
'auai.org/UAI/2019/Conference/Program_Committee/-/Custom_Max_Papers'
)
assert not reviewer_custom_loads
ac_custom_loads = client.get_edges(
invitation=
'auai.org/UAI/2019/Conference/Senior_Program_Committee/-/Custom_Max_Papers'
)
assert not ac_custom_loads
reviewer_conflicts = client.get_edges(
invitation=
'auai.org/UAI/2019/Conference/Program_Committee/-/Conflict')
assert 1 == len(reviewer_conflicts)
ac_conflicts = client.get_edges(
invitation=
'auai.org/UAI/2019/Conference/Senior_Program_Committee/-/Conflict')
assert 2 == len(ac_conflicts)
ac1_conflicts = client.get_edges(
invitation=
'auai.org/UAI/2019/Conference/Senior_Program_Committee/-/Conflict',
tail='~AreaChair_One1')
assert ac1_conflicts
assert len(ac1_conflicts)
assert ac1_conflicts[0].label == 'Institutional (level 1)'
r1_conflicts = client.get_edges(
invitation=
'auai.org/UAI/2019/Conference/Program_Committee/-/Conflict',
tail='~Reviewer_One1')
assert r1_conflicts
assert len(r1_conflicts)
assert r1_conflicts[0].label == 'Institutional (level 1)'
ac2_conflicts = client.get_edges(
invitation=
'auai.org/UAI/2019/Conference/Senior_Program_Committee/-/Conflict',
tail='*****@*****.**')
assert ac2_conflicts
assert len(ac2_conflicts)
assert ac2_conflicts[0].label == 'Institutional (level 1)'