def _build_subject_area_scores(self, submissions): ''' Create subject area scores between all users in the match group and all given submissions ''' invitation = self._create_edge_invitation( self._get_edge_invitation_id('Subject_Areas_Score'), extendable_readers=True) edges = [] user_subject_areas = list( openreview.tools.iterget_notes( self.client, invitation=self.conference.get_registration_id())) for note in submissions: note_subject_areas = note.content['subject_areas'] paper_note_id = note.id for subject_area_note in user_subject_areas: profile_id = subject_area_note.signatures[0] if profile_id in self.match_group.members: subject_areas = subject_area_note.content['subject_areas'] score = _jaccard_similarity(note_subject_areas, subject_areas) edges.append( openreview.Edge(invitation=invitation.id, head=paper_note_id, tail=profile_id, weight=float(score), readers=[self.conference.id], writers=[self.conference.id], signatures=[self.conference.id])) openreview.tools.post_bulk_edges(client=self.conference.client, edges=edges) return invitation
def _build_scores(self, score_invitation_id, score_file): ''' Given a csv file with affinity scores, create score edges ''' invitation = self._create_edge_invitation(score_invitation_id, extendable_readers=True) edges = [] with open(score_file) as file_handle: for row in csv.reader(file_handle): paper_note_id = row[0] profile_id = row[1] score = row[2] edges.append( openreview.Edge(invitation=invitation.id, head=paper_note_id, tail=profile_id, weight=float(score), readers=[self.conference.id, profile_id], writers=[self.conference.id], signatures=[self.conference.id])) openreview.tools.post_bulk_edges(client=self.conference.client, edges=edges) return invitation
def create_reviewer_edge(client, conference, name, note, reviewer, label=None, weight=None): conference_id = conference.id return client.post_edge( openreview.Edge( invitation=f'{conference.id}/Reviewers/-/{name}', readers=[ conference_id, conference.get_senior_area_chairs_id(number=note.number), conference.get_area_chairs_id(number=note.number), reviewer ], nonreaders=[conference.get_authors_id(number=note.number)], writers=[ conference_id, conference.get_senior_area_chairs_id(number=note.number), conference.get_area_chairs_id(number=note.number) ], signatures=[conference_id], head=note.id, tail=reviewer, label=label, weight=weight))
def add_config_custom_loads(self): ''' The supply deduction > 0 indicates that some reviewers cannot do the default minimum number of reviews and we want to cycle through the reviewers lowering their loads until the supply deduction is met. :return: ''' loads = { reviewer: self.params.reviewer_max_papers for reviewer in self.reviewers } if self.params.custom_load_supply_deduction: self.reduce_reviewer_loads( loads, self.params.custom_load_supply_deduction) elif self.params.custom_load_map != {}: self.set_reviewer_loads(loads, self.params.custom_load_map) # build custom_load edge for those reviewers that are different from the default max edges = [] for rev, load in loads.items(): if load != self.params.reviewer_max_papers: edge = openreview.Edge( invitation=self.conf_ids.CUSTOM_LOAD_INV_ID, label=self.config_title, head=self.conf_ids.CONF_ID, tail=rev, weight=load, readers=[self.conf_ids.CONF_ID], writers=[self.conf_ids.CONF_ID], signatures=[self.conf_ids.CONF_ID]) edges.append(edge) openreview.tools.post_bulk_edges(self.client, edges)
def _build_conflicts(self, submissions, user_profiles): ''' Create conflict edges between the given Notes and Profiles ''' invitation = self._create_edge_invitation( self.conference.get_conflict_score_id(self.match_group.id), extendable_readers=True) authorids_profiles = {} for submission in submissions: edges = [] for profile in user_profiles: authorids = submission.content['authorids'] if submission.details and submission.details.get('original'): authorids = submission.details['original']['content'][ 'authorids'] if submission.number not in authorids_profiles: profiles = _get_profiles(self.client, authorids) authorids_profiles[submission.number] = profiles author_profiles = authorids_profiles[submission.number] conflicts = openreview.tools.get_conflicts( author_profiles, profile) if conflicts: edges.append( openreview.Edge( invitation=invitation.id, head=submission.id, tail=profile.id, weight=-1, label=_conflict_label(conflicts), readers=[self.conference.id, profile.id], writers=[self.conference.id], signatures=[self.conference.id])) openreview.tools.post_bulk_edges(client=self.client, edges=edges) return invitation
def _build_edge(self, invitation, forum_id, reviewer, score, label, number): ''' Helper function for constructing an openreview.Edge object. Readers, nonreaders, writers, and signatures are automatically filled based on the invitaiton. ''' return openreview.Edge( head = forum_id, tail = reviewer, weight = score, label = label, invitation = invitation.id, readers = self._get_values(invitation, number, 'readers', forum_id, reviewer), nonreaders = self._get_values(invitation, number, 'nonreaders'), writers = self._get_values(invitation, number, 'writers'), signatures = [self.venue_id])
def create_constraint_edges(self, constraints, val): constraint_edge_inv = self.conf_ids.CONSTRAINTS_INV_ID edges = [] for paper_ix, reviewers_list in constraints.items(): p = self.paper_notes[paper_ix] for reviewer_ix in reviewers_list: r = self.reviewers[reviewer_ix] e = openreview.Edge(head=p.id, tail=r, label=self.config_title, weight=val, invitation=constraint_edge_inv, readers=[self.conf_ids.CONF_ID], writers=[self.conf_ids.CONF_ID], signatures=[r]) edges.append(e) openreview.tools.post_bulk_edges(self.client, edges)
def _build_tpms_scores(self, tpms_score_file, submissions, user_profiles): ''' Create tpms score edges given a csv file with scores, papers, and profiles. ''' # pylint: disable=too-many-locals invitation = self._create_edge_invitation( self._get_edge_invitation_id('TPMS_Score'), extendable_readers=True) submissions_per_number = {note.number: note for note in submissions} profiles_by_email = {} for profile in user_profiles: for email in profile.content['emails']: profiles_by_email[email] = profile edges = [] with open(tpms_score_file) as file_handle: for row in csv.reader(file_handle): number = int(row[0]) if number in submissions_per_number: paper_note_id = submissions_per_number[number].id profile = profiles_by_email.get(row[1]) if profile: profile_id = profile.id else: profile_id = row[1] score = row[2] edges.append( openreview.Edge( invitation=invitation.id, head=paper_note_id, tail=profile_id, weight=float(score), readers=[self.conference.id, profile.id], writers=[self.conference.id], signatures=[self.conference.id])) openreview.tools.post_bulk_edges(client=self.client, edges=edges) return invitation
def test_save_bulk(self, client): builder = openreview.conference.ConferenceBuilder(client) assert builder, 'builder is None' builder.set_conference_id('NIPS.cc/2020/Workshop/MLITS') builder.set_submission_stage(public=True) conference = builder.get_result() # Edge invitation inv1 = openreview.Invitation( id=conference.id + '/-/affinity', reply={ 'content': { 'head': { 'type': 'Note' }, 'tail': { 'type': 'Profile', }, 'label': { 'value-radio': ['Very High', 'High', 'Neutral', 'Low', 'Very Low'] }, 'weight': { 'value-regex': r'[0-9]+\.[0-9]' } } }) inv1 = client.post_invitation(inv1) # Sample note note = openreview.Note( invitation=conference.get_submission_id(), readers=['everyone'], writers=[conference.id], signatures=['~Super_User1'], content={ 'title': 'Paper title', 'abstract': 'This is an abstract', 'authorids': ['*****@*****.**'], 'authors': ['Test User'], 'pdf': '/pdf/22234qweoiuweroi22234qweoiuweroi12345678.pdf' }) note = client.post_note(note) # Edges edges = [] for p in range(1000): edge = openreview.Edge(head=note.id, tail='~Super_User1', label='High', weight=0.5, invitation=inv1.id, readers=['everyone'], writers=[conference.id], signatures=['~Super_User1']) edges.append(edge) openreview.tools.post_bulk_edges(client, edges) them = list(openreview.tools.iterget_edges(client, invitation=inv1.id)) assert len(edges) == len(them)
for i in ids: if i in confirmations: confirmation = confirmations[i] if confirmation: review_capacity = int( confirmation.content.get('emergency_review_count', '0')) if review_capacity: total_review_capacity += review_capacity edge = openreview.Edge( head='thecvf.com/ECCV/2020/Conference/Emergency_Reviewers', tail=profile.id, invitation= 'thecvf.com/ECCV/2020/Conference/Emergency_Reviewers/-/Custom_Max_Papers', readers=[ 'thecvf.com/ECCV/2020/Conference', 'thecvf.com/ECCV/2020/Conference/Area_Chairs', profile.id ], writers=['thecvf.com/ECCV/2020/Conference'], signatures=['thecvf.com/ECCV/2020/Conference'], weight=review_capacity) emergency_load_edges.append(edge) emergency_reviewer_group.members.append(profile.id) x = client.post_group(emergency_reviewer_group) print('Created updated emergency reviewers group\n') print('Posting {0} edges'.format(len(emergency_load_edges))) posted_edges = openreview.tools.post_bulk_edges(client, emergency_load_edges)
def test_setup_matching_with_recommendations(self, client, test_client, helpers): builder = openreview.conference.ConferenceBuilder(client) assert builder, 'builder is None' builder = openreview.conference.ConferenceBuilder(client) builder.set_conference_id('auai.org/UAI/2019/Conference') builder.set_conference_name('Conference on Uncertainty in Artificial Intelligence') builder.set_conference_short_name('UAI 2019') builder.set_homepage_header({ 'title': 'UAI 2019', 'subtitle': 'Conference on Uncertainty in Artificial Intelligence', 'deadline': 'Abstract Submission Deadline: 11:59 pm Samoa Standard Time, March 4, 2019, Full Submission Deadline: 11:59 pm Samoa Standard Time, March 8, 2019', 'date': 'July 22 - July 25, 2019', 'website': 'http://auai.org/uai2019/', 'location': 'Tel Aviv, Israel', 'instructions': '''<p><strong>Important Information about Anonymity:</strong><br> When you post a submission to UAI 2019, please provide the real names and email addresses of authors in the submission form below (but NOT in the manuscript). The <em>original</em> record of your submission will be private, and will contain your real name(s). The PDF in your submission should not contain the names of the authors. </p> <p><strong>Conflict of Interest:</strong><br> Please make sure that your current and previous affiliations listed on your OpenReview <a href=\"/profile\">profile page</a> is up-to-date to avoid conflict of interest.</p> <p><strong>Questions or Concerns:</strong><br> Please contact the UAI 2019 Program chairs at <a href=\"mailto:[email protected]\">[email protected]</a>. <br>Please contact the OpenReview support team at <a href=\"mailto:[email protected]\">[email protected]</a> with any OpenReview related questions or concerns. </p>''' }) print ('Homepage header set') builder.set_conference_area_chairs_name('Senior_Program_Committee') builder.set_conference_reviewers_name('Program_Committee') builder.set_override_homepage(True) now = datetime.datetime.utcnow() builder.set_submission_stage(due_date = now + datetime.timedelta(minutes = 40), double_blind= True, subject_areas=[ "Algorithms: Approximate Inference", "Algorithms: Belief Propagation", "Algorithms: Distributed and Parallel", "Algorithms: Exact Inference", ]) conference = builder.get_result() assert conference, 'conference is None' blinded_notes = list(conference.get_submissions()) ## Open reviewer recommendations now = datetime.datetime.utcnow() conference.open_recommendations(due_date = now + datetime.timedelta(minutes = 40)) ## Recommend reviewers ac1_client = helpers.get_user('*****@*****.**') ac1_client.post_edge(openreview.Edge(invitation = conference.get_recommendation_id(), readers = ['auai.org/UAI/2019/Conference', '~AreaChair_One1'], signatures = ['~AreaChair_One1'], writers = ['~AreaChair_One1'], head = blinded_notes[0].id, tail = '~Reviewer_One1', weight = 1 )) ac1_client.post_edge(openreview.Edge(invitation = conference.get_recommendation_id(), readers = ['auai.org/UAI/2019/Conference', '~AreaChair_One1'], signatures = ['~AreaChair_One1'], writers = ['~AreaChair_One1'], head = blinded_notes[1].id, tail = '*****@*****.**', weight = 2 )) ac1_client.post_edge(openreview.Edge(invitation = conference.get_recommendation_id(), readers = ['auai.org/UAI/2019/Conference', '~AreaChair_One1'], signatures = ['~AreaChair_One1'], writers = ['~AreaChair_One1'], head = blinded_notes[1].id, tail = '*****@*****.**', weight = 3 )) # Set up reviewer matching conference.setup_matching(tpms_score_file=os.path.join(os.path.dirname(__file__), 'data/reviewer_tpms_scores.csv')) print(conference.get_reviewers_id()) invitation = client.get_invitation(id='auai.org/UAI/2019/Conference/Program_Committee/-/Assignment_Configuration') assert invitation assert 'scores_specification' in invitation.reply['content'] assert 'auai.org/UAI/2019/Conference/Program_Committee/-/Bid' in invitation.reply['content']['scores_specification']['default'] assert 'auai.org/UAI/2019/Conference/Program_Committee/-/TPMS_Score' in invitation.reply['content']['scores_specification']['default'] assert 'auai.org/UAI/2019/Conference/Program_Committee/-/Subject_Areas_Score' in invitation.reply['content']['scores_specification']['default'] assert 'auai.org/UAI/2019/Conference/-/Recommendation' in invitation.reply['content']['scores_specification']['default'] assert client.get_invitation(id='auai.org/UAI/2019/Conference/Program_Committee/-/Custom_Load') assert client.get_invitation(id='auai.org/UAI/2019/Conference/Program_Committee/-/Conflict') # Set up ac matching conference.setup_matching( is_area_chair=True, tpms_score_file=os.path.join(os.path.dirname(__file__), 'data/ac_tpms_scores.csv')) assert client.get_invitation(id='auai.org/UAI/2019/Conference/Senior_Program_Committee/-/Custom_Load') assert client.get_invitation(id='auai.org/UAI/2019/Conference/Senior_Program_Committee/-/Conflict') bids = client.get_edges(invitation = conference.get_bid_id(conference.get_area_chairs_id())) assert bids assert 3 == len(bids) bids = client.get_edges(invitation = conference.get_bid_id(conference.get_reviewers_id())) assert bids assert 3 == len(bids) recommendations = client.get_edges(invitation = conference.get_recommendation_id()) assert recommendations assert 3 == len(recommendations) reviewer_custom_loads = client.get_edges( invitation='auai.org/UAI/2019/Conference/Program_Committee/-/Custom_Load') assert not reviewer_custom_loads ac_custom_loads = client.get_edges( invitation='auai.org/UAI/2019/Conference/Senior_Program_Committee/-/Custom_Load') assert not ac_custom_loads reviewer_conflicts = client.get_edges( invitation='auai.org/UAI/2019/Conference/Program_Committee/-/Conflict') assert 1 == len(reviewer_conflicts) ac_conflicts = client.get_edges( invitation='auai.org/UAI/2019/Conference/Senior_Program_Committee/-/Conflict') assert 2 == len(ac_conflicts) ac1_conflicts = client.get_edges( invitation='auai.org/UAI/2019/Conference/Senior_Program_Committee/-/Conflict', tail='~AreaChair_One1') assert ac1_conflicts assert len(ac1_conflicts) assert ac1_conflicts[0].label == 'Institutional (level 1)' r1_conflicts = client.get_edges( invitation='auai.org/UAI/2019/Conference/Program_Committee/-/Conflict', tail='~Reviewer_One1') assert r1_conflicts assert len(r1_conflicts) assert r1_conflicts[0].label == 'Institutional (level 1)' ac2_conflicts = client.get_edges( invitation='auai.org/UAI/2019/Conference/Senior_Program_Committee/-/Conflict', tail='*****@*****.**') assert ac2_conflicts assert len(ac2_conflicts) assert ac2_conflicts[0].label == 'Institutional (level 1)' submissions = conference.get_submissions() assert submissions assert 3 == len(submissions) reviewer_tpms_scores = client.get_edges( invitation='auai.org/UAI/2019/Conference/Program_Committee/-/TPMS_Score') assert 9 == len(reviewer_tpms_scores) ac_tpms_scores = client.get_edges( invitation='auai.org/UAI/2019/Conference/Senior_Program_Committee/-/TPMS_Score') assert 6 == len(ac_tpms_scores) r3_s0_tpms_scores = client.get_edges( invitation='auai.org/UAI/2019/Conference/Program_Committee/-/TPMS_Score', tail='*****@*****.**', head=submissions[0].id) assert r3_s0_tpms_scores assert 1 == len(r3_s0_tpms_scores) assert r3_s0_tpms_scores[0].weight == 0.21 r3_s1_tpms_scores = client.get_edges( invitation='auai.org/UAI/2019/Conference/Program_Committee/-/TPMS_Score', tail='*****@*****.**', head=submissions[1].id) assert r3_s1_tpms_scores assert 1 == len(r3_s1_tpms_scores) assert r3_s1_tpms_scores[0].weight == 0.31 r3_s2_tpms_scores = client.get_edges( invitation='auai.org/UAI/2019/Conference/Program_Committee/-/TPMS_Score', tail='*****@*****.**', head=submissions[2].id) assert r3_s2_tpms_scores assert 1 == len(r3_s2_tpms_scores) assert r3_s2_tpms_scores[0].weight == 0.51
def build_conflicts(match_group, submissions): edges = [] invitation = _create_edge_invitation( conference.get_invitation_id('Conflict', prefix=match_group.id), match_group) user_profiles = openreview.matching._get_profiles(client, match_group.members) user_profiles_info = [ get_profile_info(profile) for profile in user_profiles ] for submission in submissions: author_domains = set() author_emails = set() author_relations = set() author_publications = set() author_profiles = openreview.matching._get_profiles( client, submission.details['original']['content']['authorids']) for author in author_profiles: author_info = get_profile_info(author) author_domains.update(author_info['domains']) author_emails.update(author_info['emails']) author_relations.update(author_info['relations']) author_publications.update(author_info['publications']) for user_info in user_profiles_info: conflicts = set() conflicts.update(author_domains.intersection(user_info['domains'])) conflicts.update(author_relations.intersection( user_info['emails'])) conflicts.update(author_emails.intersection( user_info['relations'])) conflicts.update(author_emails.intersection(user_info['emails'])) conflicts.update( author_publications.intersection(user_info['publications'])) if conflicts: edges.append( openreview.Edge( invitation=invitation.id, head=submission.id, tail=user_info['id'], weight=-1, label='Conflict', readers=_get_edge_readers(tail=user_info['id']), writers=[conference.id], signatures=[conference.id])) openreview.tools.post_bulk_edges(client, edges=edges) # Perform sanity check edges_posted = client.get_edges_count(invitation=invitation.id) if edges_posted < len(edges): raise openreview.OpenReviewException( 'Failed during bulk post of Conflict edges! Scores found: {0}, Edges posted: {1}' .format(len(edges), edges_posted)) return edges
def test_integration_no_solution_due_to_conflicts(openreview_context): ''' Basic integration test. Makes use of the OpenReview Builder ''' openreview_client = openreview_context['openreview_client'] test_client = openreview_context['test_client'] conference_id = 'ICLR.cca/2020/Conference' num_reviewers = 3 num_papers = 1 reviews_per_paper = 3 max_papers = 1 min_papers = 1 alternates = 0 conference = clean_start_conference(openreview_client, conference_id, num_reviewers, num_papers, reviews_per_paper) reviewers_id = conference.get_reviewers_id() config = { 'title': 'integration-test', 'user_demand': str(reviews_per_paper), 'max_papers': str(max_papers), 'min_papers': str(min_papers), 'alternates': str(alternates), 'config_invitation': '{}/-/Assignment_Configuration'.format(reviewers_id), 'paper_invitation': conference.get_blind_submission_id(), 'assignment_invitation': conference.get_paper_assignment_id(reviewers_id), 'deployed_assignment_invitation': conference.get_paper_assignment_id(reviewers_id, deployed=True), 'invite_assignment_invitation': conference.get_paper_assignment_id(reviewers_id, invite=True), 'aggregate_score_invitation': '{}/-/Aggregate_Score'.format(reviewers_id), 'conflicts_invitation': conference.get_conflict_score_id(reviewers_id), 'custom_max_papers_invitation': '{}/-/Custom_Max_Papers'.format(reviewers_id), 'match_group': reviewers_id, 'scores_specification': { conference.get_affinity_score_id(reviewers_id): { 'weight': 1.0, 'default': 0.0 } }, 'status': 'Initialized', 'solver': 'MinMax' } config_note = openreview.Note( **{ 'invitation': '{}/-/Assignment_Configuration'.format(reviewers_id), 'readers': [conference.get_id()], 'writers': [conference.get_id()], 'signatures': [conference.get_id()], 'content': config }) for reviewer in openreview_client.get_group(reviewers_id).members: openreview_client.post_edge( openreview.Edge( invitation=conference.get_conflict_score_id(reviewers_id), label="Personal", weight=-1, head=conference.get_submissions()[-1].id, tail=reviewer, signatures=[conference.id], readers=[conference.id, reviewer], writers=[conference.id], )) config_note = openreview_client.post_note(config_note) assert config_note response = test_client.post('/match', data=json.dumps( {'configNoteId': config_note.id}), content_type='application/json', headers=openreview_client.headers) assert response.status_code == 200 matcher_status = wait_for_status(openreview_client, config_note.id) assert matcher_status.content['status'] == 'No Solution' assert matcher_status.content[ 'error_message'] == 'Solver could not find a solution. Adjust your parameters' paper_assignment_edges = openreview_client.get_edges( label='integration-test', invitation=conference.get_paper_assignment_id( conference.get_reviewers_id())) assert len(paper_assignment_edges) == 0
def test_setup_matching(self, client, test_client, helpers): builder = openreview.conference.ConferenceBuilder(client) assert builder, 'builder is None' builder = openreview.conference.ConferenceBuilder(client) builder.set_conference_id('auai.org/UAI/2019/Conference') builder.set_conference_name('Conference on Uncertainty in Artificial Intelligence') builder.set_conference_short_name('UAI 2019') builder.set_homepage_header({ 'title': 'UAI 2019', 'subtitle': 'Conference on Uncertainty in Artificial Intelligence', 'deadline': 'Abstract Submission Deadline: 11:59 pm Samoa Standard Time, March 4, 2019, Full Submission Deadline: 11:59 pm Samoa Standard Time, March 8, 2019', 'date': 'July 22 - July 25, 2019', 'website': 'http://auai.org/uai2019/', 'location': 'Tel Aviv, Israel', 'instructions': '''<p><strong>Important Information about Anonymity:</strong><br> When you post a submission to UAI 2019, please provide the real names and email addresses of authors in the submission form below (but NOT in the manuscript). The <em>original</em> record of your submission will be private, and will contain your real name(s). The PDF in your submission should not contain the names of the authors. </p> <p><strong>Conflict of Interest:</strong><br> Please make sure that your current and previous affiliations listed on your OpenReview <a href=\"/profile\">profile page</a> is up-to-date to avoid conflict of interest.</p> <p><strong>Questions or Concerns:</strong><br> Please contact the UAI 2019 Program chairs at <a href=\"mailto:[email protected]\">[email protected]</a>. <br>Please contact the OpenReview support team at <a href=\"mailto:[email protected]\">[email protected]</a> with any OpenReview related questions or concerns. </p>''' }) print ('Homepage header set') builder.set_conference_area_chairs_name('Senior_Program_Committee') builder.set_conference_reviewers_name('Program_Committee') builder.set_override_homepage(True) now = datetime.datetime.utcnow() builder.set_submission_stage(due_date = now + datetime.timedelta(minutes = 40), double_blind= True, subject_areas=[ "Algorithms: Approximate Inference", "Algorithms: Belief Propagation", "Algorithms: Distributed and Parallel", "Algorithms: Exact Inference", ]) builder.set_bid_stage(due_date = now + datetime.timedelta(minutes = 40), request_count = 50) conference = builder.get_result() assert conference, 'conference is None' ## Set committee conference.set_program_chairs(['*****@*****.**', '*****@*****.**']) conference.set_area_chairs(['*****@*****.**', '*****@*****.**']) conference.set_reviewers(['*****@*****.**', '*****@*****.**', '*****@*****.**']) ## Paper 1 note_1 = openreview.Note(invitation = conference.get_submission_id(), readers = ['~Test_User1', '*****@*****.**', '*****@*****.**'], writers = [conference.id, '~Test_User1', '*****@*****.**', '*****@*****.**'], signatures = ['~Test_User1'], content = { 'title': 'Paper title 1', 'abstract': 'This is an abstract', 'authorids': ['*****@*****.**', '*****@*****.**'], 'authors': ['Test User', 'Author 1'], 'subject_areas': [ 'Algorithms: Approximate Inference', 'Algorithms: Belief Propagation' ] } ) url = test_client.put_pdf(os.path.join(os.path.dirname(__file__), 'data/paper.pdf')) note_1.content['pdf'] = url note_1 = test_client.post_note(note_1) ## Paper 2 note_2 = openreview.Note(invitation = conference.get_submission_id(), readers = ['~Test_User1', '*****@*****.**', '*****@*****.**'], writers = [conference.id, '~Test_User1', '*****@*****.**', '*****@*****.**'], signatures = ['~Test_User1'], content = { 'title': 'Paper title 2', 'abstract': 'This is an abstract', 'authorids': ['*****@*****.**', '*****@*****.**'], 'authors': ['Test User', 'Author 2'], 'subject_areas': [ 'Algorithms: Approximate Inference', 'Algorithms: Exact Inference' ] } ) url = test_client.put_pdf(os.path.join(os.path.dirname(__file__), 'data/paper.pdf')) note_2.content['pdf'] = url note_2 = test_client.post_note(note_2) ## Paper 3 note_3 = openreview.Note(invitation = conference.get_submission_id(), readers = ['~Test_User1', '*****@*****.**', '*****@*****.**'], writers = [conference.id, '~Test_User1', '*****@*****.**', '*****@*****.**'], signatures = ['~Test_User1'], content = { 'title': 'Paper title 3', 'abstract': 'This is an abstract', 'authorids': ['*****@*****.**', '*****@*****.**'], 'authors': ['Test User', 'Author 3'], 'subject_areas': [ 'Algorithms: Distributed and Parallel', 'Algorithms: Exact Inference' ] } ) url = test_client.put_pdf(os.path.join(os.path.dirname(__file__), 'data/paper.pdf')) note_3.content['pdf'] = url note_3 = test_client.post_note(note_3) ## Create blind submissions builder.set_submission_stage(due_date = now, double_blind= True, subject_areas=[ "Algorithms: Approximate Inference", "Algorithms: Belief Propagation", "Algorithms: Distributed and Parallel", "Algorithms: Exact Inference", ]) conference = builder.get_result() blinded_notes = conference.create_blind_submissions() conference.set_authors() ac1_client = helpers.create_user('*****@*****.**', 'AreaChair', 'One') ac1_client.post_edge(openreview.Edge(invitation = conference.get_bid_id(conference.get_area_chairs_id()), readers = ['auai.org/UAI/2019/Conference', '~AreaChair_One1'], writers = ['auai.org/UAI/2019/Conference', '~AreaChair_One1'], signatures = ['~AreaChair_One1'], head = blinded_notes[0].id, tail = '~AreaChair_One1', label = 'High' )) ac1_client.post_edge(openreview.Edge(invitation = conference.get_bid_id(conference.get_area_chairs_id()), readers = ['auai.org/UAI/2019/Conference', '~AreaChair_One1'], writers = ['auai.org/UAI/2019/Conference', '~AreaChair_One1'], signatures = ['~AreaChair_One1'], head = blinded_notes[1].id, tail = '~AreaChair_One1', label = 'Low' )) ac1_client.post_edge(openreview.Edge(invitation = conference.get_bid_id(conference.get_area_chairs_id()), readers = ['auai.org/UAI/2019/Conference', '~AreaChair_One1'], writers = ['auai.org/UAI/2019/Conference', '~AreaChair_One1'], signatures = ['~AreaChair_One1'], head = blinded_notes[2].id, tail = '~AreaChair_One1', label = 'Very Low' )) r1_client = helpers.create_user('*****@*****.**', 'Reviewer', 'One') r1_client.post_edge(openreview.Edge(invitation = conference.get_bid_id(conference.get_reviewers_id()), readers = ['auai.org/UAI/2019/Conference', '~Reviewer_One1'], writers = ['auai.org/UAI/2019/Conference', '~Reviewer_One1'], signatures = ['~Reviewer_One1'], head = blinded_notes[0].id, tail = '~Reviewer_One1', label = 'Neutral' )) r1_client.post_edge(openreview.Edge(invitation = conference.get_bid_id(conference.get_reviewers_id()), readers = ['auai.org/UAI/2019/Conference', '~Reviewer_One1'], writers = ['auai.org/UAI/2019/Conference', '~Reviewer_One1'], signatures = ['~Reviewer_One1'], head = blinded_notes[1].id, tail = '~Reviewer_One1', label = 'Very High' )) r1_client.post_edge(openreview.Edge(invitation = conference.get_bid_id(conference.get_reviewers_id()), readers = ['auai.org/UAI/2019/Conference', '~Reviewer_One1'], writers = ['auai.org/UAI/2019/Conference', '~Reviewer_One1'], signatures = ['~Reviewer_One1'], head = blinded_notes[2].id, tail = '~Reviewer_One1', label = 'Low' )) # Set up reviewer matching conference.setup_matching() invitation = client.get_invitation(id='auai.org/UAI/2019/Conference/Program_Committee/-/Assignment_Configuration') assert invitation assert 'scores_specification' in invitation.reply['content'] assert 'auai.org/UAI/2019/Conference/Program_Committee/-/Bid' in invitation.reply['content']['scores_specification']['default'] assert client.get_invitation(id='auai.org/UAI/2019/Conference/Program_Committee/-/Custom_Load') assert client.get_invitation(id='auai.org/UAI/2019/Conference/Program_Committee/-/Conflict') assert client.get_invitation(id='auai.org/UAI/2019/Conference/Program_Committee/-/Aggregate_Score') assert client.get_invitation(id='auai.org/UAI/2019/Conference/Program_Committee/-/Paper_Assignment') # Set up AC matching conference.setup_matching(is_area_chair=True) invitation = client.get_invitation(id='auai.org/UAI/2019/Conference/Senior_Program_Committee/-/Assignment_Configuration') assert invitation assert 'scores_specification' in invitation.reply['content'] assert 'auai.org/UAI/2019/Conference/Senior_Program_Committee/-/Bid' in invitation.reply['content']['scores_specification']['default'] assert client.get_invitation(id='auai.org/UAI/2019/Conference/Senior_Program_Committee/-/Custom_Load') assert client.get_invitation(id='auai.org/UAI/2019/Conference/Senior_Program_Committee/-/Conflict') assert client.get_invitation(id='auai.org/UAI/2019/Conference/Senior_Program_Committee/-/Aggregate_Score') assert client.get_invitation(id='auai.org/UAI/2019/Conference/Senior_Program_Committee/-/Paper_Assignment') bids = client.get_edges(invitation = conference.get_bid_id(conference.get_area_chairs_id())) assert bids assert 3 == len(bids) bids = client.get_edges(invitation = conference.get_bid_id(conference.get_reviewers_id())) assert bids assert 3 == len(bids) reviewer_custom_loads = client.get_edges( invitation='auai.org/UAI/2019/Conference/Program_Committee/-/Custom_Load') assert not reviewer_custom_loads ac_custom_loads = client.get_edges( invitation='auai.org/UAI/2019/Conference/Senior_Program_Committee/-/Custom_Load') assert not ac_custom_loads reviewer_conflicts = client.get_edges( invitation='auai.org/UAI/2019/Conference/Program_Committee/-/Conflict') assert 1 == len(reviewer_conflicts) ac_conflicts = client.get_edges( invitation='auai.org/UAI/2019/Conference/Senior_Program_Committee/-/Conflict') assert 2 == len(ac_conflicts) ac1_conflicts = client.get_edges( invitation='auai.org/UAI/2019/Conference/Senior_Program_Committee/-/Conflict', tail='~AreaChair_One1') assert ac1_conflicts assert len(ac1_conflicts) assert ac1_conflicts[0].label == 'Institutional (level 1)' r1_conflicts = client.get_edges( invitation='auai.org/UAI/2019/Conference/Program_Committee/-/Conflict', tail='~Reviewer_One1') assert r1_conflicts assert len(r1_conflicts) assert r1_conflicts[0].label == 'Institutional (level 1)' ac2_conflicts = client.get_edges( invitation='auai.org/UAI/2019/Conference/Senior_Program_Committee/-/Conflict', tail='*****@*****.**') assert ac2_conflicts assert len(ac2_conflicts) assert ac2_conflicts[0].label == 'Institutional (level 1)'
confirmation = confirmations[i] if not custom_load and (i in reduced_loads): custom_load = reduced_loads[i] emergency_review_count = int(confirmation.content.get('emergency_review_count', '0')) if confirmation else 0 if custom_load: review_capacity = int(custom_load.content['reviewer_load']) - emergency_review_count else: review_capacity = 7 - emergency_review_count if review_capacity != 7: edge = openreview.Edge( head='thecvf.com/ECCV/2020/Conference/Reviewers', tail=profile.id, invitation='thecvf.com/ECCV/2020/Conference/Reviewers/-/Custom_Load', readers=[ 'thecvf.com/ECCV/2020/Conference', 'thecvf.com/ECCV/2020/Conference/Area_Chairs', profile.id], writers=['thecvf.com/ECCV/2020/Conference'], signatures=['thecvf.com/ECCV/2020/Conference'], weight=review_capacity ) custom_load_edges.append(edge) print ('Posting {0} edges'.format(len(custom_load_edges))) posted_edges = openreview.tools.post_bulk_edges(client, custom_load_edges) print ('Posted {0} edges'.format(len(posted_edges))) print ('Users with no profiles:', users_without_profiles)
custom_load = reduced_loads[i] if not max3_accepted and (i in max3_accepted_users): max3_accepted = max3_accepted_users[i] emergency_review_count = int( confirmation.content.get( 'emergency_review_count')) if confirmation else 0 if custom_load: review_capacity = int(custom_load) - emergency_review_count edge = openreview.Edge( head='ICLR.cc/2021/Conference/Reviewers', tail=profile.id, invitation= 'ICLR.cc/2021/Conference/Reviewers/-/Max_Load_Papers', readers=[ 'ICLR.cc/2021/Conference', 'ICLR.cc/2021/Conference/Area_Chairs', profile.id ], writers=['ICLR.cc/2021/Conference'], signatures=['ICLR.cc/2021/Conference'], weight=int(custom_load)) max_load_edges.append(edge) elif max3_accepted: review_capacity = 3 - emergency_review_count edge = openreview.Edge( head='ICLR.cc/2021/Conference/Reviewers', tail=profile.id, invitation= 'ICLR.cc/2021/Conference/Reviewers/-/Max_Load_Papers', readers=[ 'ICLR.cc/2021/Conference',
def test_redeploy_assigments(self, conference, client, test_client, helpers): pc_client = openreview.Client(username='******', password='******') blinded_notes = list(conference.get_submissions()) #Reviewer assignments pc_client.post_edge( openreview.Edge(invitation=conference.get_paper_assignment_id( conference.get_reviewers_id()), readers=[conference.id, '*****@*****.**'], writers=[conference.id], signatures=[conference.id], head=blinded_notes[0].id, tail='*****@*****.**', label='rev-matching-new', weight=0.98)) pc_client.post_edge( openreview.Edge(invitation=conference.get_paper_assignment_id( conference.get_reviewers_id()), readers=[conference.id, '*****@*****.**'], writers=[conference.id], signatures=[conference.id], head=blinded_notes[1].id, tail='*****@*****.**', label='rev-matching-new', weight=0.98)) pc_client.post_edge( openreview.Edge(invitation=conference.get_paper_assignment_id( conference.get_reviewers_id()), readers=[conference.id, '*****@*****.**'], writers=[conference.id], signatures=[conference.id], head=blinded_notes[2].id, tail='*****@*****.**', label='rev-matching-new', weight=0.98)) edges = client.get_edges( invitation= 'auai.org/UAI/2019/Conference/Program_Committee/-/Paper_Assignment', label='rev-matching-new') assert 3 == len(edges) conference.set_assignments(assignment_title='rev-matching-new', overwrite=True) revs_paper0 = client.get_group(conference.get_id() + '/Paper{x}/Reviewers'.format( x=blinded_notes[0].number)) assert ['*****@*****.**'] == revs_paper0.members revs_paper1 = client.get_group(conference.get_id() + '/Paper{x}/Reviewers'.format( x=blinded_notes[1].number)) assert ['~Reviewer_One1'] == revs_paper1.members revs_paper2 = client.get_group(conference.get_id() + '/Paper{x}/Reviewers'.format( x=blinded_notes[2].number)) assert ['*****@*****.**'] == revs_paper2.members now = datetime.datetime.now() conference.set_review_stage(openreview.ReviewStage(start_date=now)) invitation = client.get_invitation( id='auai.org/UAI/2019/Conference/-/Official_Review') assert invitation with pytest.raises(openreview.OpenReviewException, match=r'Review stage has started.'): conference.set_assignments(assignment_title='rev-matching-new2', overwrite=True)
def test_setup_matching_with_recommendations(self, conference, client, test_client, helpers): blinded_notes = list(conference.get_submissions()) ## Open reviewer recommendations now = datetime.datetime.utcnow() conference.open_recommendations(assignment_title='', due_date=now + datetime.timedelta(minutes=40)) ## Recommend reviewers ac1_client = helpers.get_user('*****@*****.**') ac1_client.post_edge( openreview.Edge( invitation=conference.get_recommendation_id(), readers=['auai.org/UAI/2019/Conference', '~AreaChair_One1'], signatures=['~AreaChair_One1'], writers=['~AreaChair_One1'], head=blinded_notes[0].id, tail='~Reviewer_One1', weight=1)) ac1_client.post_edge( openreview.Edge( invitation=conference.get_recommendation_id(), readers=['auai.org/UAI/2019/Conference', '~AreaChair_One1'], signatures=['~AreaChair_One1'], writers=['~AreaChair_One1'], head=blinded_notes[1].id, tail='*****@*****.**', weight=2)) ac1_client.post_edge( openreview.Edge( invitation=conference.get_recommendation_id(), readers=['auai.org/UAI/2019/Conference', '~AreaChair_One1'], signatures=['~AreaChair_One1'], writers=['~AreaChair_One1'], head=blinded_notes[1].id, tail='*****@*****.**', weight=3)) # Set up reviewer matching conference.setup_matching(tpms_score_file=os.path.join( os.path.dirname(__file__), 'data/reviewer_tpms_scores.csv'), build_conflicts=True) print(conference.get_reviewers_id()) invitation = client.get_invitation( id= 'auai.org/UAI/2019/Conference/Program_Committee/-/Assignment_Configuration' ) assert invitation assert 'scores_specification' in invitation.reply['content'] assert 'auai.org/UAI/2019/Conference/Program_Committee/-/Bid' in invitation.reply[ 'content']['scores_specification']['default'] assert 'auai.org/UAI/2019/Conference/Program_Committee/-/TPMS_Score' in invitation.reply[ 'content']['scores_specification']['default'] assert 'auai.org/UAI/2019/Conference/Program_Committee/-/Subject_Areas_Score' in invitation.reply[ 'content']['scores_specification']['default'] assert client.get_invitation( id= 'auai.org/UAI/2019/Conference/Program_Committee/-/Custom_Max_Papers' ) assert client.get_invitation( id='auai.org/UAI/2019/Conference/Program_Committee/-/Conflict') # Set up ac matching conference.setup_matching(is_area_chair=True, tpms_score_file=os.path.join( os.path.dirname(__file__), 'data/ac_tpms_scores.csv'), build_conflicts=True) invitation = client.get_invitation( id= 'auai.org/UAI/2019/Conference/Senior_Program_Committee/-/Assignment_Configuration' ) assert invitation assert 'scores_specification' in invitation.reply['content'] assert 'auai.org/UAI/2019/Conference/Senior_Program_Committee/-/Bid' in invitation.reply[ 'content']['scores_specification']['default'] assert 'auai.org/UAI/2019/Conference/Senior_Program_Committee/-/TPMS_Score' in invitation.reply[ 'content']['scores_specification']['default'] assert 'auai.org/UAI/2019/Conference/Senior_Program_Committee/-/Subject_Areas_Score' in invitation.reply[ 'content']['scores_specification']['default'] assert 'auai.org/UAI/2019/Conference/Program_Committee/-/Recommendation' in invitation.reply[ 'content']['scores_specification']['default'] assert client.get_invitation( id= 'auai.org/UAI/2019/Conference/Senior_Program_Committee/-/Custom_Max_Papers' ) assert client.get_invitation( id= 'auai.org/UAI/2019/Conference/Senior_Program_Committee/-/Conflict') bids = client.get_edges( invitation=conference.get_bid_id(conference.get_area_chairs_id())) assert bids assert 3 == len(bids) bids = client.get_edges( invitation=conference.get_bid_id(conference.get_reviewers_id())) assert bids assert 3 == len(bids) recommendations = client.get_edges( invitation=conference.get_recommendation_id()) assert recommendations assert 3 == len(recommendations) reviewer_custom_loads = client.get_edges( invitation= 'auai.org/UAI/2019/Conference/Program_Committee/-/Custom_Max_Papers' ) assert not reviewer_custom_loads ac_custom_loads = client.get_edges( invitation= 'auai.org/UAI/2019/Conference/Senior_Program_Committee/-/Custom_Max_Papers' ) assert not ac_custom_loads reviewer_conflicts = client.get_edges( invitation= 'auai.org/UAI/2019/Conference/Program_Committee/-/Conflict') assert 1 == len(reviewer_conflicts) ac_conflicts = client.get_edges( invitation= 'auai.org/UAI/2019/Conference/Senior_Program_Committee/-/Conflict') assert 2 == len(ac_conflicts) ac1_conflicts = client.get_edges( invitation= 'auai.org/UAI/2019/Conference/Senior_Program_Committee/-/Conflict', tail='~AreaChair_One1') assert ac1_conflicts assert len(ac1_conflicts) assert ac1_conflicts[0].label == 'Institutional (level 1)' r1_conflicts = client.get_edges( invitation= 'auai.org/UAI/2019/Conference/Program_Committee/-/Conflict', tail='~Reviewer_One1') assert r1_conflicts assert len(r1_conflicts) assert r1_conflicts[0].label == 'Institutional (level 1)' ac2_conflicts = client.get_edges( invitation= 'auai.org/UAI/2019/Conference/Senior_Program_Committee/-/Conflict', tail='*****@*****.**') assert ac2_conflicts assert len(ac2_conflicts) assert ac2_conflicts[0].label == 'Institutional (level 1)' submissions = conference.get_submissions() assert submissions assert 3 == len(submissions) reviewer_tpms_scores = client.get_edges( invitation= 'auai.org/UAI/2019/Conference/Program_Committee/-/TPMS_Score') assert 9 == len(reviewer_tpms_scores) ac_tpms_scores = client.get_edges( invitation= 'auai.org/UAI/2019/Conference/Senior_Program_Committee/-/TPMS_Score' ) assert 6 == len(ac_tpms_scores) r3_s0_tpms_scores = client.get_edges( invitation= 'auai.org/UAI/2019/Conference/Program_Committee/-/TPMS_Score', tail='*****@*****.**', head=submissions[0].id) assert r3_s0_tpms_scores assert 1 == len(r3_s0_tpms_scores) assert r3_s0_tpms_scores[0].weight == 0.21 r3_s1_tpms_scores = client.get_edges( invitation= 'auai.org/UAI/2019/Conference/Program_Committee/-/TPMS_Score', tail='*****@*****.**', head=submissions[1].id) assert r3_s1_tpms_scores assert 1 == len(r3_s1_tpms_scores) assert r3_s1_tpms_scores[0].weight == 0.31 r3_s2_tpms_scores = client.get_edges( invitation= 'auai.org/UAI/2019/Conference/Program_Committee/-/TPMS_Score', tail='*****@*****.**', head=submissions[2].id) assert r3_s2_tpms_scores assert 1 == len(r3_s2_tpms_scores) assert r3_s2_tpms_scores[0].weight == 0.51
secondary_ac_demands = [] total_review_demand = 0 print('Accumulating Custom_User_Demand edges') for paper_num, sub in tqdm(map_submissions.items()): weight = 0 if paper_num not in set_reject_papers: weight = 1 total_review_demand += 1 secondary_ac_demands.append( openreview.Edge( head=sub.id, tail='thecvf.com/ECCV/2020/Conference/Secondary_Area_Chairs', weight=weight, invitation= 'thecvf.com/ECCV/2020/Conference/Secondary_Area_Chairs/-/Custom_User_Demands', readers=[ 'thecvf.com/ECCV/2020/Conference', ], writers=['thecvf.com/ECCV/2020/Conference'], signatures=['thecvf.com/ECCV/2020/Conference'])) print('Custom_User_Demands: Posting {0} edges'.format( len(secondary_ac_demands))) posted_edges = openreview.tools.post_bulk_edges(client, secondary_ac_demands) print('Custom_User_Demands: Posted {0} edges\n'.format(len(posted_edges))) conflicts = [] print(
def test_set_assigments(self, conference, client, test_client, helpers): pc_client = helpers.create_user('*****@*****.**', 'TestPC', 'UAI') blinded_notes = list(conference.get_submissions()) edges = client.get_edges( invitation= 'auai.org/UAI/2019/Conference/Senior_Program_Committee/-/Paper_Assignment', label='ac-matching') assert 0 == len(edges) #Reviewer assignments pc_client.post_edge( openreview.Edge(invitation=conference.get_paper_assignment_id( conference.get_reviewers_id()), readers=[conference.id, '*****@*****.**'], writers=[conference.id], signatures=[conference.id], head=blinded_notes[0].id, tail='*****@*****.**', label='rev-matching', weight=0.98)) pc_client.post_edge( openreview.Edge(invitation=conference.get_paper_assignment_id( conference.get_reviewers_id()), readers=[conference.id, '*****@*****.**'], writers=[conference.id], signatures=[conference.id], head=blinded_notes[0].id, tail='*****@*****.**', label='rev-matching', weight=0.87)) pc_client.post_edge( openreview.Edge(invitation=conference.get_paper_assignment_id( conference.get_reviewers_id()), readers=[conference.id, '*****@*****.**'], writers=[conference.id], signatures=[conference.id], head=blinded_notes[1].id, tail='*****@*****.**', label='rev-matching', weight=0.87)) pc_client.post_edge( openreview.Edge(invitation=conference.get_paper_assignment_id( conference.get_reviewers_id()), readers=[conference.id, '*****@*****.**'], writers=[conference.id], signatures=[conference.id], head=blinded_notes[1].id, tail='*****@*****.**', label='rev-matching', weight=0.94)) pc_client.post_edge( openreview.Edge(invitation=conference.get_paper_assignment_id( conference.get_reviewers_id()), readers=[conference.id, '*****@*****.**'], writers=[conference.id], signatures=[conference.id], head=blinded_notes[2].id, tail='*****@*****.**', label='rev-matching', weight=0.94)) pc_client.post_edge( openreview.Edge(invitation=conference.get_paper_assignment_id( conference.get_reviewers_id()), readers=[conference.id, '*****@*****.**'], writers=[conference.id], signatures=[conference.id], head=blinded_notes[2].id, tail='*****@*****.**', label='rev-matching', weight=0.98)) edges = client.get_edges( invitation= 'auai.org/UAI/2019/Conference/Program_Committee/-/Paper_Assignment', label='rev-matching') assert 6 == len(edges) conference.set_assignments(assignment_title='rev-matching') revs_paper0 = client.get_group(conference.get_id() + '/Paper{x}/Reviewers'.format( x=blinded_notes[0].number)) assert 2 == len(revs_paper0.members) revs_paper1 = client.get_group(conference.get_id() + '/Paper{x}/Reviewers'.format( x=blinded_notes[1].number)) assert 2 == len(revs_paper1.members) revs_paper2 = client.get_group(conference.get_id() + '/Paper{x}/Reviewers'.format( x=blinded_notes[2].number)) assert 2 == len(revs_paper2.members)
def test_setup_matching(self, conference, client, test_client, helpers): ## Set committee conference.set_program_chairs(['*****@*****.**', '*****@*****.**']) conference.set_area_chairs(['*****@*****.**', '*****@*****.**']) conference.set_reviewers(['*****@*****.**', '*****@*****.**', '*****@*****.**']) ## Paper 1 note_1 = openreview.Note( invitation=conference.get_submission_id(), readers=['~Test_User1', '*****@*****.**', '*****@*****.**'], writers=[ conference.id, '~Test_User1', '*****@*****.**', '*****@*****.**' ], signatures=['~Test_User1'], content={ 'title': 'Paper title 1', 'abstract': 'This is an abstract', 'authorids': ['*****@*****.**', '*****@*****.**'], 'authors': ['Test User', 'Author 1'], 'subject_areas': [ 'Algorithms: Approximate Inference', 'Algorithms: Belief Propagation' ] }) url = test_client.put_attachment( os.path.join(os.path.dirname(__file__), 'data/paper.pdf'), conference.get_submission_id(), 'pdf') note_1.content['pdf'] = url note_1 = test_client.post_note(note_1) ## Paper 2 note_2 = openreview.Note( invitation=conference.get_submission_id(), readers=['~Test_User1', '*****@*****.**', '*****@*****.**'], writers=[ conference.id, '~Test_User1', '*****@*****.**', '*****@*****.**' ], signatures=['~Test_User1'], content={ 'title': 'Paper title 2', 'abstract': 'This is an abstract', 'authorids': ['*****@*****.**', '*****@*****.**'], 'authors': ['Test User', 'Author 2'], 'subject_areas': [ 'Algorithms: Approximate Inference', 'Algorithms: Exact Inference' ] }) url = test_client.put_attachment( os.path.join(os.path.dirname(__file__), 'data/paper.pdf'), conference.get_submission_id(), 'pdf') note_2.content['pdf'] = url note_2 = test_client.post_note(note_2) ## Paper 3 note_3 = openreview.Note( invitation=conference.get_submission_id(), readers=['~Test_User1', '*****@*****.**', '*****@*****.**'], writers=[ conference.id, '~Test_User1', '*****@*****.**', '*****@*****.**' ], signatures=['~Test_User1'], content={ 'title': 'Paper title 3', 'abstract': 'This is an abstract', 'authorids': ['*****@*****.**', '*****@*****.**'], 'authors': ['Test User', 'Author 3'], 'subject_areas': [ 'Algorithms: Distributed and Parallel', 'Algorithms: Exact Inference' ] }) url = test_client.put_attachment( os.path.join(os.path.dirname(__file__), 'data/paper.pdf'), conference.get_submission_id(), 'pdf') note_3.content['pdf'] = url note_3 = test_client.post_note(note_3) ## Create blind submissions conference.setup_post_submission_stage(force=True) blinded_notes = conference.get_submissions() ac1_client = helpers.create_user('*****@*****.**', 'AreaChair', 'One') ac1_client.post_edge( openreview.Edge( invitation=conference.get_bid_id( conference.get_area_chairs_id()), readers=['auai.org/UAI/2019/Conference', '~AreaChair_One1'], writers=['auai.org/UAI/2019/Conference', '~AreaChair_One1'], signatures=['~AreaChair_One1'], head=blinded_notes[0].id, tail='~AreaChair_One1', label='High')) ac1_client.post_edge( openreview.Edge( invitation=conference.get_bid_id( conference.get_area_chairs_id()), readers=['auai.org/UAI/2019/Conference', '~AreaChair_One1'], writers=['auai.org/UAI/2019/Conference', '~AreaChair_One1'], signatures=['~AreaChair_One1'], head=blinded_notes[1].id, tail='~AreaChair_One1', label='Low')) ac1_client.post_edge( openreview.Edge( invitation=conference.get_bid_id( conference.get_area_chairs_id()), readers=['auai.org/UAI/2019/Conference', '~AreaChair_One1'], writers=['auai.org/UAI/2019/Conference', '~AreaChair_One1'], signatures=['~AreaChair_One1'], head=blinded_notes[2].id, tail='~AreaChair_One1', label='Very Low')) r1_client = helpers.create_user('*****@*****.**', 'Reviewer', 'One') r1_client.post_edge( openreview.Edge( invitation=conference.get_bid_id( conference.get_reviewers_id()), readers=['auai.org/UAI/2019/Conference', '~Reviewer_One1'], writers=['auai.org/UAI/2019/Conference', '~Reviewer_One1'], signatures=['~Reviewer_One1'], head=blinded_notes[0].id, tail='~Reviewer_One1', label='Neutral')) r1_client.post_edge( openreview.Edge( invitation=conference.get_bid_id( conference.get_reviewers_id()), readers=['auai.org/UAI/2019/Conference', '~Reviewer_One1'], writers=['auai.org/UAI/2019/Conference', '~Reviewer_One1'], signatures=['~Reviewer_One1'], head=blinded_notes[1].id, tail='~Reviewer_One1', label='Very High')) r1_client.post_edge( openreview.Edge( invitation=conference.get_bid_id( conference.get_reviewers_id()), readers=['auai.org/UAI/2019/Conference', '~Reviewer_One1'], writers=['auai.org/UAI/2019/Conference', '~Reviewer_One1'], signatures=['~Reviewer_One1'], head=blinded_notes[2].id, tail='~Reviewer_One1', label='Low')) # Set up reviewer matching conference.setup_matching(build_conflicts=True) invitation = client.get_invitation( id= 'auai.org/UAI/2019/Conference/Program_Committee/-/Assignment_Configuration' ) assert invitation assert 'scores_specification' in invitation.reply['content'] assert 'auai.org/UAI/2019/Conference/Program_Committee/-/Bid' in invitation.reply[ 'content']['scores_specification']['default'] assert client.get_invitation( id= 'auai.org/UAI/2019/Conference/Program_Committee/-/Custom_Max_Papers' ) assert client.get_invitation( id='auai.org/UAI/2019/Conference/Program_Committee/-/Conflict') assert client.get_invitation( id= 'auai.org/UAI/2019/Conference/Program_Committee/-/Aggregate_Score') assert client.get_invitation( id= 'auai.org/UAI/2019/Conference/Program_Committee/-/Paper_Assignment' ) # Set up AC matching conference.setup_matching(is_area_chair=True, build_conflicts=True) invitation = client.get_invitation( id= 'auai.org/UAI/2019/Conference/Senior_Program_Committee/-/Assignment_Configuration' ) assert invitation assert 'scores_specification' in invitation.reply['content'] assert 'auai.org/UAI/2019/Conference/Senior_Program_Committee/-/Bid' in invitation.reply[ 'content']['scores_specification']['default'] assert client.get_invitation( id= 'auai.org/UAI/2019/Conference/Senior_Program_Committee/-/Custom_Max_Papers' ) assert client.get_invitation( id= 'auai.org/UAI/2019/Conference/Senior_Program_Committee/-/Conflict') assert client.get_invitation( id= 'auai.org/UAI/2019/Conference/Senior_Program_Committee/-/Aggregate_Score' ) assert client.get_invitation( id= 'auai.org/UAI/2019/Conference/Senior_Program_Committee/-/Paper_Assignment' ) bids = client.get_edges( invitation=conference.get_bid_id(conference.get_area_chairs_id())) assert bids assert 3 == len(bids) bids = client.get_edges( invitation=conference.get_bid_id(conference.get_reviewers_id())) assert bids assert 3 == len(bids) reviewer_custom_loads = client.get_edges( invitation= 'auai.org/UAI/2019/Conference/Program_Committee/-/Custom_Max_Papers' ) assert not reviewer_custom_loads ac_custom_loads = client.get_edges( invitation= 'auai.org/UAI/2019/Conference/Senior_Program_Committee/-/Custom_Max_Papers' ) assert not ac_custom_loads reviewer_conflicts = client.get_edges( invitation= 'auai.org/UAI/2019/Conference/Program_Committee/-/Conflict') assert 1 == len(reviewer_conflicts) ac_conflicts = client.get_edges( invitation= 'auai.org/UAI/2019/Conference/Senior_Program_Committee/-/Conflict') assert 2 == len(ac_conflicts) ac1_conflicts = client.get_edges( invitation= 'auai.org/UAI/2019/Conference/Senior_Program_Committee/-/Conflict', tail='~AreaChair_One1') assert ac1_conflicts assert len(ac1_conflicts) assert ac1_conflicts[0].label == 'Institutional (level 1)' r1_conflicts = client.get_edges( invitation= 'auai.org/UAI/2019/Conference/Program_Committee/-/Conflict', tail='~Reviewer_One1') assert r1_conflicts assert len(r1_conflicts) assert r1_conflicts[0].label == 'Institutional (level 1)' ac2_conflicts = client.get_edges( invitation= 'auai.org/UAI/2019/Conference/Senior_Program_Committee/-/Conflict', tail='*****@*****.**') assert ac2_conflicts assert len(ac2_conflicts) assert ac2_conflicts[0].label == 'Institutional (level 1)'