def test_open_decisions(self, client, conference, helpers): conference.open_decisions() pc_client = openreview.Client(username = '******', password = '******') notes = pc_client.get_notes(invitation='icaps-conference.org/ICAPS/2019/Workshop/HSDIP/-/Blind_Submission') assert len(notes) == 3 submission = notes[2] note = openreview.Note(invitation = 'icaps-conference.org/ICAPS/2019/Workshop/HSDIP/Paper1/-/Decision', forum = submission.id, replyto = submission.id, readers = ['icaps-conference.org/ICAPS/2019/Workshop/HSDIP/Program_Chairs'], nonreaders = ['icaps-conference.org/ICAPS/2019/Workshop/HSDIP/Paper1/Authors'], writers = ['icaps-conference.org/ICAPS/2019/Workshop/HSDIP/Program_Chairs'], signatures = ['icaps-conference.org/ICAPS/2019/Workshop/HSDIP/Program_Chairs'], content = { 'title': 'Paper Decision', 'decision': 'Accept (Oral)', 'comment': 'this is a comment' } ) decision_note = pc_client.post_note(note) note = openreview.Note(invitation = 'icaps-conference.org/ICAPS/2019/Workshop/HSDIP/Paper2/-/Decision', forum = notes[1].id, replyto = notes[1].id, readers = ['icaps-conference.org/ICAPS/2019/Workshop/HSDIP/Program_Chairs'], nonreaders = ['icaps-conference.org/ICAPS/2019/Workshop/HSDIP/Paper2/Authors'], writers = ['icaps-conference.org/ICAPS/2019/Workshop/HSDIP/Program_Chairs'], signatures = ['icaps-conference.org/ICAPS/2019/Workshop/HSDIP/Program_Chairs'], content = { 'title': 'Paper Decision', 'decision': 'Accept (Poster)', 'comment': 'this is a comment' } ) decision_note = pc_client.post_note(note) note = openreview.Note(invitation = 'icaps-conference.org/ICAPS/2019/Workshop/HSDIP/Paper3/-/Decision', forum = notes[0].id, replyto = notes[0].id, readers = ['icaps-conference.org/ICAPS/2019/Workshop/HSDIP/Program_Chairs'], nonreaders = ['icaps-conference.org/ICAPS/2019/Workshop/HSDIP/Paper3/Authors'], writers = ['icaps-conference.org/ICAPS/2019/Workshop/HSDIP/Program_Chairs'], signatures = ['icaps-conference.org/ICAPS/2019/Workshop/HSDIP/Program_Chairs'], content = { 'title': 'Paper Decision', 'decision': 'Reject', 'comment': 'this is a comment' } ) decision_note = pc_client.post_note(note) notes = conference.get_submissions(accepted=True) assert len(notes) == 2 notes = conference.get_submissions(accepted=False) assert len(notes) == 3
def test_desk_reject(self, client, helpers): author_client = openreview.Client(username = '******', password = '******') note = openreview.Note(invitation = '-Agora/COVID-19/-/Submission', readers = ['openreview.net/Support', '-Agora/COVID-19/Editors', '~Author_One1'], writers = ['openreview.net/Support', '-Agora/COVID-19/Editors', '~Author_One1'], signatures = ['~Author_One1'], content = { 'title': 'Paper title 2', 'abstract': 'This is an abstract 2', 'authorids': ['~Author_One1'], 'authors': ['Author One'], 'pdf': 'https://openreview.net', 'requested_editors': ['*****@*****.**'] } ) posted_note = author_client.post_note(note) time.sleep(2) process_logs = client.get_process_logs(id = posted_note.id) assert len(process_logs) == 1 assert process_logs[0]['status'] == 'ok' editor_client = openreview.Client(username='******', password='******') submissions = editor_client.get_notes(invitation='-Agora/COVID-19/-/Submission') assert submissions note = openreview.Note(invitation = '-Agora/COVID-19/Submission2/-/Moderation', readers = ['openreview.net/Support', '-Agora/COVID-19/Editors', '-Agora/COVID-19/Submission2/Authors'], writers = ['openreview.net/Support'], signatures = ['~Editor_One1'], forum = submissions[0].id, replyto = submissions[0].id, content = { 'resolution': 'Desk-Reject', 'comment': 'This is not good' } ) posted_note = editor_client.post_note(note) time.sleep(2) process_logs = client.get_process_logs(id = posted_note.id) assert len(process_logs) == 1 assert process_logs[0]['status'] == 'ok' assert not editor_client.get_notes(invitation='-Agora/COVID-19/-/Submission') assert len(editor_client.get_notes(invitation='-Agora/COVID-19/-/Article')) == 1 assert len(editor_client.get_notes(invitation='-Agora/COVID-19/-/Desk_Rejected')) == 1 messages = client.get_messages(subject = '[Agora/COVID-19] Your submission has been rejected') assert len(messages) == 1 recipients = [m['content']['to'] for m in messages] assert '*****@*****.**' in recipients assert messages[0]['content']['text'] == 'Unfortunately your submission has been desk-rejected by the Editor-in-Chief of this venue.\n\nFor more information, see their comment on the OpenReview submission forum here: https://openreview.net/forum?id=' + submissions[0].id + '¬eId=' + posted_note.id
def test_decisions(self, conference, helpers, test_client, client): conference.set_decision_stage(openreview.DecisionStage()) submissions = conference.get_submissions() assert len(submissions) == 5 client.post_note( openreview.Note( invitation='MICCAI.org/2021/Challenges/Paper5/-/Decision', forum=submissions[0].id, replyto=submissions[0].id, readers=[ 'MICCAI.org/2021/Challenges/Program_Chairs', 'MICCAI.org/2021/Challenges/Paper5/Area_Chairs' ], nonreaders=['MICCAI.org/2021/Challenges/Paper5/Authors'], writers=['MICCAI.org/2021/Challenges/Program_Chairs'], signatures=['MICCAI.org/2021/Challenges/Program_Chairs'], content={ 'title': 'Paper Decision', 'decision': 'Reject' })) client.post_note( openreview.Note( invitation='MICCAI.org/2021/Challenges/Paper4/-/Decision', forum=submissions[1].id, replyto=submissions[1].id, readers=[ 'MICCAI.org/2021/Challenges/Program_Chairs', 'MICCAI.org/2021/Challenges/Paper4/Area_Chairs' ], nonreaders=['MICCAI.org/2021/Challenges/Paper4/Authors'], writers=['MICCAI.org/2021/Challenges/Program_Chairs'], signatures=['MICCAI.org/2021/Challenges/Program_Chairs'], content={ 'title': 'Paper Decision', 'decision': 'Accept (Oral)' })) client.post_note( openreview.Note( invitation='MICCAI.org/2021/Challenges/Paper3/-/Decision', forum=submissions[2].id, replyto=submissions[2].id, readers=[ 'MICCAI.org/2021/Challenges/Program_Chairs', 'MICCAI.org/2021/Challenges/Paper3/Area_Chairs' ], nonreaders=['MICCAI.org/2021/Challenges/Paper3/Authors'], writers=['MICCAI.org/2021/Challenges/Program_Chairs'], signatures=['MICCAI.org/2021/Challenges/Program_Chairs'], content={ 'title': 'Paper Decision', 'decision': 'Accept (Poster)' }))
def get_note(self, configNoteId): if configNoteId == 'nonExist': raise OpenReviewException([{'type': 'Not Found'}]) elif configNoteId == 'forbidden': raise OpenReviewException([{'type': 'Forbidden'}]) elif configNoteId == 'internal error': list = [] list[2] elif configNoteId == 'already_running': note = openreview.Note(id='testId', invitation=None, readers=[], writers=[], signatures=[], content={ 'title': 'foo', 'metadata_invitation': 1, 'match_group': 1, 'paper_invitation': 1, 'assignment_invitation': 1, 'max_papers': 1, 'max_users': 1, 'min_papers': 1, 'weights': 1, 'constraints': 1, 'scores_names': [], 'scores_weights': [], 'alternates': 3, 'status': Configuration.STATUS_RUNNING }) return note else: note = openreview.Note(id='testId', invitation=None, readers=[], writers=[], signatures=[], content={ 'title': 'foo', 'metadata_invitation': 1, 'match_group': 1, 'paper_invitation': 1, 'assignment_invitation': 1, 'max_papers': 1, 'max_users': 1, 'min_papers': 1, 'weights': 1, 'constraints': 1, 'scores_names': [], 'scores_weights': [], 'alternates': 3, 'status': Configuration.STATUS_INITIALIZED }) return note
def upload_submissions(client): # repost 20 papers from AKBC 2019 # TODO: Change this to read from ICML papers for paper in client.get_notes( invitation='AKBC.ws/2019/Conference/-/Submission'): new_content = {key: value for key, value in paper.content.items() \ if key in icml.submission_inv.reply['content'].keys()} new_writers = [w.replace('AKBC.ws', 'ICML.cc') for w in paper.writers] posted_submission = client.post_note( openreview.Note( **{ 'writers': new_writers, 'readers': [icml.CONFERENCE_ID], 'content': new_content, 'invitation': icml.SUBMISSION_ID, 'signatures': [icml.CONFERENCE_ID] })) papergroup = client.post_group( openreview.Group.from_json({ 'id': 'ICML.cc/2019/Conference/Paper{}'.format( posted_submission.number), 'readers': ['everyone'], 'writers': [icml.CONFERENCE_ID], 'signatures': [icml.CONFERENCE_ID], 'signatories': [], 'members': [] })) posted_mask = client.post_note( openreview.Note( **{ 'original': posted_submission.id, 'forum': None, 'replyto': None, 'readers': ['everyone'], 'writers': [icml.CONFERENCE_ID], 'signatures': [icml.CONFERENCE_ID], 'invitation': icml.blind_submission_inv.id, 'content': { 'title': 'Anonymous Submission', 'abstract': 'Redacted', 'authorids': ['Anonymous'], 'authors': ['Anonymous'] } }))
def get_color_paper(color): author_signature = "~Jane_Doe1" author_id = "*****@*****.**" note_readers = [ "ICLR.cc/2018/Conference", "ICLR.cc/2018/Conference/Program_Chairs", author_id, author_signature ] note_signatures = [author_signature] return openreview.Note( **{ "readers": note_readers, "nonreaders": [], "signatures": note_signatures, "writers": [], "invitation": "ICLR.cc/2018/Conference/-/Submission", "content": { 'title': '{0} Paper'.format(color), 'abstract': 'This is the {0} abstract.'.format(color), 'authors': '{0} authors'.format(color), 'keywords': color, 'TL;DR': 'This is the {0} TL;DR'.format(color), 'authorids': [author_id], 'pdf': "/pdf/813abccc0d09f502de96bbdca811a42e5ac59a2a.pdf" } })
def post_metadata_note(client, note, reviewer_profiles, metadata_inv, paper_tpms_scores, manual_conflicts_by_id): author_group = note.content['authorids'] authorids = note.details['original']['content']['authorids'] paper_bid_jsons = note.details['tags'] paper_author_profiles = client.get_profiles(authorids) print('Paper', note.id) entries = _build_entries(paper_author_profiles, reviewer_profiles, paper_bid_jsons, paper_tpms_scores, manual_conflicts_by_id) new_metadata_note = openreview.Note( **{ 'forum': note.id, 'invitation': metadata_inv.id, 'readers': metadata_inv.reply['readers']['values'], 'writers': metadata_inv.reply['writers']['values'], 'signatures': metadata_inv.reply['signatures']['values'], 'content': { 'title': 'Paper Metadata', 'entries': entries } }) return client.post_note(new_metadata_note)
def test_open_reviews(self, client, conference, test_client, selenium, request_page, helpers): conference.set_reviewers(emails = ['*****@*****.**']) notes = test_client.get_notes(invitation='icaps-conference.org/ICAPS/2019/Workshop/HSDIP/-/Blind_Submission') submission = notes[2] # Reviewer reviewer_client = helpers.create_user('*****@*****.**', 'Reviewer', 'Four') conference.set_assignment('*****@*****.**', submission.number) request_page(selenium, "http://localhost:3030/forum?id=" + submission.id, reviewer_client.token) reply_row = selenium.find_element_by_class_name('reply_row') assert len(reply_row.find_elements_by_class_name('btn')) == 1 assert 'Official Review' == reply_row.find_elements_by_class_name('btn')[0].text # Author request_page(selenium, "http://localhost:3030/forum?id=" + submission.id, test_client.token) reply_row = selenium.find_element_by_class_name('reply_row') assert len(reply_row.find_elements_by_class_name('btn')) == 1 assert 'Withdraw' == reply_row.find_elements_by_class_name('btn')[0].text note = openreview.Note(invitation = 'icaps-conference.org/ICAPS/2019/Workshop/HSDIP/Paper1/-/Official_Review', forum = submission.id, replyto = submission.id, readers = ['icaps-conference.org/ICAPS/2019/Workshop/HSDIP/Program_Chairs', 'icaps-conference.org/ICAPS/2019/Workshop/HSDIP/Paper1/Reviewers', 'icaps-conference.org/ICAPS/2019/Workshop/HSDIP/Paper1/Authors'], writers = ['icaps-conference.org/ICAPS/2019/Workshop/HSDIP/Paper1/AnonReviewer1'], signatures = ['icaps-conference.org/ICAPS/2019/Workshop/HSDIP/Paper1/AnonReviewer1'], content = { 'title': 'Review title', 'review': 'Paper is very good!', 'rating': '9: Top 15% of accepted papers, strong accept', 'confidence': '4: The reviewer is confident but not absolutely certain that the evaluation is correct' } ) review_note = reviewer_client.post_note(note) assert review_note helpers.await_queue() process_logs = client.get_process_logs(id = review_note.id) assert len(process_logs) == 1 assert process_logs[0]['status'] == 'ok' messages = client.get_messages(subject = '[ICAPS HSDIP 2019] Review posted to your submission: "New paper title"') assert len(messages) == 0 messages = client.get_messages(subject = '[ICAPS HSDIP 2019] Review posted to your assigned paper: "New paper title"') assert len(messages) == 0 ## Check review visibility notes = reviewer_client.get_notes(invitation='icaps-conference.org/ICAPS/2019/Workshop/HSDIP/Paper1/-/Official_Review') assert len(notes) == 1 notes = test_client.get_notes(invitation='icaps-conference.org/ICAPS/2019/Workshop/HSDIP/Paper1/-/Official_Review') assert len(notes) == 1
def get_iclr_paper(iclr17_note): author_signature = "~Jane_Doe1" author_id = "*****@*****.**" note_readers = [ "ICLR.cc/2018/Conference", "ICLR.cc/2018/Conference/Program_Chairs", author_id, author_signature ] note_signatures = [author_signature] return openreview.Note( **{ "readers": note_readers, "nonreaders": [], "signatures": note_signatures, "writers": [], "invitation": "ICLR.cc/2018/Conference/-/Submission", "content": { 'title': iclr17_note.content['title'], 'abstract': iclr17_note.content['abstract'], 'authors': iclr17_note.content['authors'], 'keywords': iclr17_note.content['keywords'], 'TL;DR': iclr17_note.content['TL;DR'], 'authorids': [author_id], 'pdf': iclr17_note.content['pdf'] } })
def test_submit_papers(self, conference, helpers, test_client, client): pdf_url = test_client.put_attachment( os.path.join(os.path.dirname(__file__), 'data/paper.pdf'), 'MICCAI.org/2021/Challenges/-/Submission', 'pdf') domains = [ 'umass.edu', 'umass.edu', 'fb.com', 'umass.edu', 'google.com', 'mit.edu' ] for i in range(1, 6): note = openreview.Note( invitation='MICCAI.org/2021/Challenges/-/Submission', readers=[ 'MICCAI.org/2021/Challenges', '*****@*****.**', '*****@*****.**', 'andrew@' + domains[i], '~Test_User1' ], writers=[ conference.id, '~Test_User1', '*****@*****.**', 'andrew@' + domains[i] ], signatures=['~Test_User1'], content={ 'title': 'Paper title ' + str(i), 'abstract': 'This is an abstract ' + str(i), 'authorids': [ '*****@*****.**', '*****@*****.**', 'andrew@' + domains[i] ], 'authors': ['Test User', 'Peter Test', 'Andrew Mc'], 'pdf': pdf_url }) note = test_client.post_note(note) conference.setup_post_submission_stage(force=True) notes = test_client.get_notes( invitation='MICCAI.org/2021/Challenges/-/Submission') assert len(notes) == 5 assert notes[0].readers == [ 'MICCAI.org/2021/Challenges', 'MICCAI.org/2021/Challenges/Area_Chairs', 'MICCAI.org/2021/Challenges/Paper5/Reviewers', 'MICCAI.org/2021/Challenges/Paper5/Authors' ] invitations = test_client.get_invitations(replyForum=notes[0].id) assert len(invitations) == 1 assert invitations[ 0].id == 'MICCAI.org/2021/Challenges/Paper5/-/Withdraw' invitations = client.get_invitations(replyForum=notes[0].id) assert len(invitations) == 2 invitation_ids = [invitation.id for invitation in invitations] assert 'MICCAI.org/2021/Challenges/Paper5/-/Desk_Reject' in invitation_ids assert 'MICCAI.org/2021/Challenges/Paper5/-/Withdraw' in invitation_ids
def test_add_revision(self, client, helpers): author_client = openreview.Client(username='******', password='******') articles = author_client.get_notes( invitation='-Agora/COVID-19/-/Article') assert articles note = openreview.Note( invitation='-Agora/COVID-19/Article1/-/Revision', readers=['everyone'], writers=[ 'openreview.net/Support', '-Agora/COVID-19/Article1/Authors' ], signatures=['-Agora/COVID-19/Article1/Authors'], forum=articles[0].forum, referent=articles[0].id, content={ 'title': 'Paper title', 'abstract': 'This is an abstract', 'authorids': ['~Author_One1'], 'authors': ['Author One'], 'pdf': 'https://openreview.net', 'requested_editors': [ '*****@*****.**', '*****@*****.**' ] }) posted_note = author_client.post_note(note) helpers.await_queue() process_logs = client.get_process_logs(id=posted_note.id) assert len(process_logs) == 1 assert process_logs[0]['status'] == 'ok' messages = client.get_messages( subject= '[Agora/COVID-19] Your revision has been posted on your article titled "Paper title"' ) assert len(messages) == 1 recipients = [m['content']['to'] for m in messages] assert '*****@*****.**' in recipients messages = client.get_messages( subject= '[Agora/COVID-19] A revision has been posted on the article titled "Paper title"' ) assert len(messages) == 3 recipients = [m['content']['to'] for m in messages] assert '*****@*****.**' in recipients assert '*****@*****.**' in recipients assert '*****@*****.**' in recipients
def release_author_names(client): print('Release author names...') meta_reviews = list( openreview.tools.iterget_notes( client, invitation='ICLR.cc/2019/Conference/-/Paper.*/Meta_Review')) submissions = list( openreview.tools.iterget_notes( client, invitation='ICLR.cc/2019/Conference/-/Blind_Submission', details='original')) submissions_dict = {} for s in submissions: submissions_dict[s.id] = s for m in meta_reviews: if m.forum in submissions_dict: submission = submissions_dict[m.forum] decision = m.content['recommendation'] original_note = openreview.Note.from_json( submission.details['original']) if 'Reject' in decision: accepted = False if 'Oral' in decision: accepted = True if 'Poster' in decision: accepted = True overwriting_note = openreview.Note( id=submission.id, original=submission.original, invitation=iclr19.BLIND_SUBMISSION_ID, forum=submission.forum, signatures=[iclr19.CONFERENCE_ID], writers=[iclr19.CONFERENCE_ID], readers=['everyone'], content={ '_bibtex': openreview.tools.get_bibtex( original_note, 'International Conference on Learning Representations', '2019', url_forum=submission.forum, accepted=accepted, anonymous=False) }) client.post_note(overwriting_note) if accepted: revision_invitation = client.get_invitation( id='{conference_id}/-/Paper{number}/Revision'.format( conference_id=iclr19.CONFERENCE_ID, number=submission.number)) revision_invitation.expdate = 1576843200000 client.post_invitation(revision_invitation)
def dump_cached_or(entries, out_pickle): cached_or = {} for entry in entries: cached_or[entry[0]] = openreview.Note( # id "", [], [], [], {"abstract": entry[3], "title": entry[1]} ) # Hack. ICLR Recommender script accepts Openreview notes with open(out_pickle, "wb") as fd: pickle.dump(cached_or, fd)
def test_post_submission(self, client, conference, test_client): note = openreview.Note( invitation=conference.get_submission_id(), readers=['everyone'], writers=[ conference.id, '~Test_User1', '*****@*****.**', '*****@*****.**' ], signatures=['~Test_User1'], content={ 'title': 'Paper title', 'abstract': 'This is an abstract', 'authorids': ['*****@*****.**', '*****@*****.**', '*****@*****.**'], 'authors': ['Test User', 'Peter Test', 'Andrew Mc'] }) url = test_client.put_attachment( os.path.join(os.path.dirname(__file__), 'data/paper.pdf'), conference.get_submission_id(), 'pdf') note.content['pdf'] = url note = test_client.post_note(note) time.sleep(2) note = client.get_note(note.id) process_logs = client.get_process_logs(id=note.id) assert len(process_logs) == 1 assert process_logs[0]['status'] == 'ok' messages = client.get_messages( subject= 'NLP-COVID-2020 has received your submission titled Paper title') assert len(messages) == 3 recipients = [m['content']['to'] for m in messages] assert '*****@*****.**' in recipients assert '*****@*****.**' in recipients assert '*****@*****.**' in recipients messages = client.get_messages( subject= 'NLP-COVID-2020 has received a new submission titled Paper title') assert len(messages) == 1 recipients = [m['content']['to'] for m in messages] assert '*****@*****.**' in recipients assert client.get_group( 'aclweb.org/ACL/2020/Workshop/NLP-COVID/Paper1') assert client.get_group( 'aclweb.org/ACL/2020/Workshop/NLP-COVID/Paper1/Authors') assert client.get_group( 'aclweb.org/ACL/2020/Workshop/NLP-COVID/Paper1/Reviewers') assert client.get_group( 'aclweb.org/ACL/2020/Workshop/NLP-COVID/Paper1/Reviewers/Submitted' ) assert client.get_invitation( 'aclweb.org/ACL/2020/Workshop/NLP-COVID/Paper1/-/Official_Review')
def clean_start_conference(client, conference_id, num_reviewers, num_papers, reviews_per_paper): builder = openreview.conference.ConferenceBuilder(client) builder.set_conference_id(conference_id) builder.set_submission_stage( due_date=datetime.datetime(2019, 3, 25, 23, 59), remove_fields=['authors', 'abstract', 'pdf', 'keywords', 'TL;DR']) conference = builder.get_result() submission_invitation = client.get_invitation( conference.get_submission_id()) submission_invitation.reply['content']['authorids'] = { 'values-regex': '.*' } submission_invitation = client.post_invitation(submission_invitation) reviewers = [] # TODO: is there a better way to handle affinity scores? # Maybe conference.setup_matching() should allow a score matrix as input with open(AFFINITY_SCORE_FILE, 'w') as file_handle: for paper_number in range(num_papers): content = { 'title': 'Test_Paper_{}'.format(paper_number), 'authorids': [ 'testauthor{}{}@test.com'.format(paper_number, author_code) \ for author_code in ['A', 'B', 'C'] ] } submission = openreview.Note( **{ 'signatures': ['~Super_User1'], 'writers': [conference.id], 'readers': [conference.id], 'content': content, 'invitation': conference.get_submission_id() }) posted_submission = client.post_note(submission) for reviewer_number in range(num_reviewers): reviewer = '~Test_Reviewer{}'.format(reviewer_number) reviewers.append(reviewer) score = random.random() row = [ posted_submission.forum, reviewer, '{:.3f}'.format(score) ] file_handle.write(','.join(row) + '\n') conference.set_authors() conference.set_reviewers(emails=reviewers) conference.setup_matching(affinity_score_file=AFFINITY_SCORE_FILE) return conference
def test_registration(self, conference, helpers, selenium, request_page): reviewer_client = openreview.Client(username='******', password='******') reviewer_tasks_url = 'http://localhost:3030/group?id=ICLR.cc/2021/Conference/Reviewers#reviewer-tasks' request_page(selenium, reviewer_tasks_url, reviewer_client.token) assert selenium.find_element_by_link_text('Reviewer Registration') assert selenium.find_element_by_link_text('Expertise Selection') registration_notes = reviewer_client.get_notes( invitation='ICLR.cc/2021/Conference/Reviewers/-/Form') assert registration_notes assert len(registration_notes) == 1 registration_forum = registration_notes[0].forum registration_note = reviewer_client.post_note( openreview.Note( invitation='ICLR.cc/2021/Conference/Reviewers/-/Registration', forum=registration_forum, replyto=registration_forum, content={ 'profile_confirmed': 'Yes', 'expertise_confirmed': 'Yes', 'TPMS_registration_confirmed': 'Yes', 'reviewer_instructions_confirm': 'Yes', 'emergency_review_count': '0' }, signatures=['~ReviewerOne_ICLR1'], readers=[conference.get_id(), '~ReviewerOne_ICLR1'], writers=[conference.get_id(), '~ReviewerOne_ICLR1'])) assert registration_note request_page( selenium, 'http://localhost:3030/group?id=ICLR.cc/2021/Conference/Reviewers', reviewer_client.token) header = selenium.find_element_by_id('header') assert header notes = header.find_elements_by_class_name("description") assert notes assert len(notes) == 1 assert notes[ 0].text == 'This page provides information and status updates for the ICLR 2021. It will be regularly updated as the conference progresses, so please check back frequently.' request_page(selenium, reviewer_tasks_url, reviewer_client.token) assert selenium.find_element_by_link_text('Reviewer Registration') assert selenium.find_element_by_link_text('Expertise Selection') tasks = selenium.find_element_by_id('reviewer-tasks') assert tasks assert len(tasks.find_elements_by_class_name('note')) == 2 assert len(tasks.find_elements_by_class_name('completed')) == 2
def test_post_metareview(self, client, helpers): article_editor_client = openreview.Client( username='******', password='******') articles = article_editor_client.get_notes( invitation='-Agora/COVID-19/-/Article') assert articles note = openreview.Note( invitation='-Agora/COVID-19/Article1/-/Meta_Review', readers=['everyone'], writers=['openreview.net/Support', '~ArticleEditor_One1'], signatures=['~ArticleEditor_One1'], forum=articles[0].id, replyto=articles[0].id, content={ 'title': 'review title', 'metareview': 'excelent paper', 'recommendation': 'Accept' }) posted_note = article_editor_client.post_note(note) helpers.await_queue() process_logs = client.get_process_logs(id=posted_note.id) assert len(process_logs) == 1 assert process_logs[0]['status'] == 'ok' messages = client.get_messages( subject= '[Agora/COVID-19] Meta Review posted to your article titled "Paper title"' ) assert len(messages) == 1 recipients = [m['content']['to'] for m in messages] assert '*****@*****.**' in recipients messages = client.get_messages( subject= '[Agora/COVID-19] Your meta review has been posted on your assigned article titled "Paper title"' ) assert len(messages) == 1 recipients = [m['content']['to'] for m in messages] assert '*****@*****.**' in recipients messages = client.get_messages( subject= '[Agora/COVID-19] A meta review has been posted on the article titled "Paper title"' ) assert len(messages) == 2 recipients = [m['content']['to'] for m in messages] assert '*****@*****.**' in recipients assert '*****@*****.**' in recipients
def create_blind_note(note): return openreview.Note(original=note.id, invitation=conference_config.BLIND_SUBMISSION_ID, forum=None, signatures=[conference_config.CONFERENCE_ID], writers=[conference_config.CONFERENCE_ID], readers=['everyone'], content={ "authors": ['Anonymous'], "authorids": [], "_bibtex": None })
def test_post_review(self, client, helpers): reviewer_client = helpers.create_user(email='*****@*****.**', first='ArticleReviewer', last='One') articles = reviewer_client.get_notes( invitation='-Agora/COVID-19/-/Article') assert articles note = openreview.Note( invitation='-Agora/COVID-19/Article1/-/Review', readers=['everyone'], writers=['openreview.net/Support', '~ArticleReviewer_One1'], signatures=['~ArticleReviewer_One1'], forum=articles[0].id, replyto=articles[0].id, content={ 'title': 'review title', 'review': 'excelent paper' }) posted_note = reviewer_client.post_note(note) helpers.await_queue() process_logs = client.get_process_logs(id=posted_note.id) assert len(process_logs) == 1 assert process_logs[0]['status'] == 'ok' messages = client.get_messages( subject= '[Agora/COVID-19] Review posted to your article titled "Paper title"' ) assert len(messages) == 1 recipients = [m['content']['to'] for m in messages] assert '*****@*****.**' in recipients messages = client.get_messages( subject= '[Agora/COVID-19] Your review has been posted on your assigned article titled "Paper title"' ) assert len(messages) == 1 recipients = [m['content']['to'] for m in messages] assert '*****@*****.**' in recipients messages = client.get_messages( subject= '[Agora/COVID-19] A review has been posted on the article titled "Paper title"' ) assert len(messages) == 1 recipients = [m['content']['to'] for m in messages] assert '*****@*****.**' in recipients
def test_regular_names(self, client, helpers): builder = openreview.conference.ConferenceBuilder(client) assert builder, 'builder is None' builder.set_conference_id('NIPS.cc/2020/Workshop/MLITS') builder.set_submission_stage(public=True) conference = builder.get_result() note = openreview.Note(invitation = conference.get_submission_id(), readers = ['NIPS.cc/2020/Workshop/MLITS','*****@*****.**', '*****@*****.**', '*****@*****.**','~Bibtex_User1'], writers = [conference.id, '~Bibtex_User1', '*****@*****.**', '*****@*****.**'], signatures = ['~Bibtex_User1'], content = { 'title': 'Paper title has GANs and an Ô', 'abstract': 'This is an abstract with #s galore', 'authorids': ['*****@*****.**', '*****@*****.**', '*****@*****.**'], 'authors': ['Bibtex User', 'Peter Teët', 'Andrew McC'] } ) test_client = helpers.create_user('*****@*****.**', 'Bibtex', 'User') url = test_client.put_attachment(os.path.join(os.path.dirname(__file__), 'data/paper.pdf'), conference.get_submission_id(), 'pdf') note.content['pdf'] = url posted_note = test_client.post_note(note) bibtex = openreview.tools.get_bibtex(posted_note, conference.id, '2020', accepted=True, anonymous=False, baseurl=client.baseurl ) valid_bibtex = '''@inproceedings{ user2020paper, title={Paper title has {GAN}s and an \^O}, author={Bibtex User and Peter Te{\\"e}t and Andrew McC}, booktitle={NIPS.cc/2020/Workshop/MLITS}, year={2020}, url={''' valid_bibtex = valid_bibtex+client.baseurl+'/forum?id='+posted_note.forum+'''} }''' assert bibtex == valid_bibtex # test accepted False and names reversed bibtex = openreview.tools.get_bibtex(posted_note, conference.id, '2020', accepted=False, anonymous=False, names_reversed=True) valid_bibtex = '''@misc{ user2020paper, title={Paper title has {GAN}s and an \^O}, author={User, Bibtex and Te{\\"e}t, Peter and McC, Andrew}, year={2020}, url={https://openreview.net/forum?id=''' valid_bibtex = valid_bibtex + posted_note.forum + '''} }''' assert bibtex == valid_bibtex
def test_post_submission_stage(self, conference, helpers, test_client, client): year = datetime.datetime.now().year conference.setup_final_deadline_stage(force=True) submissions = conference.get_submissions(sort='number:desc') assert len(submissions) == 3 assert submissions[0].readers == ['everyone'] assert submissions[1].readers == ['everyone'] assert submissions[2].readers == ['everyone'] ## Withdraw paper test_client.post_note( openreview.Note( invitation= 'eswc-conferences.org/ESWC/2021/Conference/Paper5/-/Withdraw', forum=submissions[0].forum, replyto=submissions[0].forum, readers=['everyone'], writers=[ conference.get_id(), 'eswc-conferences.org/ESWC/2021/Conference/Paper5/Authors' ], signatures=[ 'eswc-conferences.org/ESWC/2021/Conference/Paper5/Authors' ], content={ 'title': 'Submission Withdrawn by the Authors', 'withdrawal confirmation': 'I have read and agree with the venue\'s withdrawal policy on behalf of myself and my co-authors.' })) helpers.await_queue() withdrawn_notes = client.get_notes( invitation= 'eswc-conferences.org/ESWC/2021/Conference/-/Withdrawn_Submission') assert len(withdrawn_notes) == 2 withdrawn_notes[0].readers == ['everyone'] withdrawn_notes[1].readers == [ 'eswc-conferences.org/ESWC/2021/Conference/Paper1/Authors', 'eswc-conferences.org/ESWC/2021/Conference/Paper1/Reviewers', 'eswc-conferences.org/ESWC/2021/Conference/Paper1/Area_Chairs', 'eswc-conferences.org/ESWC/2021/Conference/Program_Chairs' ] assert withdrawn_notes[0].content['_bibtex'] == '''@misc{ user''' + str(year) + '''paper, title={Paper title 5}, author={Test User and Peter Test and Andrew Mc}, year={''' + str(year) + '''}, url={https://openreview.net/forum?id=''' + withdrawn_notes[0].id + '''}
def test_post_submission(self, client, helpers): author_client = helpers.create_user(email='*****@*****.**', first='Author', last='One') note = openreview.Note(invitation='-Agora/COVID-19/-/Submission', readers=[ 'openreview.net/Support', '-Agora/COVID-19/Editors', '~Author_One1' ], writers=[ 'openreview.net/Support', '-Agora/COVID-19/Editors', '~Author_One1' ], signatures=['~Author_One1'], content={ 'title': 'Paper title', 'abstract': 'This is an abstract', 'authorids': ['~Author_One1'], 'authors': ['Author One'], 'pdf': 'https://openreview.net', 'requested_editors': ['*****@*****.**'] }) posted_note = author_client.post_note(note) helpers.await_queue() process_logs = client.get_process_logs(id=posted_note.id) assert len(process_logs) == 1 assert process_logs[0]['status'] == 'ok' messages = client.get_messages( subject= 'Agora COVID-19 has received your submission titled "Paper title"') assert len(messages) == 1 recipients = [m['content']['to'] for m in messages] assert '*****@*****.**' in recipients assert messages[0]['content'][ 'text'] == 'Your submission to Agora COVID-19 has been posted.\n\nTitle: Paper title\nYour submission will be examined by the Editor-in-Chief of the venue and you will receive an email with their response shortly.\nTo your submission can be viewed on OpenReview here: https://openreview.net/forum?id=' + posted_note.id messages = client.get_messages( subject= 'Agora COVID-19 has received a submission titled "Paper title"') assert len(messages) == 1 recipients = [m['content']['to'] for m in messages] assert '*****@*****.**' in recipients
def test_moderate_submission(self, client, helpers): editor_client = helpers.create_user(email='*****@*****.**', first='Editor', last='One') submissions = editor_client.get_notes( invitation='-Agora/COVID-19/-/Submission') assert submissions note = openreview.Note( invitation='-Agora/COVID-19/Submission1/-/Moderation', readers=[ 'openreview.net/Support', '-Agora/COVID-19/Editors', '-Agora/COVID-19/Submission1/Authors' ], writers=['openreview.net/Support'], signatures=['~Editor_One1'], forum=submissions[0].id, replyto=submissions[0].id, content={ 'resolution': 'Accept', 'comment': 'Thanks for submitting!' }) posted_note = editor_client.post_note(note) helpers.await_queue() process_logs = client.get_process_logs(id=posted_note.id) assert len(process_logs) == 1 assert process_logs[0]['status'] == 'ok' assert not editor_client.get_notes( invitation='-Agora/COVID-19/-/Submission') assert len( editor_client.get_notes( invitation='-Agora/COVID-19/-/Article')) == 1 assert len( editor_client.get_notes( invitation='-Agora/COVID-19/-/Desk_Rejected')) == 0 messages = client.get_messages( subject='[Agora/COVID-19] Your submission has been accepted') assert len(messages) == 1 recipients = [m['content']['to'] for m in messages] assert '*****@*****.**' in recipients assert messages[0]['content'][ 'text'] == 'Congratulations, your submission titled "Paper title" has been accepted by ~Editor_One1, the Editor-in-Chief of this venue.\nYour article is now visible to the public and an editor will be assigned soon based on your suggestions.\n\nThe article can be viewed on OpenReview here: https://openreview.net/forum?id=' + submissions[ 0].id
def test_open_revise_reviews(self, client, conference, test_client, selenium, request_page, helpers): notes = test_client.get_notes(invitation='icaps-conference.org/ICAPS/2019/Workshop/HSDIP/-/Blind_Submission') submission = notes[2] reviews = client.get_notes(invitation='icaps-conference.org/ICAPS/2019/Workshop/HSDIP/Paper1/-/Official_Review') assert reviews review = reviews[0] now = datetime.datetime.utcnow() conference.open_revise_reviews(due_date = now + datetime.timedelta(minutes = 10)) note = openreview.Note(invitation = 'icaps-conference.org/ICAPS/2019/Workshop/HSDIP/Paper1/AnonReviewer1/-/Review_Revision', forum = submission.id, referent = review.id, replyto=review.replyto, readers = ['icaps-conference.org/ICAPS/2019/Workshop/HSDIP/Program_Chairs', 'icaps-conference.org/ICAPS/2019/Workshop/HSDIP/Paper1/Reviewers', 'icaps-conference.org/ICAPS/2019/Workshop/HSDIP/Paper1/Authors'], writers = ['icaps-conference.org/ICAPS/2019/Workshop/HSDIP', 'icaps-conference.org/ICAPS/2019/Workshop/HSDIP/Paper1/AnonReviewer1'], signatures = ['icaps-conference.org/ICAPS/2019/Workshop/HSDIP/Paper1/AnonReviewer1'], content = { 'title': 'UPDATED Review title', 'review': 'Paper is very good!', 'rating': '9: Top 15% of accepted papers, strong accept', 'confidence': '4: The reviewer is confident but not absolutely certain that the evaluation is correct' } ) reviewer_client = openreview.Client(username='******', password='******') review_note = reviewer_client.post_note(note) assert review_note helpers.await_queue() process_logs = client.get_process_logs(id = review_note.id) assert len(process_logs) == 1 assert process_logs[0]['status'] == 'ok' messages = client.get_messages(subject = '.*ICAPS HSDIP 2019.*Revised review posted to your submission') assert len(messages) == 3 recipients = [m['content']['to'] for m in messages] assert '*****@*****.**' in recipients assert '*****@*****.**' in recipients assert '*****@*****.**' in recipients messages = client.get_messages(subject = '[ICAPS HSDIP 2019] Your revised review has been received on your assigned Paper number: 1, Paper title: "Paper title"') assert len(messages) == 1 recipients = [m['content']['to'] for m in messages] assert '*****@*****.**' in recipients
def create_config_note(self): ''' creates a config note but does not post it so that the sublcass override method can add additional stuff before posting happens. :return: ''' self.config_note = openreview.Note( **{ 'invitation': self.get_assignment_configuration_id(), 'readers': [self.conference.id], 'writers': [ self.conference.id, ], 'signatures': [self.conference.id], 'content': { 'title': self.config_title, Configuration.SCORES_SPECIFICATION: self.params.scores_config[Params.SCORES_SPEC], 'max_users': str(self.params.num_reviews_needed_per_paper ), # max number of reviewers a paper can have 'min_users': str(self.params.num_reviews_needed_per_paper ), # min number of reviewers a paper can have 'max_papers': str(self.params.reviewer_max_papers ), # max number of papers a reviewer can review 'min_papers': '1', # min number of papers a reviewer can review 'alternates': '2', 'constraints': {}, 'custom_loads': {}, 'config_invitation': self.conf_ids.CONFIG_ID, 'paper_invitation': self.conference.get_blind_submission_id(), 'metadata_invitation': self.get_metadata_id(), 'assignment_invitation': self.get_paper_assignment_id(), 'match_group': self.conference.get_reviewers_id(), 'status': 'Initialized' } }) self.add_config_custom_loads() self.add_config_constraints()
def test_venue_bid_stage(self, client, test_client, selenium, request_page, helpers, venue): reviewer_client = helpers.create_user('*****@*****.**', 'Venue', 'Reviewer') reviewer_group_id = '{}/Reviewers'.format(venue['venue_id']) reviewer_group = client.get_group(reviewer_group_id) client.add_members_to_group(reviewer_group, '~Venue_Reviewer1') reviewer_url = 'http://localhost:3030/group?id={}#reviewer-tasks'.format( reviewer_group_id) request_page(selenium, reviewer_url, reviewer_client.token) with pytest.raises(NoSuchElementException): assert selenium.find_element_by_link_text('Reviewer Bid') now = datetime.datetime.utcnow() start_date = now - datetime.timedelta(days=2) due_date = now + datetime.timedelta(days=3) bid_stage_note = test_client.post_note( openreview.Note(content={ 'bid_start_date': start_date.strftime('%Y/%m/%d'), 'bid_due_date': due_date.strftime('%Y/%m/%d') }, forum=venue['request_form_note'].forum, replyto=venue['request_form_note'].forum, referent=venue['request_form_note'].forum, invitation='{}/-/Request{}/Bid_Stage'.format( venue['support_group_id'], venue['request_form_note'].number), readers=[ '{}/Program_Chairs'.format(venue['venue_id']), venue['support_group_id'] ], signatures=['~Test_User1'], writers=['~Test_User1'])) assert bid_stage_note time.sleep(5) process_logs = client.get_process_logs(id=bid_stage_note.id) assert len(process_logs) == 1 assert process_logs[0][ 'invitation'] == '{}/-/Request{}/Bid_Stage'.format( venue['support_group_id'], venue['request_form_note'].number) assert process_logs[0]['status'] == 'ok' request_page(selenium, reviewer_url, reviewer_client.token) assert selenium.find_element_by_link_text('Reviewer Bid')
def test_submission_additional_files(self, test_client): pc_client = openreview.Client(username='******', password='******') builder = openreview.conference.ConferenceBuilder(pc_client) assert builder, 'builder is None' builder.set_conference_id('thecvf.com/ECCV/2020/Conference') builder.has_area_chairs(True) now = datetime.datetime.utcnow() builder.set_submission_stage(double_blind = True, public = False, due_date = now + datetime.timedelta(minutes = 10), additional_fields= { 'video': { 'description': 'Short video with presentation of the paper, it supports: mov, mp4, zip', 'required': True, 'value-file': { 'fileTypes': ['mov', 'mp4', 'zip'], 'size': 50000000000 } }, 'supplemental_material': { 'description': 'Paper appendix', 'required': False, 'value-file': { 'fileTypes': ['pdf'], 'size': 500000000000 } } }) conference = builder.get_result() note = openreview.Note(invitation = conference.get_submission_id(), readers = ['thecvf.com/ECCV/2020/Conference', '*****@*****.**', '*****@*****.**', '*****@*****.**', '~Test_User1'], writers = [conference.id, '~Test_User1', '*****@*****.**', '*****@*****.**'], signatures = ['~Test_User1'], content = { 'title': 'Paper title', 'abstract': 'This is an abstract', 'authorids': ['*****@*****.**', '*****@*****.**', '*****@*****.**'], 'authors': ['Test User', 'Peter Test', 'Andrew Mc'] } ) url = test_client.put_attachment(os.path.join(os.path.dirname(__file__), 'data/paper.pdf'), conference.get_submission_id(), 'pdf') note.content['pdf'] = url url = test_client.put_attachment(os.path.join(os.path.dirname(__file__), 'data/paper.pdf.zip'), conference.get_submission_id(), 'video') note.content['video'] = url test_client.post_note(note)
def resubmit(client, old_papers_inv, new_papers_inv): for paper in openreview.tools.iterget_notes(client, invitation=old_papers_inv): new_content = paper.content new_paper = openreview.Note(**{ 'invitation': new_papers_inv, 'writers': paper.signatures, 'readers': [iclr19.CONFERENCE_ID], 'signatures': paper.signatures, 'content': new_content }) try: p = client.post_note(new_paper) print("{} -> {}".format(paper.id, p.id)) except openreview.OpenReviewException as e: pass
def create_cv_note(title, abstract, authors, authorids, keywords): return openreview.Note( **{ 'readers': ['everyone'], 'writers': [], 'signatures': ['~Super_User1'], 'nonreaders': [], 'invitation': 'cv-foundation.org/ECCV/2018/Demo/-/Submission', 'content': { 'title': title.replace('\n', ''), 'abstract': abstract, 'authors': authors, 'authorids': authorids, 'keywords': keywords } })
def get_decision_note(pc_decision, comment, forum): decision_note = openreview.Note( **{ 'forum': forum, 'replyto': forum, 'invitation': 'ICLR.cc/2018/Workshop/-/Acceptance_Decision', 'signatures': ['ICLR.cc/2018/Workshop/Program_Chairs'], 'writers': ['ICLR.cc/2018/Workshop/Program_Chairs'], 'readers': ['ICLR.cc/2018/Workshop/Program_Chairs'], 'content': { 'title': "ICLR 2018 Workshop Acceptance Decision", 'decision': pc_decision, 'comment': comment } }) return decision_note