def test_open_decisions(self, client, conference, helpers):
conference.open_decisions()
pc_client = openreview.Client(username = '******', password = '******')
notes = pc_client.get_notes(invitation='icaps-conference.org/ICAPS/2019/Workshop/HSDIP/-/Blind_Submission')
assert len(notes) == 3
submission = notes[2]
note = openreview.Note(invitation = 'icaps-conference.org/ICAPS/2019/Workshop/HSDIP/Paper1/-/Decision',
forum = submission.id,
replyto = submission.id,
readers = ['icaps-conference.org/ICAPS/2019/Workshop/HSDIP/Program_Chairs'],
nonreaders = ['icaps-conference.org/ICAPS/2019/Workshop/HSDIP/Paper1/Authors'],
writers = ['icaps-conference.org/ICAPS/2019/Workshop/HSDIP/Program_Chairs'],
signatures = ['icaps-conference.org/ICAPS/2019/Workshop/HSDIP/Program_Chairs'],
content = {
'title': 'Paper Decision',
'decision': 'Accept (Oral)',
'comment': 'this is a comment'
}
)
decision_note = pc_client.post_note(note)
note = openreview.Note(invitation = 'icaps-conference.org/ICAPS/2019/Workshop/HSDIP/Paper2/-/Decision',
forum = notes[1].id,
replyto = notes[1].id,
readers = ['icaps-conference.org/ICAPS/2019/Workshop/HSDIP/Program_Chairs'],
nonreaders = ['icaps-conference.org/ICAPS/2019/Workshop/HSDIP/Paper2/Authors'],
writers = ['icaps-conference.org/ICAPS/2019/Workshop/HSDIP/Program_Chairs'],
signatures = ['icaps-conference.org/ICAPS/2019/Workshop/HSDIP/Program_Chairs'],
content = {
'title': 'Paper Decision',
'decision': 'Accept (Poster)',
'comment': 'this is a comment'
}
)
decision_note = pc_client.post_note(note)
note = openreview.Note(invitation = 'icaps-conference.org/ICAPS/2019/Workshop/HSDIP/Paper3/-/Decision',
forum = notes[0].id,
replyto = notes[0].id,
readers = ['icaps-conference.org/ICAPS/2019/Workshop/HSDIP/Program_Chairs'],
nonreaders = ['icaps-conference.org/ICAPS/2019/Workshop/HSDIP/Paper3/Authors'],
writers = ['icaps-conference.org/ICAPS/2019/Workshop/HSDIP/Program_Chairs'],
signatures = ['icaps-conference.org/ICAPS/2019/Workshop/HSDIP/Program_Chairs'],
content = {
'title': 'Paper Decision',
'decision': 'Reject',
'comment': 'this is a comment'
}
)
decision_note = pc_client.post_note(note)
notes = conference.get_submissions(accepted=True)
assert len(notes) == 2
notes = conference.get_submissions(accepted=False)
assert len(notes) == 3
def test_desk_reject(self, client, helpers):
author_client = openreview.Client(username = '******', password = '******')
note = openreview.Note(invitation = '-Agora/COVID-19/-/Submission',
readers = ['openreview.net/Support', '-Agora/COVID-19/Editors', '~Author_One1'],
writers = ['openreview.net/Support', '-Agora/COVID-19/Editors', '~Author_One1'],
signatures = ['~Author_One1'],
content = {
'title': 'Paper title 2',
'abstract': 'This is an abstract 2',
'authorids': ['~Author_One1'],
'authors': ['Author One'],
'pdf': 'https://openreview.net',
'requested_editors': ['*****@*****.**']
}
)
posted_note = author_client.post_note(note)
time.sleep(2)
process_logs = client.get_process_logs(id = posted_note.id)
assert len(process_logs) == 1
assert process_logs[0]['status'] == 'ok'
editor_client = openreview.Client(username='******', password='******')
submissions = editor_client.get_notes(invitation='-Agora/COVID-19/-/Submission')
assert submissions
note = openreview.Note(invitation = '-Agora/COVID-19/Submission2/-/Moderation',
readers = ['openreview.net/Support', '-Agora/COVID-19/Editors', '-Agora/COVID-19/Submission2/Authors'],
writers = ['openreview.net/Support'],
signatures = ['~Editor_One1'],
forum = submissions[0].id,
replyto = submissions[0].id,
content = {
'resolution': 'Desk-Reject',
'comment': 'This is not good'
}
)
posted_note = editor_client.post_note(note)
time.sleep(2)
process_logs = client.get_process_logs(id = posted_note.id)
assert len(process_logs) == 1
assert process_logs[0]['status'] == 'ok'
assert not editor_client.get_notes(invitation='-Agora/COVID-19/-/Submission')
assert len(editor_client.get_notes(invitation='-Agora/COVID-19/-/Article')) == 1
assert len(editor_client.get_notes(invitation='-Agora/COVID-19/-/Desk_Rejected')) == 1
messages = client.get_messages(subject = '[Agora/COVID-19] Your submission has been rejected')
assert len(messages) == 1
recipients = [m['content']['to'] for m in messages]
assert '*****@*****.**' in recipients
assert messages[0]['content']['text'] == 'Unfortunately your submission has been desk-rejected by the Editor-in-Chief of this venue.\n\nFor more information, see their comment on the OpenReview submission forum here: https://openreview.net/forum?id=' + submissions[0].id + '¬eId=' + posted_note.id
def test_decisions(self, conference, helpers, test_client, client):
conference.set_decision_stage(openreview.DecisionStage())
submissions = conference.get_submissions()
assert len(submissions) == 5
client.post_note(
openreview.Note(
invitation='MICCAI.org/2021/Challenges/Paper5/-/Decision',
forum=submissions[0].id,
replyto=submissions[0].id,
readers=[
'MICCAI.org/2021/Challenges/Program_Chairs',
'MICCAI.org/2021/Challenges/Paper5/Area_Chairs'
],
nonreaders=['MICCAI.org/2021/Challenges/Paper5/Authors'],
writers=['MICCAI.org/2021/Challenges/Program_Chairs'],
signatures=['MICCAI.org/2021/Challenges/Program_Chairs'],
content={
'title': 'Paper Decision',
'decision': 'Reject'
}))
client.post_note(
openreview.Note(
invitation='MICCAI.org/2021/Challenges/Paper4/-/Decision',
forum=submissions[1].id,
replyto=submissions[1].id,
readers=[
'MICCAI.org/2021/Challenges/Program_Chairs',
'MICCAI.org/2021/Challenges/Paper4/Area_Chairs'
],
nonreaders=['MICCAI.org/2021/Challenges/Paper4/Authors'],
writers=['MICCAI.org/2021/Challenges/Program_Chairs'],
signatures=['MICCAI.org/2021/Challenges/Program_Chairs'],
content={
'title': 'Paper Decision',
'decision': 'Accept (Oral)'
}))
client.post_note(
openreview.Note(
invitation='MICCAI.org/2021/Challenges/Paper3/-/Decision',
forum=submissions[2].id,
replyto=submissions[2].id,
readers=[
'MICCAI.org/2021/Challenges/Program_Chairs',
'MICCAI.org/2021/Challenges/Paper3/Area_Chairs'
],
nonreaders=['MICCAI.org/2021/Challenges/Paper3/Authors'],
writers=['MICCAI.org/2021/Challenges/Program_Chairs'],
signatures=['MICCAI.org/2021/Challenges/Program_Chairs'],
content={
'title': 'Paper Decision',
'decision': 'Accept (Poster)'
}))
def get_note(self, configNoteId):
if configNoteId == 'nonExist':
raise OpenReviewException([{'type': 'Not Found'}])
elif configNoteId == 'forbidden':
raise OpenReviewException([{'type': 'Forbidden'}])
elif configNoteId == 'internal error':
list = []
list[2]
elif configNoteId == 'already_running':
note = openreview.Note(id='testId',
invitation=None,
readers=[],
writers=[],
signatures=[],
content={
'title': 'foo',
'metadata_invitation': 1,
'match_group': 1,
'paper_invitation': 1,
'assignment_invitation': 1,
'max_papers': 1,
'max_users': 1,
'min_papers': 1,
'weights': 1,
'constraints': 1,
'scores_names': [],
'scores_weights': [],
'alternates': 3,
'status': Configuration.STATUS_RUNNING
})
return note
else:
note = openreview.Note(id='testId',
invitation=None,
readers=[],
writers=[],
signatures=[],
content={
'title': 'foo',
'metadata_invitation': 1,
'match_group': 1,
'paper_invitation': 1,
'assignment_invitation': 1,
'max_papers': 1,
'max_users': 1,
'min_papers': 1,
'weights': 1,
'constraints': 1,
'scores_names': [],
'scores_weights': [],
'alternates': 3,
'status':
Configuration.STATUS_INITIALIZED
})
return note
def upload_submissions(client):
# repost 20 papers from AKBC 2019
# TODO: Change this to read from ICML papers
for paper in client.get_notes(
invitation='AKBC.ws/2019/Conference/-/Submission'):
new_content = {key: value for key, value in paper.content.items() \
if key in icml.submission_inv.reply['content'].keys()}
new_writers = [w.replace('AKBC.ws', 'ICML.cc') for w in paper.writers]
posted_submission = client.post_note(
openreview.Note(
**{
'writers': new_writers,
'readers': [icml.CONFERENCE_ID],
'content': new_content,
'invitation': icml.SUBMISSION_ID,
'signatures': [icml.CONFERENCE_ID]
}))
papergroup = client.post_group(
openreview.Group.from_json({
'id':
'ICML.cc/2019/Conference/Paper{}'.format(
posted_submission.number),
'readers': ['everyone'],
'writers': [icml.CONFERENCE_ID],
'signatures': [icml.CONFERENCE_ID],
'signatories': [],
'members': []
}))
posted_mask = client.post_note(
openreview.Note(
**{
'original': posted_submission.id,
'forum': None,
'replyto': None,
'readers': ['everyone'],
'writers': [icml.CONFERENCE_ID],
'signatures': [icml.CONFERENCE_ID],
'invitation': icml.blind_submission_inv.id,
'content': {
'title': 'Anonymous Submission',
'abstract': 'Redacted',
'authorids': ['Anonymous'],
'authors': ['Anonymous']
}
}))
def get_color_paper(color):
author_signature = "~Jane_Doe1"
author_id = "*****@*****.**"
note_readers = [
"ICLR.cc/2018/Conference", "ICLR.cc/2018/Conference/Program_Chairs",
author_id, author_signature
]
note_signatures = [author_signature]
return openreview.Note(
**{
"readers": note_readers,
"nonreaders": [],
"signatures": note_signatures,
"writers": [],
"invitation": "ICLR.cc/2018/Conference/-/Submission",
"content": {
'title': '{0} Paper'.format(color),
'abstract': 'This is the {0} abstract.'.format(color),
'authors': '{0} authors'.format(color),
'keywords': color,
'TL;DR': 'This is the {0} TL;DR'.format(color),
'authorids': [author_id],
'pdf': "/pdf/813abccc0d09f502de96bbdca811a42e5ac59a2a.pdf"
}
})
def post_metadata_note(client, note, reviewer_profiles, metadata_inv,
paper_tpms_scores, manual_conflicts_by_id):
author_group = note.content['authorids']
authorids = note.details['original']['content']['authorids']
paper_bid_jsons = note.details['tags']
paper_author_profiles = client.get_profiles(authorids)
print('Paper', note.id)
entries = _build_entries(paper_author_profiles, reviewer_profiles,
paper_bid_jsons, paper_tpms_scores,
manual_conflicts_by_id)
new_metadata_note = openreview.Note(
**{
'forum': note.id,
'invitation': metadata_inv.id,
'readers': metadata_inv.reply['readers']['values'],
'writers': metadata_inv.reply['writers']['values'],
'signatures': metadata_inv.reply['signatures']['values'],
'content': {
'title': 'Paper Metadata',
'entries': entries
}
})
return client.post_note(new_metadata_note)
def test_open_reviews(self, client, conference, test_client, selenium, request_page, helpers):
conference.set_reviewers(emails = ['*****@*****.**'])
notes = test_client.get_notes(invitation='icaps-conference.org/ICAPS/2019/Workshop/HSDIP/-/Blind_Submission')
submission = notes[2]
# Reviewer
reviewer_client = helpers.create_user('*****@*****.**', 'Reviewer', 'Four')
conference.set_assignment('*****@*****.**', submission.number)
request_page(selenium, "http://localhost:3030/forum?id=" + submission.id, reviewer_client.token)
reply_row = selenium.find_element_by_class_name('reply_row')
assert len(reply_row.find_elements_by_class_name('btn')) == 1
assert 'Official Review' == reply_row.find_elements_by_class_name('btn')[0].text
# Author
request_page(selenium, "http://localhost:3030/forum?id=" + submission.id, test_client.token)
reply_row = selenium.find_element_by_class_name('reply_row')
assert len(reply_row.find_elements_by_class_name('btn')) == 1
assert 'Withdraw' == reply_row.find_elements_by_class_name('btn')[0].text
note = openreview.Note(invitation = 'icaps-conference.org/ICAPS/2019/Workshop/HSDIP/Paper1/-/Official_Review',
forum = submission.id,
replyto = submission.id,
readers = ['icaps-conference.org/ICAPS/2019/Workshop/HSDIP/Program_Chairs',
'icaps-conference.org/ICAPS/2019/Workshop/HSDIP/Paper1/Reviewers',
'icaps-conference.org/ICAPS/2019/Workshop/HSDIP/Paper1/Authors'],
writers = ['icaps-conference.org/ICAPS/2019/Workshop/HSDIP/Paper1/AnonReviewer1'],
signatures = ['icaps-conference.org/ICAPS/2019/Workshop/HSDIP/Paper1/AnonReviewer1'],
content = {
'title': 'Review title',
'review': 'Paper is very good!',
'rating': '9: Top 15% of accepted papers, strong accept',
'confidence': '4: The reviewer is confident but not absolutely certain that the evaluation is correct'
}
)
review_note = reviewer_client.post_note(note)
assert review_note
helpers.await_queue()
process_logs = client.get_process_logs(id = review_note.id)
assert len(process_logs) == 1
assert process_logs[0]['status'] == 'ok'
messages = client.get_messages(subject = '[ICAPS HSDIP 2019] Review posted to your submission: "New paper title"')
assert len(messages) == 0
messages = client.get_messages(subject = '[ICAPS HSDIP 2019] Review posted to your assigned paper: "New paper title"')
assert len(messages) == 0
## Check review visibility
notes = reviewer_client.get_notes(invitation='icaps-conference.org/ICAPS/2019/Workshop/HSDIP/Paper1/-/Official_Review')
assert len(notes) == 1
notes = test_client.get_notes(invitation='icaps-conference.org/ICAPS/2019/Workshop/HSDIP/Paper1/-/Official_Review')
assert len(notes) == 1
def get_iclr_paper(iclr17_note):
author_signature = "~Jane_Doe1"
author_id = "*****@*****.**"
note_readers = [
"ICLR.cc/2018/Conference", "ICLR.cc/2018/Conference/Program_Chairs",
author_id, author_signature
]
note_signatures = [author_signature]
return openreview.Note(
**{
"readers": note_readers,
"nonreaders": [],
"signatures": note_signatures,
"writers": [],
"invitation": "ICLR.cc/2018/Conference/-/Submission",
"content": {
'title': iclr17_note.content['title'],
'abstract': iclr17_note.content['abstract'],
'authors': iclr17_note.content['authors'],
'keywords': iclr17_note.content['keywords'],
'TL;DR': iclr17_note.content['TL;DR'],
'authorids': [author_id],
'pdf': iclr17_note.content['pdf']
}
})
def test_submit_papers(self, conference, helpers, test_client, client):
pdf_url = test_client.put_attachment(
os.path.join(os.path.dirname(__file__), 'data/paper.pdf'),
'MICCAI.org/2021/Challenges/-/Submission', 'pdf')
domains = [
'umass.edu', 'umass.edu', 'fb.com', 'umass.edu', 'google.com',
'mit.edu'
]
for i in range(1, 6):
note = openreview.Note(
invitation='MICCAI.org/2021/Challenges/-/Submission',
readers=[
'MICCAI.org/2021/Challenges', '*****@*****.**',
'*****@*****.**', 'andrew@' + domains[i], '~Test_User1'
],
writers=[
conference.id, '~Test_User1', '*****@*****.**',
'andrew@' + domains[i]
],
signatures=['~Test_User1'],
content={
'title':
'Paper title ' + str(i),
'abstract':
'This is an abstract ' + str(i),
'authorids': [
'*****@*****.**', '*****@*****.**',
'andrew@' + domains[i]
],
'authors': ['Test User', 'Peter Test', 'Andrew Mc'],
'pdf':
pdf_url
})
note = test_client.post_note(note)
conference.setup_post_submission_stage(force=True)
notes = test_client.get_notes(
invitation='MICCAI.org/2021/Challenges/-/Submission')
assert len(notes) == 5
assert notes[0].readers == [
'MICCAI.org/2021/Challenges',
'MICCAI.org/2021/Challenges/Area_Chairs',
'MICCAI.org/2021/Challenges/Paper5/Reviewers',
'MICCAI.org/2021/Challenges/Paper5/Authors'
]
invitations = test_client.get_invitations(replyForum=notes[0].id)
assert len(invitations) == 1
assert invitations[
0].id == 'MICCAI.org/2021/Challenges/Paper5/-/Withdraw'
invitations = client.get_invitations(replyForum=notes[0].id)
assert len(invitations) == 2
invitation_ids = [invitation.id for invitation in invitations]
assert 'MICCAI.org/2021/Challenges/Paper5/-/Desk_Reject' in invitation_ids
assert 'MICCAI.org/2021/Challenges/Paper5/-/Withdraw' in invitation_ids
def test_add_revision(self, client, helpers):
author_client = openreview.Client(username='******',
password='******')
articles = author_client.get_notes(
invitation='-Agora/COVID-19/-/Article')
assert articles
note = openreview.Note(
invitation='-Agora/COVID-19/Article1/-/Revision',
readers=['everyone'],
writers=[
'openreview.net/Support', '-Agora/COVID-19/Article1/Authors'
],
signatures=['-Agora/COVID-19/Article1/Authors'],
forum=articles[0].forum,
referent=articles[0].id,
content={
'title':
'Paper title',
'abstract':
'This is an abstract',
'authorids': ['~Author_One1'],
'authors': ['Author One'],
'pdf':
'https://openreview.net',
'requested_editors': [
'*****@*****.**',
'*****@*****.**'
]
})
posted_note = author_client.post_note(note)
helpers.await_queue()
process_logs = client.get_process_logs(id=posted_note.id)
assert len(process_logs) == 1
assert process_logs[0]['status'] == 'ok'
messages = client.get_messages(
subject=
'[Agora/COVID-19] Your revision has been posted on your article titled "Paper title"'
)
assert len(messages) == 1
recipients = [m['content']['to'] for m in messages]
assert '*****@*****.**' in recipients
messages = client.get_messages(
subject=
'[Agora/COVID-19] A revision has been posted on the article titled "Paper title"'
)
assert len(messages) == 3
recipients = [m['content']['to'] for m in messages]
assert '*****@*****.**' in recipients
assert '*****@*****.**' in recipients
assert '*****@*****.**' in recipients
def release_author_names(client):
print('Release author names...')
meta_reviews = list(
openreview.tools.iterget_notes(
client,
invitation='ICLR.cc/2019/Conference/-/Paper.*/Meta_Review'))
submissions = list(
openreview.tools.iterget_notes(
client,
invitation='ICLR.cc/2019/Conference/-/Blind_Submission',
details='original'))
submissions_dict = {}
for s in submissions:
submissions_dict[s.id] = s
for m in meta_reviews:
if m.forum in submissions_dict:
submission = submissions_dict[m.forum]
decision = m.content['recommendation']
original_note = openreview.Note.from_json(
submission.details['original'])
if 'Reject' in decision:
accepted = False
if 'Oral' in decision:
accepted = True
if 'Poster' in decision:
accepted = True
overwriting_note = openreview.Note(
id=submission.id,
original=submission.original,
invitation=iclr19.BLIND_SUBMISSION_ID,
forum=submission.forum,
signatures=[iclr19.CONFERENCE_ID],
writers=[iclr19.CONFERENCE_ID],
readers=['everyone'],
content={
'_bibtex':
openreview.tools.get_bibtex(
original_note,
'International Conference on Learning Representations',
'2019',
url_forum=submission.forum,
accepted=accepted,
anonymous=False)
})
client.post_note(overwriting_note)
if accepted:
revision_invitation = client.get_invitation(
id='{conference_id}/-/Paper{number}/Revision'.format(
conference_id=iclr19.CONFERENCE_ID,
number=submission.number))
revision_invitation.expdate = 1576843200000
client.post_invitation(revision_invitation)
def dump_cached_or(entries, out_pickle):
cached_or = {}
for entry in entries:
cached_or[entry[0]] = openreview.Note( # id
"", [], [], [], {"abstract": entry[3], "title": entry[1]}
) # Hack. ICLR Recommender script accepts Openreview notes
with open(out_pickle, "wb") as fd:
pickle.dump(cached_or, fd)
def test_post_submission(self, client, conference, test_client):
note = openreview.Note(
invitation=conference.get_submission_id(),
readers=['everyone'],
writers=[
conference.id, '~Test_User1', '*****@*****.**',
'*****@*****.**'
],
signatures=['~Test_User1'],
content={
'title': 'Paper title',
'abstract': 'This is an abstract',
'authorids':
['*****@*****.**', '*****@*****.**', '*****@*****.**'],
'authors': ['Test User', 'Peter Test', 'Andrew Mc']
})
url = test_client.put_attachment(
os.path.join(os.path.dirname(__file__), 'data/paper.pdf'),
conference.get_submission_id(), 'pdf')
note.content['pdf'] = url
note = test_client.post_note(note)
time.sleep(2)
note = client.get_note(note.id)
process_logs = client.get_process_logs(id=note.id)
assert len(process_logs) == 1
assert process_logs[0]['status'] == 'ok'
messages = client.get_messages(
subject=
'NLP-COVID-2020 has received your submission titled Paper title')
assert len(messages) == 3
recipients = [m['content']['to'] for m in messages]
assert '*****@*****.**' in recipients
assert '*****@*****.**' in recipients
assert '*****@*****.**' in recipients
messages = client.get_messages(
subject=
'NLP-COVID-2020 has received a new submission titled Paper title')
assert len(messages) == 1
recipients = [m['content']['to'] for m in messages]
assert '*****@*****.**' in recipients
assert client.get_group(
'aclweb.org/ACL/2020/Workshop/NLP-COVID/Paper1')
assert client.get_group(
'aclweb.org/ACL/2020/Workshop/NLP-COVID/Paper1/Authors')
assert client.get_group(
'aclweb.org/ACL/2020/Workshop/NLP-COVID/Paper1/Reviewers')
assert client.get_group(
'aclweb.org/ACL/2020/Workshop/NLP-COVID/Paper1/Reviewers/Submitted'
)
assert client.get_invitation(
'aclweb.org/ACL/2020/Workshop/NLP-COVID/Paper1/-/Official_Review')
def clean_start_conference(client, conference_id, num_reviewers, num_papers,
reviews_per_paper):
builder = openreview.conference.ConferenceBuilder(client)
builder.set_conference_id(conference_id)
builder.set_submission_stage(
due_date=datetime.datetime(2019, 3, 25, 23, 59),
remove_fields=['authors', 'abstract', 'pdf', 'keywords', 'TL;DR'])
conference = builder.get_result()
submission_invitation = client.get_invitation(
conference.get_submission_id())
submission_invitation.reply['content']['authorids'] = {
'values-regex': '.*'
}
submission_invitation = client.post_invitation(submission_invitation)
reviewers = []
# TODO: is there a better way to handle affinity scores?
# Maybe conference.setup_matching() should allow a score matrix as input
with open(AFFINITY_SCORE_FILE, 'w') as file_handle:
for paper_number in range(num_papers):
content = {
'title': 'Test_Paper_{}'.format(paper_number),
'authorids': [
'testauthor{}{}@test.com'.format(paper_number, author_code) \
for author_code in ['A', 'B', 'C']
]
}
submission = openreview.Note(
**{
'signatures': ['~Super_User1'],
'writers': [conference.id],
'readers': [conference.id],
'content': content,
'invitation': conference.get_submission_id()
})
posted_submission = client.post_note(submission)
for reviewer_number in range(num_reviewers):
reviewer = '~Test_Reviewer{}'.format(reviewer_number)
reviewers.append(reviewer)
score = random.random()
row = [
posted_submission.forum, reviewer, '{:.3f}'.format(score)
]
file_handle.write(','.join(row) + '\n')
conference.set_authors()
conference.set_reviewers(emails=reviewers)
conference.setup_matching(affinity_score_file=AFFINITY_SCORE_FILE)
return conference
def test_registration(self, conference, helpers, selenium, request_page):
reviewer_client = openreview.Client(username='******',
password='******')
reviewer_tasks_url = 'http://localhost:3030/group?id=ICLR.cc/2021/Conference/Reviewers#reviewer-tasks'
request_page(selenium, reviewer_tasks_url, reviewer_client.token)
assert selenium.find_element_by_link_text('Reviewer Registration')
assert selenium.find_element_by_link_text('Expertise Selection')
registration_notes = reviewer_client.get_notes(
invitation='ICLR.cc/2021/Conference/Reviewers/-/Form')
assert registration_notes
assert len(registration_notes) == 1
registration_forum = registration_notes[0].forum
registration_note = reviewer_client.post_note(
openreview.Note(
invitation='ICLR.cc/2021/Conference/Reviewers/-/Registration',
forum=registration_forum,
replyto=registration_forum,
content={
'profile_confirmed': 'Yes',
'expertise_confirmed': 'Yes',
'TPMS_registration_confirmed': 'Yes',
'reviewer_instructions_confirm': 'Yes',
'emergency_review_count': '0'
},
signatures=['~ReviewerOne_ICLR1'],
readers=[conference.get_id(), '~ReviewerOne_ICLR1'],
writers=[conference.get_id(), '~ReviewerOne_ICLR1']))
assert registration_note
request_page(
selenium,
'http://localhost:3030/group?id=ICLR.cc/2021/Conference/Reviewers',
reviewer_client.token)
header = selenium.find_element_by_id('header')
assert header
notes = header.find_elements_by_class_name("description")
assert notes
assert len(notes) == 1
assert notes[
0].text == 'This page provides information and status updates for the ICLR 2021. It will be regularly updated as the conference progresses, so please check back frequently.'
request_page(selenium, reviewer_tasks_url, reviewer_client.token)
assert selenium.find_element_by_link_text('Reviewer Registration')
assert selenium.find_element_by_link_text('Expertise Selection')
tasks = selenium.find_element_by_id('reviewer-tasks')
assert tasks
assert len(tasks.find_elements_by_class_name('note')) == 2
assert len(tasks.find_elements_by_class_name('completed')) == 2
def test_post_metareview(self, client, helpers):
article_editor_client = openreview.Client(
username='******', password='******')
articles = article_editor_client.get_notes(
invitation='-Agora/COVID-19/-/Article')
assert articles
note = openreview.Note(
invitation='-Agora/COVID-19/Article1/-/Meta_Review',
readers=['everyone'],
writers=['openreview.net/Support', '~ArticleEditor_One1'],
signatures=['~ArticleEditor_One1'],
forum=articles[0].id,
replyto=articles[0].id,
content={
'title': 'review title',
'metareview': 'excelent paper',
'recommendation': 'Accept'
})
posted_note = article_editor_client.post_note(note)
helpers.await_queue()
process_logs = client.get_process_logs(id=posted_note.id)
assert len(process_logs) == 1
assert process_logs[0]['status'] == 'ok'
messages = client.get_messages(
subject=
'[Agora/COVID-19] Meta Review posted to your article titled "Paper title"'
)
assert len(messages) == 1
recipients = [m['content']['to'] for m in messages]
assert '*****@*****.**' in recipients
messages = client.get_messages(
subject=
'[Agora/COVID-19] Your meta review has been posted on your assigned article titled "Paper title"'
)
assert len(messages) == 1
recipients = [m['content']['to'] for m in messages]
assert '*****@*****.**' in recipients
messages = client.get_messages(
subject=
'[Agora/COVID-19] A meta review has been posted on the article titled "Paper title"'
)
assert len(messages) == 2
recipients = [m['content']['to'] for m in messages]
assert '*****@*****.**' in recipients
assert '*****@*****.**' in recipients
def create_blind_note(note):
return openreview.Note(original=note.id,
invitation=conference_config.BLIND_SUBMISSION_ID,
forum=None,
signatures=[conference_config.CONFERENCE_ID],
writers=[conference_config.CONFERENCE_ID],
readers=['everyone'],
content={
"authors": ['Anonymous'],
"authorids": [],
"_bibtex": None
})
def test_post_review(self, client, helpers):
reviewer_client = helpers.create_user(email='*****@*****.**',
first='ArticleReviewer',
last='One')
articles = reviewer_client.get_notes(
invitation='-Agora/COVID-19/-/Article')
assert articles
note = openreview.Note(
invitation='-Agora/COVID-19/Article1/-/Review',
readers=['everyone'],
writers=['openreview.net/Support', '~ArticleReviewer_One1'],
signatures=['~ArticleReviewer_One1'],
forum=articles[0].id,
replyto=articles[0].id,
content={
'title': 'review title',
'review': 'excelent paper'
})
posted_note = reviewer_client.post_note(note)
helpers.await_queue()
process_logs = client.get_process_logs(id=posted_note.id)
assert len(process_logs) == 1
assert process_logs[0]['status'] == 'ok'
messages = client.get_messages(
subject=
'[Agora/COVID-19] Review posted to your article titled "Paper title"'
)
assert len(messages) == 1
recipients = [m['content']['to'] for m in messages]
assert '*****@*****.**' in recipients
messages = client.get_messages(
subject=
'[Agora/COVID-19] Your review has been posted on your assigned article titled "Paper title"'
)
assert len(messages) == 1
recipients = [m['content']['to'] for m in messages]
assert '*****@*****.**' in recipients
messages = client.get_messages(
subject=
'[Agora/COVID-19] A review has been posted on the article titled "Paper title"'
)
assert len(messages) == 1
recipients = [m['content']['to'] for m in messages]
assert '*****@*****.**' in recipients
def test_regular_names(self, client, helpers):
builder = openreview.conference.ConferenceBuilder(client)
assert builder, 'builder is None'
builder.set_conference_id('NIPS.cc/2020/Workshop/MLITS')
builder.set_submission_stage(public=True)
conference = builder.get_result()
note = openreview.Note(invitation = conference.get_submission_id(),
readers = ['NIPS.cc/2020/Workshop/MLITS','*****@*****.**', '*****@*****.**', '*****@*****.**','~Bibtex_User1'],
writers = [conference.id, '~Bibtex_User1', '*****@*****.**', '*****@*****.**'],
signatures = ['~Bibtex_User1'],
content = {
'title': 'Paper title has GANs and an Ô',
'abstract': 'This is an abstract with #s galore',
'authorids': ['*****@*****.**', '*****@*****.**', '*****@*****.**'],
'authors': ['Bibtex User', 'Peter Teët', 'Andrew McC']
}
)
test_client = helpers.create_user('*****@*****.**', 'Bibtex', 'User')
url = test_client.put_attachment(os.path.join(os.path.dirname(__file__), 'data/paper.pdf'), conference.get_submission_id(), 'pdf')
note.content['pdf'] = url
posted_note = test_client.post_note(note)
bibtex = openreview.tools.get_bibtex(posted_note, conference.id, '2020', accepted=True, anonymous=False, baseurl=client.baseurl )
valid_bibtex = '''@inproceedings{
user2020paper,
title={Paper title has {GAN}s and an \^O},
author={Bibtex User and Peter Te{\\"e}t and Andrew McC},
booktitle={NIPS.cc/2020/Workshop/MLITS},
year={2020},
url={'''
valid_bibtex = valid_bibtex+client.baseurl+'/forum?id='+posted_note.forum+'''}
}'''
assert bibtex == valid_bibtex
# test accepted False and names reversed
bibtex = openreview.tools.get_bibtex(posted_note, conference.id, '2020', accepted=False, anonymous=False, names_reversed=True)
valid_bibtex = '''@misc{
user2020paper,
title={Paper title has {GAN}s and an \^O},
author={User, Bibtex and Te{\\"e}t, Peter and McC, Andrew},
year={2020},
url={https://openreview.net/forum?id='''
valid_bibtex = valid_bibtex + posted_note.forum + '''}
}'''
assert bibtex == valid_bibtex
def test_post_submission_stage(self, conference, helpers, test_client,
client):
year = datetime.datetime.now().year
conference.setup_final_deadline_stage(force=True)
submissions = conference.get_submissions(sort='number:desc')
assert len(submissions) == 3
assert submissions[0].readers == ['everyone']
assert submissions[1].readers == ['everyone']
assert submissions[2].readers == ['everyone']
## Withdraw paper
test_client.post_note(
openreview.Note(
invitation=
'eswc-conferences.org/ESWC/2021/Conference/Paper5/-/Withdraw',
forum=submissions[0].forum,
replyto=submissions[0].forum,
readers=['everyone'],
writers=[
conference.get_id(),
'eswc-conferences.org/ESWC/2021/Conference/Paper5/Authors'
],
signatures=[
'eswc-conferences.org/ESWC/2021/Conference/Paper5/Authors'
],
content={
'title':
'Submission Withdrawn by the Authors',
'withdrawal confirmation':
'I have read and agree with the venue\'s withdrawal policy on behalf of myself and my co-authors.'
}))
helpers.await_queue()
withdrawn_notes = client.get_notes(
invitation=
'eswc-conferences.org/ESWC/2021/Conference/-/Withdrawn_Submission')
assert len(withdrawn_notes) == 2
withdrawn_notes[0].readers == ['everyone']
withdrawn_notes[1].readers == [
'eswc-conferences.org/ESWC/2021/Conference/Paper1/Authors',
'eswc-conferences.org/ESWC/2021/Conference/Paper1/Reviewers',
'eswc-conferences.org/ESWC/2021/Conference/Paper1/Area_Chairs',
'eswc-conferences.org/ESWC/2021/Conference/Program_Chairs'
]
assert withdrawn_notes[0].content['_bibtex'] == '''@misc{
user''' + str(year) + '''paper,
title={Paper title 5},
author={Test User and Peter Test and Andrew Mc},
year={''' + str(year) + '''},
url={https://openreview.net/forum?id=''' + withdrawn_notes[0].id + '''}
def test_post_submission(self, client, helpers):
author_client = helpers.create_user(email='*****@*****.**',
first='Author',
last='One')
note = openreview.Note(invitation='-Agora/COVID-19/-/Submission',
readers=[
'openreview.net/Support',
'-Agora/COVID-19/Editors', '~Author_One1'
],
writers=[
'openreview.net/Support',
'-Agora/COVID-19/Editors', '~Author_One1'
],
signatures=['~Author_One1'],
content={
'title':
'Paper title',
'abstract':
'This is an abstract',
'authorids': ['~Author_One1'],
'authors': ['Author One'],
'pdf':
'https://openreview.net',
'requested_editors':
['*****@*****.**']
})
posted_note = author_client.post_note(note)
helpers.await_queue()
process_logs = client.get_process_logs(id=posted_note.id)
assert len(process_logs) == 1
assert process_logs[0]['status'] == 'ok'
messages = client.get_messages(
subject=
'Agora COVID-19 has received your submission titled "Paper title"')
assert len(messages) == 1
recipients = [m['content']['to'] for m in messages]
assert '*****@*****.**' in recipients
assert messages[0]['content'][
'text'] == 'Your submission to Agora COVID-19 has been posted.\n\nTitle: Paper title\nYour submission will be examined by the Editor-in-Chief of the venue and you will receive an email with their response shortly.\nTo your submission can be viewed on OpenReview here: https://openreview.net/forum?id=' + posted_note.id
messages = client.get_messages(
subject=
'Agora COVID-19 has received a submission titled "Paper title"')
assert len(messages) == 1
recipients = [m['content']['to'] for m in messages]
assert '*****@*****.**' in recipients
def test_moderate_submission(self, client, helpers):
editor_client = helpers.create_user(email='*****@*****.**',
first='Editor',
last='One')
submissions = editor_client.get_notes(
invitation='-Agora/COVID-19/-/Submission')
assert submissions
note = openreview.Note(
invitation='-Agora/COVID-19/Submission1/-/Moderation',
readers=[
'openreview.net/Support', '-Agora/COVID-19/Editors',
'-Agora/COVID-19/Submission1/Authors'
],
writers=['openreview.net/Support'],
signatures=['~Editor_One1'],
forum=submissions[0].id,
replyto=submissions[0].id,
content={
'resolution': 'Accept',
'comment': 'Thanks for submitting!'
})
posted_note = editor_client.post_note(note)
helpers.await_queue()
process_logs = client.get_process_logs(id=posted_note.id)
assert len(process_logs) == 1
assert process_logs[0]['status'] == 'ok'
assert not editor_client.get_notes(
invitation='-Agora/COVID-19/-/Submission')
assert len(
editor_client.get_notes(
invitation='-Agora/COVID-19/-/Article')) == 1
assert len(
editor_client.get_notes(
invitation='-Agora/COVID-19/-/Desk_Rejected')) == 0
messages = client.get_messages(
subject='[Agora/COVID-19] Your submission has been accepted')
assert len(messages) == 1
recipients = [m['content']['to'] for m in messages]
assert '*****@*****.**' in recipients
assert messages[0]['content'][
'text'] == 'Congratulations, your submission titled "Paper title" has been accepted by ~Editor_One1, the Editor-in-Chief of this venue.\nYour article is now visible to the public and an editor will be assigned soon based on your suggestions.\n\nThe article can be viewed on OpenReview here: https://openreview.net/forum?id=' + submissions[
0].id
def test_open_revise_reviews(self, client, conference, test_client, selenium, request_page, helpers):
notes = test_client.get_notes(invitation='icaps-conference.org/ICAPS/2019/Workshop/HSDIP/-/Blind_Submission')
submission = notes[2]
reviews = client.get_notes(invitation='icaps-conference.org/ICAPS/2019/Workshop/HSDIP/Paper1/-/Official_Review')
assert reviews
review = reviews[0]
now = datetime.datetime.utcnow()
conference.open_revise_reviews(due_date = now + datetime.timedelta(minutes = 10))
note = openreview.Note(invitation = 'icaps-conference.org/ICAPS/2019/Workshop/HSDIP/Paper1/AnonReviewer1/-/Review_Revision',
forum = submission.id,
referent = review.id,
replyto=review.replyto,
readers = ['icaps-conference.org/ICAPS/2019/Workshop/HSDIP/Program_Chairs',
'icaps-conference.org/ICAPS/2019/Workshop/HSDIP/Paper1/Reviewers',
'icaps-conference.org/ICAPS/2019/Workshop/HSDIP/Paper1/Authors'],
writers = ['icaps-conference.org/ICAPS/2019/Workshop/HSDIP', 'icaps-conference.org/ICAPS/2019/Workshop/HSDIP/Paper1/AnonReviewer1'],
signatures = ['icaps-conference.org/ICAPS/2019/Workshop/HSDIP/Paper1/AnonReviewer1'],
content = {
'title': 'UPDATED Review title',
'review': 'Paper is very good!',
'rating': '9: Top 15% of accepted papers, strong accept',
'confidence': '4: The reviewer is confident but not absolutely certain that the evaluation is correct'
}
)
reviewer_client = openreview.Client(username='******', password='******')
review_note = reviewer_client.post_note(note)
assert review_note
helpers.await_queue()
process_logs = client.get_process_logs(id = review_note.id)
assert len(process_logs) == 1
assert process_logs[0]['status'] == 'ok'
messages = client.get_messages(subject = '.*ICAPS HSDIP 2019.*Revised review posted to your submission')
assert len(messages) == 3
recipients = [m['content']['to'] for m in messages]
assert '*****@*****.**' in recipients
assert '*****@*****.**' in recipients
assert '*****@*****.**' in recipients
messages = client.get_messages(subject = '[ICAPS HSDIP 2019] Your revised review has been received on your assigned Paper number: 1, Paper title: "Paper title"')
assert len(messages) == 1
recipients = [m['content']['to'] for m in messages]
assert '*****@*****.**' in recipients
def create_config_note(self):
'''
creates a config note but does not post it so that the sublcass override method can add additional stuff before posting happens.
:return:
'''
self.config_note = openreview.Note(
**{
'invitation': self.get_assignment_configuration_id(),
'readers': [self.conference.id],
'writers': [
self.conference.id,
],
'signatures': [self.conference.id],
'content': {
'title':
self.config_title,
Configuration.SCORES_SPECIFICATION:
self.params.scores_config[Params.SCORES_SPEC],
'max_users':
str(self.params.num_reviews_needed_per_paper
), # max number of reviewers a paper can have
'min_users':
str(self.params.num_reviews_needed_per_paper
), # min number of reviewers a paper can have
'max_papers':
str(self.params.reviewer_max_papers
), # max number of papers a reviewer can review
'min_papers':
'1', # min number of papers a reviewer can review
'alternates':
'2',
'constraints': {},
'custom_loads': {},
'config_invitation':
self.conf_ids.CONFIG_ID,
'paper_invitation':
self.conference.get_blind_submission_id(),
'metadata_invitation':
self.get_metadata_id(),
'assignment_invitation':
self.get_paper_assignment_id(),
'match_group':
self.conference.get_reviewers_id(),
'status':
'Initialized'
}
})
self.add_config_custom_loads()
self.add_config_constraints()
def test_venue_bid_stage(self, client, test_client, selenium, request_page,
helpers, venue):
reviewer_client = helpers.create_user('*****@*****.**',
'Venue', 'Reviewer')
reviewer_group_id = '{}/Reviewers'.format(venue['venue_id'])
reviewer_group = client.get_group(reviewer_group_id)
client.add_members_to_group(reviewer_group, '~Venue_Reviewer1')
reviewer_url = 'http://localhost:3030/group?id={}#reviewer-tasks'.format(
reviewer_group_id)
request_page(selenium, reviewer_url, reviewer_client.token)
with pytest.raises(NoSuchElementException):
assert selenium.find_element_by_link_text('Reviewer Bid')
now = datetime.datetime.utcnow()
start_date = now - datetime.timedelta(days=2)
due_date = now + datetime.timedelta(days=3)
bid_stage_note = test_client.post_note(
openreview.Note(content={
'bid_start_date': start_date.strftime('%Y/%m/%d'),
'bid_due_date': due_date.strftime('%Y/%m/%d')
},
forum=venue['request_form_note'].forum,
replyto=venue['request_form_note'].forum,
referent=venue['request_form_note'].forum,
invitation='{}/-/Request{}/Bid_Stage'.format(
venue['support_group_id'],
venue['request_form_note'].number),
readers=[
'{}/Program_Chairs'.format(venue['venue_id']),
venue['support_group_id']
],
signatures=['~Test_User1'],
writers=['~Test_User1']))
assert bid_stage_note
time.sleep(5)
process_logs = client.get_process_logs(id=bid_stage_note.id)
assert len(process_logs) == 1
assert process_logs[0][
'invitation'] == '{}/-/Request{}/Bid_Stage'.format(
venue['support_group_id'], venue['request_form_note'].number)
assert process_logs[0]['status'] == 'ok'
request_page(selenium, reviewer_url, reviewer_client.token)
assert selenium.find_element_by_link_text('Reviewer Bid')
def test_submission_additional_files(self, test_client):
pc_client = openreview.Client(username='******', password='******')
builder = openreview.conference.ConferenceBuilder(pc_client)
assert builder, 'builder is None'
builder.set_conference_id('thecvf.com/ECCV/2020/Conference')
builder.has_area_chairs(True)
now = datetime.datetime.utcnow()
builder.set_submission_stage(double_blind = True,
public = False,
due_date = now + datetime.timedelta(minutes = 10),
additional_fields= {
'video': {
'description': 'Short video with presentation of the paper, it supports: mov, mp4, zip',
'required': True,
'value-file': {
'fileTypes': ['mov', 'mp4', 'zip'],
'size': 50000000000
}
},
'supplemental_material': {
'description': 'Paper appendix',
'required': False,
'value-file': {
'fileTypes': ['pdf'],
'size': 500000000000
}
}
})
conference = builder.get_result()
note = openreview.Note(invitation = conference.get_submission_id(),
readers = ['thecvf.com/ECCV/2020/Conference', '*****@*****.**', '*****@*****.**', '*****@*****.**', '~Test_User1'],
writers = [conference.id, '~Test_User1', '*****@*****.**', '*****@*****.**'],
signatures = ['~Test_User1'],
content = {
'title': 'Paper title',
'abstract': 'This is an abstract',
'authorids': ['*****@*****.**', '*****@*****.**', '*****@*****.**'],
'authors': ['Test User', 'Peter Test', 'Andrew Mc']
}
)
url = test_client.put_attachment(os.path.join(os.path.dirname(__file__), 'data/paper.pdf'), conference.get_submission_id(), 'pdf')
note.content['pdf'] = url
url = test_client.put_attachment(os.path.join(os.path.dirname(__file__), 'data/paper.pdf.zip'), conference.get_submission_id(), 'video')
note.content['video'] = url
test_client.post_note(note)
def resubmit(client, old_papers_inv, new_papers_inv):
for paper in openreview.tools.iterget_notes(client, invitation=old_papers_inv):
new_content = paper.content
new_paper = openreview.Note(**{
'invitation': new_papers_inv,
'writers': paper.signatures,
'readers': [iclr19.CONFERENCE_ID],
'signatures': paper.signatures,
'content': new_content
})
try:
p = client.post_note(new_paper)
print("{} -> {}".format(paper.id, p.id))
except openreview.OpenReviewException as e:
pass
def create_cv_note(title, abstract, authors, authorids, keywords):
return openreview.Note(
**{
'readers': ['everyone'],
'writers': [],
'signatures': ['~Super_User1'],
'nonreaders': [],
'invitation': 'cv-foundation.org/ECCV/2018/Demo/-/Submission',
'content': {
'title': title.replace('\n', ''),
'abstract': abstract,
'authors': authors,
'authorids': authorids,
'keywords': keywords
}
})
def get_decision_note(pc_decision, comment, forum):
decision_note = openreview.Note(
**{
'forum': forum,
'replyto': forum,
'invitation': 'ICLR.cc/2018/Workshop/-/Acceptance_Decision',
'signatures': ['ICLR.cc/2018/Workshop/Program_Chairs'],
'writers': ['ICLR.cc/2018/Workshop/Program_Chairs'],
'readers': ['ICLR.cc/2018/Workshop/Program_Chairs'],
'content': {
'title': "ICLR 2018 Workshop Acceptance Decision",
'decision': pc_decision,
'comment': comment
}
})
return decision_note
