def get_data(invitation):
CONF = 'ICLR.cc/2019/Conference'
paper_inv = CONF + '/Paper.*'
anon_reviewers = tools.iterget_groups(client,
regex=paper_inv + '/AnonReviewer.*')
current_reviewers = tools.iterget_groups(client,
regex=paper_inv + '/Reviewers$')
submissions = tools.iterget_notes(client,
invitation=CONF + '/-/Blind_Submission')
notes = tools.iterget_notes(client,
invitation=CONF + '/-/Paper.*/' + invitation)
papers = {}
reviews = {}
reviewers = {}
reviewers_by_paper = {}
for paper in submissions:
papers[paper.number] = paper.id
# reviews[reviewer name] = review note id
for n in notes:
signature = n.signatures[0]
reviews[signature] = n.id
# reviewers[paper_num+reviewer_name] = anonymous reviewer name
for r in anon_reviewers:
reviewer_id = r.id
paper_number = reviewer_id.split('Paper')[1].split('/AnonReviewer')[0]
members = r.members
if members:
reviewers[paper_number + '_' + members[0]] = reviewer_id
# otherwise the reviewer was removed
# reviewers_by_paper[paper_number][reviewer name]=review id
for r in current_reviewers:
reviewer_id = r.id
members = r.members
if members:
paper_number = int(
reviewer_id.split('Paper')[1].split('/Reviewers')[0])
# check if paper is current
if paper_number in papers:
if paper_number not in reviewers_by_paper:
reviewers_by_paper[paper_number] = {}
for m in members:
reviewer_id = reviewers.get(str(paper_number) + '_' + m, m)
reviewers_by_paper[paper_number][m] = reviews.get(
reviewer_id, None)
return reviewers_by_paper
'values': [sac_group]
},
'content': {
'outstanding_meta_review': {
'description': 'Please indicate if you consider this an outstanding meta review.',
'order': 1,
'required': True,
'value-radio': ['Yes', 'No'],
'default': 'No'
}
}
}
)
client.post_invitation(invitation)
reviews = list(tools.iterget_notes(client, invitation='aclweb.org/ACL/2022/Conference/-/Official_Review', sort='number:asc'))
print(f'{len(reviews)} posted reviews')
for rev in tqdm(reviews):
nonreaders = rev.nonreaders
r = re.compile("aclweb.org/ACL/2022/Conference/.*/Senior_Area_Chairs")
try:
sac_group = list(filter(r.match, rev.readers))[0]
create_review_rating_invitation(nonreaders, sac_group, rev)
except IndexError:
print('SACs are not readers of the review, paper ID: ', rev.forum)
metareviews = list(tools.iterget_notes(client, invitation='aclweb.org/ACL/2022/Conference/-/Meta_Review', sort='number:asc'))
print(f'{len(metareviews)} posted metareviews')
for meta in tqdm(metareviews):
# We use this to add to the paper_reviewer_score note for the paper. We find the entry for that
# user and add in a value of conflicts: 1 which is enough for the matcher to say that this user
# has a conflict with this paper.
# The file o-max-reviewers-per-paper.csv gives each reviewer followed by the max number of papers
# they are willing to review. These are read in and are added to the config note as custom_loads
# which inside the content as a dictionary
# They also gave another file which we didn't read because we manually configure the config note
# to have min_users and max_users on each paper of 3 with 5 alternates.
client = openreview.Client(baseurl='http://openreview.localhost', username='******', password='******')
print(client)
cvpr_paper_notes = list(tools.iterget_notes(client, invitation=SUBMISSION_ID))
reviewers_group = client.get_group(REVIEWERS_ID)
reviewers = reviewers_group.members
print("Num papers", len(cvpr_paper_notes))
print("Num reviewers", len(reviewers))
# Create the invitation for the Paper-Reviewer_Scores note (used to be called meta-data)
inv = openreview.Invitation(
id = PAPER_REVIEWER_SCORE_ID,
readers = ['everyone'],
signatures = [CONF_ID],
writers = [CONF_ID],
invitees = [],
reply = {
'forum': None,
'replyto': None,
args = parser.parse_args()
## Initialize the client library with username and password
client = openreview.Client(baseurl=args.baseurl,
username=args.username,
password=args.password)
print("Connecting to " + client.baseurl)
conference = config.get_conference(client)
conference.set_authors()
print('replacing members with IDs')
reviewers_group = client.get_group(conference.get_reviewers_id())
openreview.tools.replace_members_with_ids(client, reviewers_group)
iterator = tools.iterget_notes(client,
invitation=conference.get_submission_id())
for paper in iterator:
paper_num = str(paper.number)
paperinv = conference.get_id(
) + '/-/Paper' + paper_num + '/Official_Review'
paperGroup = conference.get_id() + '/Paper' + paper_num
## Confidential Review
review_reply = {
'forum': paper.id,
'replyto': paper.id,
'writers': {
'values-regex': paperGroup + '/AnonReviewer[0-9]+|~.*'
},
'signatures': {
'values-regex': paperGroup + '/AnonReviewer[0-9]+|~.*'
return author_email
return None
# load recent comments
min_date = datetime.datetime.utcnow().timestamp() * 1000
if args.freq == "Daily":
min_date -= 24 * 60 * 60 * 1000
elif args.freq == "Weekly":
min_date -= 7 * 24 * 60 * 60 * 1000
else:
print("Invalid freq: " + args.freq)
quit()
comments = tools.iterget_notes(client,
mintcdate=min_date,
invitation=conference_id + '/.*/-/Comment')
comment_by_forum = {}
for comment in comments:
if comment.forum not in comment_by_forum:
comment_by_forum[comment.forum] = []
comment_by_forum[comment.forum].append(comment)
for forum in comment_by_forum.keys():
forumNote = client.get_note(id=forum)
# get all notification tags for this paper
tags = tools.iterget_tags(client,
invitation=conference_id +
'/-/Notification_Subscription',
forum=forum,
tag=args.freq)
return author_email
return None
# load recent comments
min_date = datetime.datetime.utcnow().timestamp() * 1000
if args.freq == "Daily":
min_date -= 24 * 60 * 60 * 1000
elif args.freq == "Weekly":
min_date -= 7 * 24 * 60 * 60 * 1000
else:
print("Invalid freq: " + args.freq)
quit()
comments = tools.iterget_notes(client,
mintcdate=min_date,
invitation=CONFERENCE_ID + '/.*/-/Comment')
comment_by_forum = {}
for comment in comments:
if comment.forum not in comment_by_forum:
comment_by_forum[comment.forum] = []
comment_by_forum[comment.forum].append(comment)
for forum in comment_by_forum.keys():
forumNote = client.get_note(id=forum)
# get all notification tags for this paper
tags = tools.iterget_tags(client,
invitation=CONFERENCE_ID +
'/-/Notification_Subscription',
forum=forum,
tag=args.freq)
},
'readers': {
'description':
'The users who will be allowed to read the above content.',
'values-copied':
[conference_id, conference_id + '/Program_Chairs', '{signatures}']
},
'writers': {
'values-copied': [conference_id, '{signatures}']
}
})
client.post_invitation(notify_inv)
# for each submission, create group (so sub invites can be published)
# create invite for comment
notes = tools.iterget_notes(client,
invitation=conference_id + '/-/NeurIPS_Submission')
for note in notes:
# need paper group to publish sub-groups
paper_group = client.post_group(
openreview.Group(id='{conference_id}/{number}'.format(
conference_id=conference_id, number=note.number),
signatures=[conference_id],
signatories=[conference_id],
readers=[conference_id],
writers=[conference_id]))
# add comment invite
comment_inv = openreview.Invitation(
id=paper_group.id + '/-/Comment',
readers=['everyone'],
import openreview
from openreview import tools
from cvpr2019 import *
# Only after running the matcher successfully from within OR does this script get run.
# It goes through the assignment notes that it creates and generates an XML file
# which provides the list of reviewers for each paper submission. The output file is
# ../data/reviewer_exports.xml
client = openreview.Client(baseurl='http://openreview.localhost',
username='******',
password='')
print(client)
assignment_notes = list(tools.iterget_notes(client, invitation=ASSIGNMENT_ID))
print(len(assignment_notes))
print(assignment_notes[0])
'''
<?xml version="1.0"?>
<assignments>
<submission submissionId="1166">
<user email="*****@*****.**"/>
<user email="*****@*****.**"/>
<user email="*****@*****.**"/>
</submission>
'''
file = open(INPUT_FILES_DIR + "reviewers_export.xml", 'w')
file.write("<?xml version=\"1.0\"?>\n<assignments>\n")
assignment_notes = tools.iterget_notes(client, invitation=ASSIGNMENT_ID)
i = 0
for assign_note in assignment_notes:
},
'readers': {
'description':
'The users who will be allowed to read the above content.',
'values-copied':
[CONFERENCE_ID, CONFERENCE_ID + '/Program_Chairs', '{signatures}']
},
'writers': {
'values-copied': [CONFERENCE_ID, '{signatures}']
}
})
client.post_invitation(notify_inv)
# for each submission, create group (so sub invites can be published)
# create invite for comment
notes = tools.iterget_notes(client, invitation=ACCEPTED_PAPER_ID)
for note in notes:
print("Processing: " + str(note.number))
# need paper group to publish sub-groups
paper_group = client.post_group(
openreview.Group(id='{conference_id}/{number}'.format(
conference_id=CONFERENCE_ID, number=note.number),
signatures=[CONFERENCE_ID],
signatories=[CONFERENCE_ID],
readers=[CONFERENCE_ID],
writers=[CONFERENCE_ID]))
# add comment invite
comment_inv = openreview.Invitation(
id=paper_group.id + '/-/Comment',
username=args.username,
password=args.password)
print('connecting to {0}'.format(client.baseurl))
conference = config.get_conference(client)
print('make reviews public')
invitations = tools.iterget_invitations(client,
regex=conference.get_id() +
'/-/Paper.*/Official_Review')
for invite in invitations:
invite.reply['readers']['values'] = ['everyone']
client.post_invitation(invite)
reviews = tools.iterget_notes(client,
invitation=conference.get_id() +
'/-/Paper.*/Official_Review')
for review in reviews:
review.readers = ['everyone']
client.post_note(review)
print('open comment invitations')
conference.set_authors()
conference.open_comments(name='Official_Comment', public=False, anonymous=True)
print('replacing members with IDs')
reviewers_group = client.get_group(conference.get_reviewers_id())
areachairs_group = client.get_group(conference.get_area_chairs_id())
openreview.tools.replace_members_with_ids(client, reviewers_group)
openreview.tools.replace_members_with_ids(client, areachairs_group)
n_papers = 0
paper_note_ids = []
for paper in papers:
n_papers += 1
content = {
'title': "Paper-" + paper,
'number': paper,
}
posted_submission = client.post_note(
openreview.Note(
**{
'signatures': [CONF_ID],
'writers': [CONF_ID],
'readers': [CONF_ID],
'content': content,
'invitation': SUBMISSION_ID
}))
paper_note_ids.append(posted_submission.id)
print(len(paper_note_ids))
# Final results: reviewers in a group and paper notes.
reviewers_group = client.get_group(REVIEWERS_ID)
print(len(reviewers_group.members), "reviewers")
cvpr_papers = list(tools.iterget_notes(client, invitation=SUBMISSION_ID))
print(len(cvpr_papers), "papers")
## Should see
# 2840 reviewers
# 5062 papers