def get_data(invitation): CONF = 'ICLR.cc/2019/Conference' paper_inv = CONF + '/Paper.*' anon_reviewers = tools.iterget_groups(client, regex=paper_inv + '/AnonReviewer.*') current_reviewers = tools.iterget_groups(client, regex=paper_inv + '/Reviewers$') submissions = tools.iterget_notes(client, invitation=CONF + '/-/Blind_Submission') notes = tools.iterget_notes(client, invitation=CONF + '/-/Paper.*/' + invitation) papers = {} reviews = {} reviewers = {} reviewers_by_paper = {} for paper in submissions: papers[paper.number] = paper.id # reviews[reviewer name] = review note id for n in notes: signature = n.signatures[0] reviews[signature] = n.id # reviewers[paper_num+reviewer_name] = anonymous reviewer name for r in anon_reviewers: reviewer_id = r.id paper_number = reviewer_id.split('Paper')[1].split('/AnonReviewer')[0] members = r.members if members: reviewers[paper_number + '_' + members[0]] = reviewer_id # otherwise the reviewer was removed # reviewers_by_paper[paper_number][reviewer name]=review id for r in current_reviewers: reviewer_id = r.id members = r.members if members: paper_number = int( reviewer_id.split('Paper')[1].split('/Reviewers')[0]) # check if paper is current if paper_number in papers: if paper_number not in reviewers_by_paper: reviewers_by_paper[paper_number] = {} for m in members: reviewer_id = reviewers.get(str(paper_number) + '_' + m, m) reviewers_by_paper[paper_number][m] = reviews.get( reviewer_id, None) return reviewers_by_paper
'values': [sac_group] }, 'content': { 'outstanding_meta_review': { 'description': 'Please indicate if you consider this an outstanding meta review.', 'order': 1, 'required': True, 'value-radio': ['Yes', 'No'], 'default': 'No' } } } ) client.post_invitation(invitation) reviews = list(tools.iterget_notes(client, invitation='aclweb.org/ACL/2022/Conference/-/Official_Review', sort='number:asc')) print(f'{len(reviews)} posted reviews') for rev in tqdm(reviews): nonreaders = rev.nonreaders r = re.compile("aclweb.org/ACL/2022/Conference/.*/Senior_Area_Chairs") try: sac_group = list(filter(r.match, rev.readers))[0] create_review_rating_invitation(nonreaders, sac_group, rev) except IndexError: print('SACs are not readers of the review, paper ID: ', rev.forum) metareviews = list(tools.iterget_notes(client, invitation='aclweb.org/ACL/2022/Conference/-/Meta_Review', sort='number:asc')) print(f'{len(metareviews)} posted metareviews') for meta in tqdm(metareviews):
# We use this to add to the paper_reviewer_score note for the paper. We find the entry for that # user and add in a value of conflicts: 1 which is enough for the matcher to say that this user # has a conflict with this paper. # The file o-max-reviewers-per-paper.csv gives each reviewer followed by the max number of papers # they are willing to review. These are read in and are added to the config note as custom_loads # which inside the content as a dictionary # They also gave another file which we didn't read because we manually configure the config note # to have min_users and max_users on each paper of 3 with 5 alternates. client = openreview.Client(baseurl='http://openreview.localhost', username='******', password='******') print(client) cvpr_paper_notes = list(tools.iterget_notes(client, invitation=SUBMISSION_ID)) reviewers_group = client.get_group(REVIEWERS_ID) reviewers = reviewers_group.members print("Num papers", len(cvpr_paper_notes)) print("Num reviewers", len(reviewers)) # Create the invitation for the Paper-Reviewer_Scores note (used to be called meta-data) inv = openreview.Invitation( id = PAPER_REVIEWER_SCORE_ID, readers = ['everyone'], signatures = [CONF_ID], writers = [CONF_ID], invitees = [], reply = { 'forum': None, 'replyto': None,
args = parser.parse_args() ## Initialize the client library with username and password client = openreview.Client(baseurl=args.baseurl, username=args.username, password=args.password) print("Connecting to " + client.baseurl) conference = config.get_conference(client) conference.set_authors() print('replacing members with IDs') reviewers_group = client.get_group(conference.get_reviewers_id()) openreview.tools.replace_members_with_ids(client, reviewers_group) iterator = tools.iterget_notes(client, invitation=conference.get_submission_id()) for paper in iterator: paper_num = str(paper.number) paperinv = conference.get_id( ) + '/-/Paper' + paper_num + '/Official_Review' paperGroup = conference.get_id() + '/Paper' + paper_num ## Confidential Review review_reply = { 'forum': paper.id, 'replyto': paper.id, 'writers': { 'values-regex': paperGroup + '/AnonReviewer[0-9]+|~.*' }, 'signatures': { 'values-regex': paperGroup + '/AnonReviewer[0-9]+|~.*'
return author_email return None # load recent comments min_date = datetime.datetime.utcnow().timestamp() * 1000 if args.freq == "Daily": min_date -= 24 * 60 * 60 * 1000 elif args.freq == "Weekly": min_date -= 7 * 24 * 60 * 60 * 1000 else: print("Invalid freq: " + args.freq) quit() comments = tools.iterget_notes(client, mintcdate=min_date, invitation=conference_id + '/.*/-/Comment') comment_by_forum = {} for comment in comments: if comment.forum not in comment_by_forum: comment_by_forum[comment.forum] = [] comment_by_forum[comment.forum].append(comment) for forum in comment_by_forum.keys(): forumNote = client.get_note(id=forum) # get all notification tags for this paper tags = tools.iterget_tags(client, invitation=conference_id + '/-/Notification_Subscription', forum=forum, tag=args.freq)
return author_email return None # load recent comments min_date = datetime.datetime.utcnow().timestamp() * 1000 if args.freq == "Daily": min_date -= 24 * 60 * 60 * 1000 elif args.freq == "Weekly": min_date -= 7 * 24 * 60 * 60 * 1000 else: print("Invalid freq: " + args.freq) quit() comments = tools.iterget_notes(client, mintcdate=min_date, invitation=CONFERENCE_ID + '/.*/-/Comment') comment_by_forum = {} for comment in comments: if comment.forum not in comment_by_forum: comment_by_forum[comment.forum] = [] comment_by_forum[comment.forum].append(comment) for forum in comment_by_forum.keys(): forumNote = client.get_note(id=forum) # get all notification tags for this paper tags = tools.iterget_tags(client, invitation=CONFERENCE_ID + '/-/Notification_Subscription', forum=forum, tag=args.freq)
}, 'readers': { 'description': 'The users who will be allowed to read the above content.', 'values-copied': [conference_id, conference_id + '/Program_Chairs', '{signatures}'] }, 'writers': { 'values-copied': [conference_id, '{signatures}'] } }) client.post_invitation(notify_inv) # for each submission, create group (so sub invites can be published) # create invite for comment notes = tools.iterget_notes(client, invitation=conference_id + '/-/NeurIPS_Submission') for note in notes: # need paper group to publish sub-groups paper_group = client.post_group( openreview.Group(id='{conference_id}/{number}'.format( conference_id=conference_id, number=note.number), signatures=[conference_id], signatories=[conference_id], readers=[conference_id], writers=[conference_id])) # add comment invite comment_inv = openreview.Invitation( id=paper_group.id + '/-/Comment', readers=['everyone'],
import openreview from openreview import tools from cvpr2019 import * # Only after running the matcher successfully from within OR does this script get run. # It goes through the assignment notes that it creates and generates an XML file # which provides the list of reviewers for each paper submission. The output file is # ../data/reviewer_exports.xml client = openreview.Client(baseurl='http://openreview.localhost', username='******', password='') print(client) assignment_notes = list(tools.iterget_notes(client, invitation=ASSIGNMENT_ID)) print(len(assignment_notes)) print(assignment_notes[0]) ''' <?xml version="1.0"?> <assignments> <submission submissionId="1166"> <user email="*****@*****.**"/> <user email="*****@*****.**"/> <user email="*****@*****.**"/> </submission> ''' file = open(INPUT_FILES_DIR + "reviewers_export.xml", 'w') file.write("<?xml version=\"1.0\"?>\n<assignments>\n") assignment_notes = tools.iterget_notes(client, invitation=ASSIGNMENT_ID) i = 0 for assign_note in assignment_notes:
}, 'readers': { 'description': 'The users who will be allowed to read the above content.', 'values-copied': [CONFERENCE_ID, CONFERENCE_ID + '/Program_Chairs', '{signatures}'] }, 'writers': { 'values-copied': [CONFERENCE_ID, '{signatures}'] } }) client.post_invitation(notify_inv) # for each submission, create group (so sub invites can be published) # create invite for comment notes = tools.iterget_notes(client, invitation=ACCEPTED_PAPER_ID) for note in notes: print("Processing: " + str(note.number)) # need paper group to publish sub-groups paper_group = client.post_group( openreview.Group(id='{conference_id}/{number}'.format( conference_id=CONFERENCE_ID, number=note.number), signatures=[CONFERENCE_ID], signatories=[CONFERENCE_ID], readers=[CONFERENCE_ID], writers=[CONFERENCE_ID])) # add comment invite comment_inv = openreview.Invitation( id=paper_group.id + '/-/Comment',
username=args.username, password=args.password) print('connecting to {0}'.format(client.baseurl)) conference = config.get_conference(client) print('make reviews public') invitations = tools.iterget_invitations(client, regex=conference.get_id() + '/-/Paper.*/Official_Review') for invite in invitations: invite.reply['readers']['values'] = ['everyone'] client.post_invitation(invite) reviews = tools.iterget_notes(client, invitation=conference.get_id() + '/-/Paper.*/Official_Review') for review in reviews: review.readers = ['everyone'] client.post_note(review) print('open comment invitations') conference.set_authors() conference.open_comments(name='Official_Comment', public=False, anonymous=True) print('replacing members with IDs') reviewers_group = client.get_group(conference.get_reviewers_id()) areachairs_group = client.get_group(conference.get_area_chairs_id()) openreview.tools.replace_members_with_ids(client, reviewers_group) openreview.tools.replace_members_with_ids(client, areachairs_group)
n_papers = 0 paper_note_ids = [] for paper in papers: n_papers += 1 content = { 'title': "Paper-" + paper, 'number': paper, } posted_submission = client.post_note( openreview.Note( **{ 'signatures': [CONF_ID], 'writers': [CONF_ID], 'readers': [CONF_ID], 'content': content, 'invitation': SUBMISSION_ID })) paper_note_ids.append(posted_submission.id) print(len(paper_note_ids)) # Final results: reviewers in a group and paper notes. reviewers_group = client.get_group(REVIEWERS_ID) print(len(reviewers_group.members), "reviewers") cvpr_papers = list(tools.iterget_notes(client, invitation=SUBMISSION_ID)) print(len(cvpr_papers), "papers") ## Should see # 2840 reviewers # 5062 papers