def score(evaluation, dry_run=False):
if type(evaluation) != Evaluation:
evaluation = syn.getEvaluation(evaluation)
print '\n\nScoring ', evaluation.id, evaluation.name
print "-" * 60
sys.stdout.flush()
for submission, status in syn.getSubmissionBundles(evaluation, status='VALIDATED'):
status.status = "INVALID"
## refetch the submission so that we get the file path
## to be later replaced by a "downloadFiles" flag on getSubmissionBundles
submission = syn.getSubmission(submission)
try:
score, message = conf.score_submission(evaluation, submission)
print "scored:", submission.id, submission.name, submission.userId, score
## fill in team in submission status annotations
if 'teamId' in submission:
team = syn.restGET('/team/{id}'.format(id=submission.teamId))
if 'name' in team:
score['team'] = team['name']
else:
score['team'] = submission.teamId
elif 'userId' in submission:
profile = syn.getUserProfile(submission.userId)
score['team'] = get_user_name(profile)
else:
score['team'] = '?'
add_annotations = synapseclient.annotations.to_submission_status_annotations(score,is_private=True)
for i in add_annotations:
if status.annotations.get(i) is not None:
status.annotations[i].extend(add_annotations[i])
else:
status.annotations[i] = add_annotations[i]
#status.annotations = synapseclient.annotations.to_submission_status_annotations(score,is_private=True)
status.status = "SCORED"
### Add in DATE as a public annotation and change team annotation to not private
## if there's a table configured, update it
if not dry_run and evaluation.id in conf.leaderboard_tables:
update_leaderboard_table(conf.leaderboard_tables[evaluation.id], submission, fields=score, dry_run=False)
except Exception as ex1:
sys.stderr.write('\n\nError scoring submission %s %s:\n' % (submission.name, submission.id))
st = StringIO()
traceback.print_exc(file=st)
sys.stderr.write(st.getvalue())
sys.stderr.write('\n')
message = st.getvalue()
if conf.ADMIN_USER_IDS:
submission_info = "submission id: %s\nsubmission name: %s\nsubmitted by user id: %s\n\n" % (submission.id, submission.name, submission.userId)
messages.error_notification(userIds=conf.ADMIN_USER_IDS, message=submission_info+st.getvalue())
if not dry_run:
status = syn.store(status)
## send message AFTER storing status to ensure we don't get repeat messages
profile = syn.getUserProfile(submission.userId)
if status.status == 'SCORED':
messages.scoring_succeeded(
userIds=[submission.userId],
message=message,
username=get_user_name(profile),
queue_name=evaluation.name,
submission_name=submission.name,
submission_id=submission.id)
else:
messages.scoring_error(
userIds=[submission.userId],
message=message,
username=get_user_name(profile),
queue_name=evaluation.name,
submission_name=submission.name,
submission_id=submission.id)
sys.stdout.write('\n')
def parallel_score(submissionId,evaluation,dry_run):
submission = syn.getSubmission(submissionId)
status = syn.getSubmissionStatus(submissionId)
logFile = open(os.path.join(LOG_DIR,status['id'] + "_log.txt",'w'))
status.status = "INVALID"
try:
score, message = conf.score_submission(evaluation, submission)
logFile.write("scored: %s %s %s %s" % (submission.id,submission.name,submission.userId,str(score)))
logFile.flush()
## fill in team in submission status annotations
if 'teamId' in submission:
team = syn.restGET('/team/{id}'.format(id=submission.teamId))
if 'name' in team:
score['team'] = team['name']
else:
score['team'] = submission.teamId
elif 'userId' in submission:
profile = syn.getUserProfile(submission.userId)
score['team'] = get_user_name(profile)
else:
score['team'] = '?'
add_annotations = synapseclient.annotations.to_submission_status_annotations(score,is_private=True)
status = update_single_submission_status(status, add_annotations)
status.status = "SCORED"
### Add in DATE as a public annotation and change team annotation to not private
## if there's a table configured, update it
if not dry_run and evaluation.id in conf.leaderboard_tables:
update_leaderboard_table(conf.leaderboard_tables[evaluation.id], submission, fields=score, dry_run=False)
except Exception as ex1:
logFile.write('\n\nError scoring submission %s %s:\n' % (submission.name, submission.id))
st = StringIO()
traceback.print_exc(file=st)
# sys.stderr.write(st.getvalue())
# sys.stderr.write('\n')
message = st.getvalue()
logFile.write(message)
logFile.flush()
if conf.ADMIN_USER_IDS:
submission_info = "submission id: %s\nsubmission name: %s\nsubmitted by user id: %s\n\n" % (submission.id, submission.name, submission.userId)
messages.error_notification(userIds=conf.ADMIN_USER_IDS, message=submission_info+message)
if not dry_run:
status = syn.store(status)
## send message AFTER storing status to ensure we don't get repeat messages
profile = syn.getUserProfile(submission.userId)
if status.status == 'SCORED':
messages.scoring_succeeded(
userIds=[submission.userId],
message=message,
username=get_user_name(profile),
queue_name=evaluation.name,
submission_name=submission.name,
submission_id=submission.id)
else:
messages.scoring_error(
userIds=conf.ADMIN_USER_IDS,
message=message,
username="******",
queue_name=evaluation.name,
submission_name=submission.name,
submission_id=submission.id)
#sys.stdout.write('\n')
logFile.close()
def score(evaluation, dry_run=False):
if type(evaluation) != Evaluation:
evaluation = syn.getEvaluation(evaluation)
print '\n\nScoring ', evaluation.id, evaluation.name
print "-" * 60
sys.stdout.flush()
for submission, status in syn.getSubmissionBundles(evaluation, status='VALIDATED'):
status.status = "INVALID"
## refetch the submission so that we get the file path
## to be later replaced by a "downloadFiles" flag on getSubmissionBundles
submission = syn.getSubmission(submission)
try:
score, message = conf.score_submission(evaluation, submission)
print "scored:", submission.id, submission.name, submission.userId, score
## fill in team in submission status annotations
if 'teamId' in submission:
team = syn.restGET('/team/{id}'.format(id=submission.teamId))
if 'name' in team:
score['team'] = team['name']
else:
score['team'] = submission.teamId
elif 'userId' in submission:
profile = syn.getUserProfile(submission.userId)
score['team'] = get_user_name(profile)
else:
score['team'] = '?'
status.annotations = synapseclient.annotations.to_submission_status_annotations(score,is_private=True)
status.status = "SCORED"
## if there's a table configured, update it
if not dry_run and evaluation.id in conf.leaderboard_tables:
update_leaderboard_table(conf.leaderboard_tables[evaluation.id], submission, fields=score, dry_run=False)
except Exception as ex1:
sys.stderr.write('\n\nError scoring submission %s %s:\n' % (submission.name, submission.id))
st = StringIO()
traceback.print_exc(file=st)
sys.stderr.write(st.getvalue())
sys.stderr.write('\n')
message = st.getvalue()
if conf.ADMIN_USER_IDS:
submission_info = "submission id: %s\nsubmission name: %s\nsubmitted by user id: %s\n\n" % (submission.id, submission.name, submission.userId)
messages.error_notification(userIds=conf.ADMIN_USER_IDS, message=submission_info+st.getvalue())
if not dry_run:
status = syn.store(status)
## send message AFTER storing status to ensure we don't get repeat messages
profile = syn.getUserProfile(submission.userId)
if status.status == 'SCORED':
messages.scoring_succeeded(
userIds=[submission.userId],
message=message,
username=get_user_name(profile),
queue_name=evaluation.name,
submission_name=submission.name,
submission_id=submission.id)
else:
messages.scoring_failed(
userIds=[submission.userId],
message=message,
username=get_user_name(profile),
queue_name=evaluation.name,
submission_name=submission.name,
submission_id=submission.id)
sys.stdout.write('\n')