def update_status(syn: Synapse, queue_info: pd.Series):
"""If internal submission is invalid, then make update main leaderboard
with site_submission_status to INVALID
Args:
syn: Synapse connection
queue_info: One row of queue mapping information
{"main": main queue
"internal": internal queue
"site": site}
"""
# Get submissions that are processing in internal queues
processing_subs = (
f"select objectId from evaluation_{queue_info['main']} where "
f"{queue_info['site']}_submission_status == 'EVALUATION_IN_PROGRESS'")
processing_submissions = list(evaluation_queue_query(syn, processing_subs))
# For all the submisisons that are processing, obtain the status in
# the internal queues. Make main submission invalid.
for sub in processing_submissions:
internal_query_str = (
f"select name from evaluation_{queue_info['internal']} where "
f"status == 'INVALID' and name == '{sub['objectId']}'")
internal_subs = list(evaluation_queue_query(syn, internal_query_str))
if internal_subs:
internal_status = {
f"{queue_info['site']}_submission_status": "INVALID"
}
annotate_with_retry(syn=syn,
submissionid=internal_subs[0]['name'],
annotation_dict=internal_status,
is_private=False)
def _get_invalid_submissions(syn, queueid):
"""Get a submission queue's invalid submissions
Args:
syn: Synapse connection
queueid: Evaluation queue id
Returns:
list of invalid submissions
"""
query_str = (f"select name from evaluation_{queueid} where "
"status == 'INVALID'")
submissions = list(evaluation_queue_query(syn, query_str))
invalid_submissions = [sub['name'] for sub in submissions]
return invalid_submissions
def convert_overall_status(syn: Synapse, main_queueid: str, sites: list):
"""If all internal sites have INVALID status, make main status REJECTED
"""
# Format site query str
site_status_keys = [
f"{site}_submission_status == 'INVALID'" for site in sites
]
site_strs = " and ".join(site_status_keys)
# Get submissions that have all sites that are invalid
query_str = (f"select objectId from evaluation_{main_queueid} where "
f"{site_strs} and status != 'REJECTED'")
print(query_str)
invalid_subs = list(evaluation_queue_query(syn, query_str))
for invalid_sub in invalid_subs:
print(invalid_sub['objectId'])
annotate_with_retry(syn=syn,
submissionid=invalid_sub['objectId'],
status="REJECTED")
def get_score_results(syn, main_queue, internal_queue, site=None):
"""Get score results for site"""
print("Getting score results")
dataset_mapping = syn.tableQuery(
f"SELECT * FROM syn22093564 where queue = '{internal_queue}'")
dataset_mappingdf = dataset_mapping.asDataFrame()
if dataset_mappingdf.empty:
raise ValueError("No queue id")
dataset_mappingdf = dataset_mappingdf.fillna("")
# Get dataset info
dataset_info = dataset_mappingdf.to_dict("records")[0]
train = dataset_info['train_dataset_version']
infer = dataset_info['infer_dataset_version']
query = (f"select * from evaluation_{main_queue} where "
f"{site}_train_dataset_version == '{train}' and "
f"{site}_infer_dataset_version == '{infer}' and "
f"{site}_submission_status == 'SCORED'")
results = list(evaluation_queue_query(syn, query))
resultsdf = pd.DataFrame(results)
return resultsdf
def main(fanout_queue, main_queues):
"""Invoke REJECTION"""
syn = synapseclient.login()
# Get fanout queue validated submissions
query_str = (f"select objectId from evaluation_{fanout_queue} where "
"prediction_file_status == 'VALIDATED' "
"and status == 'ACCEPTED'")
fanout_submissions = list(evaluation_queue_query(syn, query_str))
# Get invalid main queue submissions
invalid_submissions = []
for main_queue in main_queues:
invalid_submissions.extend(_get_invalid_submissions(syn, main_queue))
for sub in fanout_submissions:
fanout_subid = sub['objectId']
# If fanout submission id matches a invalid submission
# REJECT the fanout queue submission
if fanout_subid in invalid_submissions:
print(fanout_subid)
change_submission_status(syn, fanout_subid, "REJECTED")
def count_submissions(syn, evaluations, start=None, end=None):
"""Tally up all valid submissions from given evaluations.
Returns:
total: total number of submissions from challenge queues
"""
if start:
start = convert_to_epoch(start)
if end:
end = convert_to_epoch(end)
total = 0
for eval_id in evaluations:
query = f"SELECT * FROM evaluation_{eval_id} WHERE status == 'ACCEPTED'"
if start:
query += f" AND createdOn >= {start}"
if end:
query += f" AND createdOn <= {end}"
total += sum(1 for sub in utils.evaluation_queue_query(
syn, query, limit=13))
return total
def command_query(syn, args):
querydf = pd.DataFrame(list(utils.evaluation_queue_query(syn, args.uri, args.limit, args.offset)))
print(querydf.to_csv(index=False))