def get_benchmark(ctx, benchmark):
"""Show benchmark information."""
b_id = benchmark if benchmark else config.BENCHMARK_ID()
if b_id is None:
click.echo('no benchmark specified')
return
url = ctx.obj['URLS'].get_benchmark(b_id)
try:
r = requests.get(url)
r.raise_for_status()
body = r.json()
if ctx.obj['RAW']:
click.echo(json.dumps(body, indent=4))
else:
click.echo('{} ({})'.format(body['name'], body['id']))
if 'description' in body:
click.echo('\n{}'.format(body['description']))
if 'instructions' in body:
click.echo('\n{}'.format(body['instructions']))
click.echo('\nParameters:')
for p in body.get('parameters', list()):
name = p['name']
data_type = p['type']
click.echo(' {} ({})'.format(name, data_type))
resources = body.get('postproc', dict()).get('resources')
if resources is not None:
click.echo('\nResources:')
for res in resources:
r_id = res['id']
r_name = res['name']
click.echo(' {} ({})'.format(r_name, r_id))
except (requests.ConnectionError, requests.HTTPError) as ex:
click.echo('{}'.format(ex))
def list_submissions(ctx, benchmark):
"""Show submissions for a benchmark or user."""
b_id = benchmark if benchmark else config.BENCHMARK_ID()
url = ctx.obj['URLS'].list_submissions(benchmark_id=b_id)
headers = ctx.obj['HEADERS']
try:
r = requests.get(url, headers=headers)
r.raise_for_status()
body = r.json()
if ctx.obj['RAW']:
click.echo(json.dumps(body, indent=4))
else:
table = ResultTable(['ID', 'Name'], [PARA_STRING] * 2)
for s in body['submissions']:
table.add([s['id'], s['name']])
for line in table.format():
click.echo(line)
except (requests.ConnectionError, requests.HTTPError) as ex:
click.echo('{}'.format(ex))
def download_resource(ctx, benchmark, resource, all, output):
"""Download a run resource file."""
# We cannot have a resource and the all flag being True
if resource is not None and all:
click.echo('invalid argument combination')
return
elif resource is None and not all:
click.echo('select resource or all')
return
b_id = benchmark if benchmark else config.BENCHMARK_ID()
urls = ctx.obj['URLS']
if resource is not None:
url = urls.download_benchmark_file(benchmark_id=b_id,
resource_id=resource)
else:
url = urls.download_benchmark_archive(benchmark_id=b_id)
headers = ctx.obj['HEADERS']
try:
r = requests.get(url, headers=headers)
r.raise_for_status()
content = r.headers['Content-Disposition']
if output is not None:
filename = output
elif 'filename=' in content:
filename = content[content.find('filename='):].split('=')[1]
if filename.startswith('"') or filename.startswith("'"):
filename = filename[1:]
if filename.endswith('"') or filename.endswith("'"):
filename = filename[:-1]
else:
click.echo('not output filename found')
return
# Write the file contents in the response to the specified path
# Based on https://www.techcoil.com/blog/how-to-download-a-file-via-http-post-and-http-get-with-python-3-requests-library/ # noqa: E501
targetdir = os.path.dirname(filename)
if targetdir:
util.create_dir(targetdir)
with open(filename, 'wb') as local_file:
for chunk in r.iter_content(chunk_size=128):
local_file.write(chunk)
except (requests.ConnectionError, requests.HTTPError) as ex:
click.echo('{}'.format(ex))
def create_submission(ctx, benchmark, name, members):
"""Create a new submission."""
b_id = benchmark if benchmark else config.BENCHMARK_ID()
if b_id is None:
click.echo('no benchmark specified')
return
url = ctx.obj['URLS'].create_submission(benchmark_id=b_id)
headers = ctx.obj['HEADERS']
data = {'name': name}
if members is not None:
data['members'] = members.split(',')
try:
r = requests.post(url, json=data, headers=headers)
r.raise_for_status()
body = r.json()
if ctx.obj['RAW']:
click.echo(json.dumps(body, indent=4))
else:
s_id = body['id']
click.echo('export {}={}'.format(config.ROB_SUBMISSION, s_id))
except (requests.ConnectionError, requests.HTTPError) as ex:
click.echo('{}'.format(ex))
def get_leaderboard(ctx, benchmark, all):
"""Show benchmark leaderboard."""
b_id = benchmark if benchmark else config.BENCHMARK_ID()
if b_id is None:
click.echo('no benchmark specified')
return
url = ctx.obj['URLS'].get_leaderboard(b_id, include_all=all)
headers = ctx.obj['HEADERS']
try:
r = requests.get(url, headers=headers)
r.raise_for_status()
body = r.json()
if ctx.obj['RAW']:
click.echo(json.dumps(body, indent=4))
else:
headline = ['Rank', 'Submission']
types = NUMERIC_TYPES
for col in body['schema']:
headline.append(col['name'])
types.append(col['type'])
table = ResultTable(headline=headline, types=types)
rank = 1
for run in body['ranking']:
row = [str(rank), run['submission']['name']]
result = dict()
for val in run['results']:
result[val['id']] = val['value']
for col in body['schema']:
col_id = col['id']
row.append(str(result[col_id]))
table.add(row)
rank += 1
for line in table.format():
click.echo(line)
except (requests.ConnectionError, requests.HTTPError) as ex:
click.echo('{}'.format(ex))