def test_delete_submission(client, benchmark_id):
"""Test deleting a previously created submission."""
# -- Setup ----------------------------------------------------------------
# Create two users that are logged in.
user_1, token_1 = create_user(client, '0000')
user_2, token_2 = create_user(client, '0001')
headers_1 = {HEADER_TOKEN: token_1}
headers_2 = {HEADER_TOKEN: token_2}
# Create one submission for user 1.
url = CREATE_SUBMISSION.format(config.API_PATH(), benchmark_id)
r = client.post(url, json={labels.GROUP_NAME: 'S1'}, headers=headers_1)
submission_id = r.json[labels.GROUP_ID]
# -- Delete submission ----------------------------------------------------
url = ACCESS_SUBMISSION.format(config.API_PATH(), submission_id)
r = client.get(url, headers=headers_1)
assert r.status_code == 200
# User 2 cannot delete the submission.
url = DELETE_SUBMISSION.format(config.API_PATH(), submission_id)
r = client.delete(url, headers=headers_2)
assert r.status_code == 403
# User 1 can delete the submission.
r = client.delete(url, headers=headers_1)
assert r.status_code == 204
# Deleting an unknown submission results in statis 404
r = client.delete(url, headers=headers_1)
assert r.status_code == 404
def test_update_submission(client, benchmark_id):
"""Test updating properties of a submission."""
# -- Setup ----------------------------------------------------------------
# Create two users that are logged in.
user_1, token_1 = create_user(client, '0000')
user_2, token_2 = create_user(client, '0001')
headers_1 = {HEADER_TOKEN: token_1}
headers_2 = {HEADER_TOKEN: token_2}
# Create one submission for user 1.
url = CREATE_SUBMISSION.format(config.API_PATH(), benchmark_id)
r = client.post(url, json={labels.GROUP_NAME: 'S1'}, headers=headers_1)
submission_id = r.json[labels.GROUP_ID]
# -- Update submission ----------------------------------------------------
url = UPDATE_SUBMISSION.format(config.API_PATH(), submission_id)
body = {labels.GROUP_NAME: 'S1 (updated)'}
r = client.put(url, json=body, headers=headers_2)
assert r.status_code == 403
r = client.put(url, json=body, headers=headers_1)
assert r.status_code == 200
assert r.json[labels.GROUP_NAME] == 'S1 (updated)'
# After adding user 2 as a member they can update the submission as well.
body = {labels.GROUP_MEMBERS: [user_1, user_2]}
r = client.put(url, json=body, headers=headers_1)
assert r.status_code == 200
body = {labels.GROUP_NAME: 'S1 (new)'}
r = client.put(url, json=body, headers=headers_2)
assert r.status_code == 200
assert r.json[labels.GROUP_NAME] == 'S1 (new)'
def test_uploads_listings(client, benchmark_id):
"""Tests uploading and retrieving a file."""
# -- Setup ----------------------------------------------------------------
# Create new user and submission. Then upload a single file.
user_1, token_1 = create_user(client, '0000')
headers = {HEADER_TOKEN: token_1}
url = CREATE_SUBMISSION.format(config.API_PATH(), benchmark_id)
r = client.post(url, json={labels.GROUP_NAME: 'S1'}, headers=headers)
submission_id = r.json[labels.GROUP_ID]
data = dict()
with open(SMALL_FILE, 'rb') as f:
data['file'] = (io.BytesIO(f.read()), 'names.txt')
url = SUBMISSION_FILES.format(config.API_PATH(), submission_id)
r = client.post(
url,
data=data,
content_type='multipart/form-data',
headers=headers
)
assert r.status_code == 201
file_id = r.json[flbls.FILE_ID]
# -- Listing uploaded files -----------------------------------------------
r = client.get(url, headers=headers)
assert r.status_code == 200
doc = r.json
assert len(doc[flbls.FILE_LIST]) == 1
# Delete the file.
url = SUBMISSION_FILE.format(config.API_PATH(), submission_id, file_id)
r = client.delete(url, headers=headers)
assert r.status_code == 204
url = SUBMISSION_FILES.format(config.API_PATH(), submission_id)
r = client.get(url, headers=headers)
assert r.status_code == 200
doc = r.json
assert len(doc[flbls.FILE_LIST]) == 0
def test_list_submissions(client, benchmark_id):
"""Test to retrieve a list of submissions that a user is a member of."""
# -- Setup ----------------------------------------------------------------
# Create two users that are logged in.
user_1, token_1 = create_user(client, '0000')
user_2, token_2 = create_user(client, '0001')
headers_1 = {HEADER_TOKEN: token_1}
headers_2 = {HEADER_TOKEN: token_2}
# Create two submissions. User 1 is member of both submissions but user 2
# is only a member of one submission.
url = CREATE_SUBMISSION.format(config.API_PATH(), benchmark_id)
client.post(url, json={labels.GROUP_NAME: 'S1'}, headers=headers_1)
client.post(url,
json={
labels.GROUP_NAME: 'S2',
labels.GROUP_MEMBERS: [user_1]
},
headers=headers_2)
# -- Retrieve submission listings ------------------------------------------
url = SUBMISSION_LIST.format(config.API_PATH(), benchmark_id)
r = client.get(url, headers=headers_1)
assert r.status_code == 200
doc = r.json
assert len(doc[labels.GROUP_LIST]) == 2
# Retureving an unknown submission results in stats 404.
r = client.get(url, headers=headers_2)
assert r.status_code == 200
doc = r.json
assert len(doc[labels.GROUP_LIST]) == 1
def create_user(client, username):
"""Create a new user and login the user. The given user name will also be
used as the user password.
Returns the user identifier and the access token.
Parameters
----------
client: flask.app client
Client for the Flask app
username: string
Unique user name
Returns
-------
(string, string)
"""
data = {
labels.USER_NAME: username,
labels.USER_PASSWORD: username,
labels.VERIFY_USER: False
}
r = client.post(config.API_PATH() + '/users/register', json=data)
user_id = r.json[labels.USER_ID]
data = {labels.USER_NAME: username, labels.USER_PASSWORD: username}
r = client.post(config.API_PATH() + '/users/login', json=data)
token = r.json[labels.USER_TOKEN]
return user_id, token
def test_cancel_run(prepare_submission):
"""Test cancelling a submission run."""
# Create user, submission and upload the run file.
client, headers, benchmark_id, submission_id, file_id = prepare_submission
# -- Start run ------------------------------------------------------------
url = SUBMISSION_RUN.format(config.API_PATH(), submission_id)
body = {
rlbls.RUN_ARGUMENTS: [{
'name': 'names',
'value': serialize_fh(file_id)
}, {
'name': 'greeting',
'value': 'Hi'
}, {
'name': 'sleeptime',
'value': 5
}]
}
r = client.post(url, json=body, headers=headers)
assert r.status_code == 201
run_id = r.json['id']
# -- Cancel and delete run ------------------------------------------------
url = RUN_CANCEL.format(config.API_PATH(), run_id)
r = client.put(url, json={rlbls.CANCEL_REASON: 'Test'}, headers=headers)
assert r.status_code == 200
url = RUN_CANCEL.format(config.API_PATH(), run_id)
# Error when cancelling inactive run or providing invalid body.
r = client.put(url, headers=headers)
assert r.status_code == 400
r = client.put(url, json={'messgae': 'invalid'}, headers=headers)
assert r.status_code == 400
def test_get_benchmark(client, benchmark_id):
"""Test getting a benchmark handle."""
url = config.API_PATH() + '/workflows/{}'.format(benchmark_id)
r = client.get(url)
assert r.status_code == 200
# Attempt to access unknown benchmark
url = config.API_PATH() + '/workflows/undefined'
r = client.get(url)
assert r.status_code == 404
def test_reset_password(client):
"""Test password reset requests."""
# Create user and login
USER1 = dict(USER)
USER1[LABELS['VERIFY']] = False
r = client.post(config.API_PATH() + '/users/register', json=USER1)
data = {LABELS['NAME']: 'user1', LABELS['PASSWORD']: '******'}
r = client.post(config.API_PATH() + '/users/login', json=data)
token = json.loads(r.data)[LABELS['TOKEN']]
headers = {HEADER_TOKEN: token}
r = client.get(config.API_PATH() + '/users/whoami', headers=headers)
assert r.status_code == 200
# Reset password for user1. This should also invalidate the access token.
data = {LABELS['NAME']: 'user1'}
url = config.API_PATH() + '/users/password/request'
r = client.post(url, json=data, headers=headers)
assert r.status_code == 200
req_id = json.loads(r.data)[LABELS['REQUEST_ID']]
data = {LABELS['REQUEST_ID']: req_id, LABELS['PASSWORD']: '******'}
url = config.API_PATH() + '/users/password/reset'
r = client.post(url, json=data, headers=headers)
assert r.status_code == 200
r = client.get(config.API_PATH() + '/users', headers=headers)
assert r.status_code == 200
# The old password is invalid
data = {LABELS['NAME']: 'user1', LABELS['PASSWORD']: '******'}
r = client.post(config.API_PATH() + '/users/login', json=data)
assert r.status_code == 404
data = {LABELS['NAME']: 'user1', LABELS['PASSWORD']: '******'}
r = client.post(config.API_PATH() + '/users/login', json=data)
assert r.status_code == 200
def test_submission_uploads(client, benchmark_id):
"""Tests uploading and retrieving a file."""
# -- Setup ----------------------------------------------------------------
# Create new user and submission.
user_1, token_1 = create_user(client, '0000')
headers = {HEADER_TOKEN: token_1}
url = CREATE_SUBMISSION.format(config.API_PATH(), benchmark_id)
r = client.post(url, json={labels.GROUP_NAME: 'S1'}, headers=headers)
submission_id = r.json[labels.GROUP_ID]
# -- Upload files ---------------------------------------------------------
data = dict()
with open(SMALL_FILE, 'rb') as f:
data['file'] = (io.BytesIO(f.read()), 'names.txt')
url = SUBMISSION_FILES.format(config.API_PATH(), submission_id)
r = client.post(
url,
data=data,
content_type='multipart/form-data',
headers=headers
)
assert r.status_code == 201
file_id = r.json[flbls.FILE_ID]
# Attempt to upload a file that is too large
data = dict()
with open(LARGE_FILE, 'rb') as f:
data['file'] = (io.BytesIO(f.read()), 'names.txt')
r = client.post(
url,
data=data,
content_type='multipart/form-data',
headers=headers
)
assert r.status_code == 413
# Download the file.
url = SUBMISSION_FILE.format(config.API_PATH(), submission_id, file_id)
r = client.get(url, headers=headers)
assert r.status_code == 200
assert b'Alice' in r.data
assert b'Bob' in r.data
# -- Error cases ----------------------------------------------------------
data = {'file': (io.BytesIO(b'Alice'), '')}
url = SUBMISSION_FILES.format(config.API_PATH(), submission_id)
r = client.post(url, data=data, content_type='multipart/form-data', headers=headers)
assert r.status_code == 400
data = {}
url = SUBMISSION_FILES.format(config.API_PATH(), submission_id)
r = client.post(url, data=data, content_type='multipart/form-data', headers=headers)
assert r.status_code == 400
def test_list_benchmarks(client):
"""Test getting a list of available workflows."""
# The benchmark listing contains one element (independently of whether the
# user is logged in or not).
r = client.get(config.API_PATH() + '/workflows')
assert r.status_code == 200
doc = r.json
assert len(doc[labels.WORKFLOW_LIST]) == 1
# Create user and the request header that contains the API key for the
# logged in user.
_, token = create_user(client, '0000')
headers = {HEADER_TOKEN: token}
r = client.get(config.API_PATH() + '/workflows', headers=headers)
assert r.status_code == 200
doc = r.json
assert len(doc[labels.WORKFLOW_LIST]) == 1
def test_create_submission(client, benchmark_id):
"""Unit tests that create a new submission."""
# Create user and the request header that contains the API key for the
# logged in user.
user_1, token_1 = create_user(client, '0000')
headers_1 = {HEADER_TOKEN: token_1}
# -- Create submission ----------------------------------------------------
url = CREATE_SUBMISSION.format(config.API_PATH(), benchmark_id)
body = {labels.GROUP_NAME: 'First submission'}
r = client.post(url, json=body, headers=headers_1)
assert r.status_code == 201
# Creating a submission with the same name will return a BAD REQUEST
r = client.post(url, json=body, headers=headers_1)
assert r.status_code == 400
# Creating a submission with an unknown member will return a BAD REQUEST
err_body = {
labels.GROUP_NAME: 'Error submission',
labels.GROUP_MEMBERS: ['unknown']
}
r = client.post(url, json=err_body, headers=headers_1)
assert r.status_code == 400
# Invalid request when group members is not a list.
err_body = {
labels.GROUP_NAME: 'Error submission',
labels.GROUP_MEMBERS: 'unknown'
}
r = client.post(url, json=err_body, headers=headers_1)
assert r.status_code == 400
def prepare_submission(client, benchmark_id):
"""Prepare submission and return the client, user token (header), the
benahcmark identifier , submission identifier and uploaded file identifier.
"""
_, token = create_user(client, '0000')
headers = {HEADER_TOKEN: token}
url = SUBMISSION_CREATE.format(config.API_PATH(), benchmark_id)
r = client.post(url, json={glbls.GROUP_NAME: 'S1'}, headers=headers)
submission_id = r.json[glbls.GROUP_ID]
data = {'file': (io.BytesIO(b'Alice\nBob'), 'names.txt')}
url = SUBMISSION_FILES.format(config.API_PATH(), submission_id)
r = client.post(url,
data=data,
content_type='multipart/form-data',
headers=headers)
file_id = r.json[flbls.FILE_ID]
return client, headers, benchmark_id, submission_id, file_id
def test_list_users(client):
"""Test user listings."""
# Create an inactive user
r = client.post(config.API_PATH() + '/users/register', json=USER)
user_id = json.loads(r.data)[LABELS['ID']]
# The list of users is still empty. However, since the user is not logged
# in we also cannot access the listing
r = client.get(config.API_PATH() + '/users')
assert r.status_code == 403
# Activate and authenticate the user
data = {LABELS['ID']: user_id}
r = client.post(config.API_PATH() + '/users/activate', json=data)
data = {LABELS['NAME']: 'user1', LABELS['PASSWORD']: '******'}
r = client.post(config.API_PATH() + '/users/login', json=data)
token = json.loads(r.data)[LABELS['TOKEN']]
headers = {HEADER_TOKEN: token}
r = client.get(config.API_PATH() + '/users', headers=headers)
assert r.status_code == 200
assert len(json.loads(r.data)[LABELS['USERS']]) == 1
# Create user that is active and get listing
USER2 = {
LABELS['NAME']: 'user2',
LABELS['PASSWORD']: '******',
LABELS['VERIFY']: False
}
r = client.post(config.API_PATH() + '/users/register', json=USER2)
assert r.status_code == 201
r = client.get(config.API_PATH() + '/users', headers=headers)
assert r.status_code == 200
assert len(json.loads(r.data)[LABELS['USERS']]) == 2
def test_register_user(client):
"""Test creating and activating a user."""
# Create an inactive user
r = client.post(config.API_PATH() + '/users/register', json=USER)
assert r.status_code == 201
obj = json.loads(r.data)
user_id = obj[LABELS['ID']]
user_name = obj[LABELS['NAME']]
assert user_name == 'user1'
# Ensure that the user cannot login until they have been activated
data = {LABELS['NAME']: 'user1', LABELS['PASSWORD']: '******'}
r = client.post(config.API_PATH() + '/users/login', json=data)
# Inactive users are unknown users
assert r.status_code == 404
# User can login after activation and the user list has one element
data = {LABELS['ID']: user_id}
r = client.post(config.API_PATH() + '/users/activate', json=data)
assert r.status_code == 200
data = {LABELS['NAME']: 'user1', LABELS['PASSWORD']: '******'}
r = client.post(config.API_PATH() + '/users/login', json=data)
assert r.status_code == 200
token = json.loads(r.data)[LABELS['TOKEN']]
headers = {HEADER_TOKEN: token}
# Whoami (user1)
r = client.get(config.API_PATH() + '/users/whoami', headers=headers)
assert r.status_code == 200
obj = json.loads(r.data)
assert obj[LABELS['ID']] == user_id
assert obj[LABELS['NAME']] == 'user1'
assert obj[LABELS['TOKEN']] == token
# Create a user with an existing user name will return a BAD REQUEST
r = client.post(config.API_PATH() + '/users/register', json=USER)
assert r.status_code == 400
# Logout user without access token causes error
r = client.post(config.API_PATH() + '/users/logout')
assert r.status_code == 403
# Logout user that is currently logged in
r = client.post(config.API_PATH() + '/users/logout', headers=headers)
assert r.status_code == 200
# After logout the token is invalid but we can still get a list of users.
r = client.get(config.API_PATH() + '/users', headers=headers)
assert r.status_code == 200
# Whoami will fail with an invalid token.
r = client.get(config.API_PATH() + '/users/whoami', headers=headers)
assert r.status_code == 403
def test_delete_run(prepare_submission):
"""Test deleting a submission run."""
# Create user, submission and upload the run file.
client, headers, benchmark_id, submission_id, file_id = prepare_submission
# -- Start run ------------------------------------------------------------
url = SUBMISSION_RUN.format(config.API_PATH(), submission_id)
body = {
rlbls.RUN_ARGUMENTS: [{
'name': 'names',
'value': serialize_fh(file_id)
}, {
'name': 'greeting',
'value': 'Hi'
}, {
'name': 'sleeptime',
'value': 0
}]
}
r = client.post(url, json=body, headers=headers)
assert r.status_code == 201
run_id = r.json['id']
url = RUN_GET.format(config.API_PATH(), run_id)
r = client.get(url, headers=headers)
assert r.status_code == 200
obj = r.json
while obj['state'] == st.STATE_RUNNING:
time.sleep(1)
r = client.get(url, headers=headers)
assert r.status_code == 200
obj = r.json
assert obj['state'] == st.STATE_SUCCESS
# -- Delete run -----------------------------------------------------------
url = RUNS_LIST.format(config.API_PATH(), submission_id)
r = client.get(url, headers=headers)
doc = r.json
assert len(doc[rlbls.RUN_LIST]) == 1
url = RUN_DELETE.format(config.API_PATH(), run_id)
r = client.delete(url, headers=headers)
assert r.status_code == 204
url = RUNS_LIST.format(config.API_PATH(), submission_id)
r = client.get(url, headers=headers)
doc = r.json
assert len(doc[rlbls.RUN_LIST]) == 0
def test_get_submission(client, benchmark_id):
"""Unit test to retrieve the handle for a user submission."""
# -- Setup ----------------------------------------------------------------
# Create user and the request header that contains the API key for the
# logged in user.
user_1, token_1 = create_user(client, '0000')
headers_1 = {HEADER_TOKEN: token_1}
# Create submission and get the submission identifier.
url = CREATE_SUBMISSION.format(config.API_PATH(), benchmark_id)
r = client.post(url,
json={labels.GROUP_NAME: 'My submission'},
headers=headers_1)
submission_id = r.json[labels.GROUP_ID]
# -- Retrieve submission handles ------------------------------------------
url = ACCESS_SUBMISSION.format(config.API_PATH(), submission_id)
r = client.get(url, headers=headers_1)
assert r.status_code == 200
# Retureving an unknown submission results in stats 404.
url = ACCESS_SUBMISSION.format(config.API_PATH(), 'unknown')
r = client.get(url, headers=headers_1)
assert r.status_code == 404
# Platform (ROB).
#
# Copyright (C) 2019-2021 NYU.
#
# ROB is free software; you can redistribute it and/or modify it under the
# terms of the MIT License; see LICENSE file for more details.
"""Blueprint for user authentication and the user manager service."""
from flask import Blueprint, jsonify, make_response, request
from robflask.api.util import ACCESS_TOKEN, jsonbody
import flowserv.view.user as labels
import robflask.config as config
bp = Blueprint('users', __name__, url_prefix=config.API_PATH())
@bp.route('/users', methods=['GET'])
def list_users():
"""Get listing of registered users. Only users that are registered and
currently logged in are allowed to query the database.
Returns
-------
flask.response_class
Raises
------
flowserv.error.UnauthenticatedAccessError
"""
# Copyright (C) 2019-2021 NYU.
#
# ROB is free software; you can redistribute it and/or modify it under the
# terms of the MIT License; see LICENSE file for more details.
"""Blueprint for benchmark resources and benchmark leader boards."""
from flask import Blueprint, jsonify, make_response, request, send_file
from flowserv.model.template.schema import SortColumn
from robflask.api.util import ACCESS_TOKEN
import robflask.config as config
bp = Blueprint('workflows', __name__, url_prefix=config.API_PATH())
@bp.route('/workflows', methods=['GET'])
def list_benchmarks():
"""Get listing of available benchmarks. The benchmark listing is available
to everyone, independent of whether they are currently authenticated or
not.
"""
from robflask.service import service
with service() as api:
r = api.workflows().list_workflows()
return make_response(jsonify(r), 200)
@bp.route('/workflows/<string:workflow_id>', methods=['GET'])
# This file is part of the Reproducible Open Benchmarks for Data Analysis
# Platform (ROB).
#
# Copyright (C) 2019-2021 NYU.
#
# ROB is free software; you can redistribute it and/or modify it under the
# terms of the MIT License; see LICENSE file for more details.
"""Blueprint for the service descriptor."""
from flask import Blueprint, jsonify, request
from robflask.api.util import ACCESS_TOKEN
import robflask.config as config
bp = Blueprint('service', __name__, url_prefix=config.API_PATH())
@bp.route('/', methods=['GET'])
def service_descriptor():
"""Get the API service descriptor."""
# If the request contains an access token we validate that the token is
# still active. The access token is optional for the service descriptor.
# Make sure not to raise an error if no token is present.
from robflask.service import service
with service(access_token=ACCESS_TOKEN(request, raise_error=False)) as api:
return jsonify(api.server().to_dict()), 200
def test_service_descriptor(client):
"""Get service descriptor and ensure that the version is set correclty."""
r = client.get(config.API_PATH() + '/')
assert r.status_code == 200
def test_apipath():
"""Test getting the API path from the service environment."""
assert config.API_PATH() is not None
#
# ROB is free software; you can redistribute it and/or modify it under the
# terms of the MIT License; see LICENSE file for more details.
"""Blueprint for submission resources."""
from flask import Blueprint, jsonify, make_response, request
from flowserv.error import UnknownUserError
from robflask.api.util import ACCESS_TOKEN, jsonbody
import flowserv.view.group as labels
import robflask.config as config
import robflask.error as err
bp = Blueprint('submissions', __name__, url_prefix=config.API_PATH())
@bp.route('/workflows/<string:workflow_id>/groups', methods=['POST'])
def create_submission(workflow_id):
"""Create a new submission for a given benchmark. The user has to be
authenticated in order to be able to create a new submission.
"""
# Get the access token first to raise an error immediately if no token is
# present (to avoid unnecessarily instantiating the service API).
token = ACCESS_TOKEN(request)
# Verify that the request contains a valid Json object that contains the
# submission name and an optional list of member identifier.
obj = jsonbody(request,
mandatory=[labels.GROUP_NAME],
optional=[labels.GROUP_MEMBERS])
def test_submission_run(prepare_submission):
"""Tests start and monitor a run and access run resources."""
# Create user, submission and upload the run file.
client, headers, benchmark_id, submission_id, file_id = prepare_submission
# -- Start run ------------------------------------------------------------
url = SUBMISSION_RUN.format(config.API_PATH(), submission_id)
body = {
rlbls.RUN_ARGUMENTS: [{
'name': 'names',
'value': serialize_fh(file_id)
}, {
'name': 'greeting',
'value': 'Hi'
}, {
'name': 'sleeptime',
'value': 2
}]
}
r = client.post(url, json=body, headers=headers)
assert r.status_code == 201
run_id = r.json['id']
# -- Monitor run state ----------------------------------------------------
url = RUN_GET.format(config.API_PATH(), run_id)
r = client.get(url, headers=headers)
assert r.status_code == 200
obj = r.json
while obj['state'] == st.STATE_RUNNING:
time.sleep(1)
r = client.get(url, headers=headers)
assert r.status_code == 200
obj = r.json
assert obj['state'] == st.STATE_SUCCESS
# -- Run resources --------------------------------------------------------
resources = {r['name']: r for r in obj['files']}
assert len(resources) == 2
assert 'results/greetings.txt' in resources
assert 'results/analytics.json' in resources
result_file_id = resources['results/greetings.txt']['id']
res_url = RUN_FILE.format(config.API_PATH(), run_id, result_file_id)
r = client.get(res_url, headers=headers)
assert r.status_code == 200
data = str(r.data)
assert 'Hi Alice' in data
assert 'Hi Bob' in data
# Run archive
url = RUN_ARCHIVE.format(config.API_PATH(), run_id)
r = client.get(url, headers=headers)
assert r.status_code == 200
# -- Workflow resources ---------------------------------------------------
url = BENCHMARK_GET.format(config.API_PATH(), benchmark_id)
b = client.get(url).json
counter = 0
while 'postproc' not in b:
counter += 1
if counter == 10:
break
time.sleep(1)
b = client.get(url).json
assert counter < 10
counter = 0
while b['postproc']['state'] != st.STATE_SUCCESS:
counter += 1
if counter == 10:
break
time.sleep(1)
b = client.get(url).json
assert counter < 10
url = BENCHMARK_ARCHIVE.format(config.API_PATH(), benchmark_id)
r = client.get(url)
assert r.status_code == 200
assert 'results.tar.gz' in r.headers['Content-Disposition']
resource_id = b['postproc']['files'][0]['id']
url = BENCHMARK_FILE.format(config.API_PATH(), benchmark_id, resource_id)
r = client.get(url)
assert r.status_code == 200
assert 'results/compare.json' in r.headers['Content-Disposition']
# -- Leaderboard ----------------------------------------------------------
url = BENCHMARK_LEADERBOARD.format(config.API_PATH(), benchmark_id)
r = client.get(url)
assert r.status_code == 200
url += '?includeAll'
r = client.get(url)
assert r.status_code == 200
url += '=true'
r = client.get(url)
assert r.status_code == 200
url += '&orderBy=max_len:asc,max_line:desc,avg_count'
r = client.get(url)
assert r.status_code == 200
# Error for runs with invalid arguments.
url = SUBMISSION_RUN.format(config.API_PATH(), submission_id)
body = {
rlbls.RUN_ARGUMENTS: [{
'name': 'names',
'value': serialize_fh(file_id)
}, {
'name': 'greeting',
'value': 'Hi'
}, {
'name': 'sleepfor',
'value': 2
}]
}
r = client.post(url, json=body, headers=headers)
assert r.status_code == 400