def test_delete_submission(client, benchmark_id): """Test deleting a previously created submission.""" # -- Setup ---------------------------------------------------------------- # Create two users that are logged in. user_1, token_1 = create_user(client, '0000') user_2, token_2 = create_user(client, '0001') headers_1 = {HEADER_TOKEN: token_1} headers_2 = {HEADER_TOKEN: token_2} # Create one submission for user 1. url = CREATE_SUBMISSION.format(config.API_PATH(), benchmark_id) r = client.post(url, json={labels.GROUP_NAME: 'S1'}, headers=headers_1) submission_id = r.json[labels.GROUP_ID] # -- Delete submission ---------------------------------------------------- url = ACCESS_SUBMISSION.format(config.API_PATH(), submission_id) r = client.get(url, headers=headers_1) assert r.status_code == 200 # User 2 cannot delete the submission. url = DELETE_SUBMISSION.format(config.API_PATH(), submission_id) r = client.delete(url, headers=headers_2) assert r.status_code == 403 # User 1 can delete the submission. r = client.delete(url, headers=headers_1) assert r.status_code == 204 # Deleting an unknown submission results in statis 404 r = client.delete(url, headers=headers_1) assert r.status_code == 404
def test_update_submission(client, benchmark_id): """Test updating properties of a submission.""" # -- Setup ---------------------------------------------------------------- # Create two users that are logged in. user_1, token_1 = create_user(client, '0000') user_2, token_2 = create_user(client, '0001') headers_1 = {HEADER_TOKEN: token_1} headers_2 = {HEADER_TOKEN: token_2} # Create one submission for user 1. url = CREATE_SUBMISSION.format(config.API_PATH(), benchmark_id) r = client.post(url, json={labels.GROUP_NAME: 'S1'}, headers=headers_1) submission_id = r.json[labels.GROUP_ID] # -- Update submission ---------------------------------------------------- url = UPDATE_SUBMISSION.format(config.API_PATH(), submission_id) body = {labels.GROUP_NAME: 'S1 (updated)'} r = client.put(url, json=body, headers=headers_2) assert r.status_code == 403 r = client.put(url, json=body, headers=headers_1) assert r.status_code == 200 assert r.json[labels.GROUP_NAME] == 'S1 (updated)' # After adding user 2 as a member they can update the submission as well. body = {labels.GROUP_MEMBERS: [user_1, user_2]} r = client.put(url, json=body, headers=headers_1) assert r.status_code == 200 body = {labels.GROUP_NAME: 'S1 (new)'} r = client.put(url, json=body, headers=headers_2) assert r.status_code == 200 assert r.json[labels.GROUP_NAME] == 'S1 (new)'
def test_uploads_listings(client, benchmark_id): """Tests uploading and retrieving a file.""" # -- Setup ---------------------------------------------------------------- # Create new user and submission. Then upload a single file. user_1, token_1 = create_user(client, '0000') headers = {HEADER_TOKEN: token_1} url = CREATE_SUBMISSION.format(config.API_PATH(), benchmark_id) r = client.post(url, json={labels.GROUP_NAME: 'S1'}, headers=headers) submission_id = r.json[labels.GROUP_ID] data = dict() with open(SMALL_FILE, 'rb') as f: data['file'] = (io.BytesIO(f.read()), 'names.txt') url = SUBMISSION_FILES.format(config.API_PATH(), submission_id) r = client.post( url, data=data, content_type='multipart/form-data', headers=headers ) assert r.status_code == 201 file_id = r.json[flbls.FILE_ID] # -- Listing uploaded files ----------------------------------------------- r = client.get(url, headers=headers) assert r.status_code == 200 doc = r.json assert len(doc[flbls.FILE_LIST]) == 1 # Delete the file. url = SUBMISSION_FILE.format(config.API_PATH(), submission_id, file_id) r = client.delete(url, headers=headers) assert r.status_code == 204 url = SUBMISSION_FILES.format(config.API_PATH(), submission_id) r = client.get(url, headers=headers) assert r.status_code == 200 doc = r.json assert len(doc[flbls.FILE_LIST]) == 0
def test_list_submissions(client, benchmark_id): """Test to retrieve a list of submissions that a user is a member of.""" # -- Setup ---------------------------------------------------------------- # Create two users that are logged in. user_1, token_1 = create_user(client, '0000') user_2, token_2 = create_user(client, '0001') headers_1 = {HEADER_TOKEN: token_1} headers_2 = {HEADER_TOKEN: token_2} # Create two submissions. User 1 is member of both submissions but user 2 # is only a member of one submission. url = CREATE_SUBMISSION.format(config.API_PATH(), benchmark_id) client.post(url, json={labels.GROUP_NAME: 'S1'}, headers=headers_1) client.post(url, json={ labels.GROUP_NAME: 'S2', labels.GROUP_MEMBERS: [user_1] }, headers=headers_2) # -- Retrieve submission listings ------------------------------------------ url = SUBMISSION_LIST.format(config.API_PATH(), benchmark_id) r = client.get(url, headers=headers_1) assert r.status_code == 200 doc = r.json assert len(doc[labels.GROUP_LIST]) == 2 # Retureving an unknown submission results in stats 404. r = client.get(url, headers=headers_2) assert r.status_code == 200 doc = r.json assert len(doc[labels.GROUP_LIST]) == 1
def create_user(client, username): """Create a new user and login the user. The given user name will also be used as the user password. Returns the user identifier and the access token. Parameters ---------- client: flask.app client Client for the Flask app username: string Unique user name Returns ------- (string, string) """ data = { labels.USER_NAME: username, labels.USER_PASSWORD: username, labels.VERIFY_USER: False } r = client.post(config.API_PATH() + '/users/register', json=data) user_id = r.json[labels.USER_ID] data = {labels.USER_NAME: username, labels.USER_PASSWORD: username} r = client.post(config.API_PATH() + '/users/login', json=data) token = r.json[labels.USER_TOKEN] return user_id, token
def test_cancel_run(prepare_submission): """Test cancelling a submission run.""" # Create user, submission and upload the run file. client, headers, benchmark_id, submission_id, file_id = prepare_submission # -- Start run ------------------------------------------------------------ url = SUBMISSION_RUN.format(config.API_PATH(), submission_id) body = { rlbls.RUN_ARGUMENTS: [{ 'name': 'names', 'value': serialize_fh(file_id) }, { 'name': 'greeting', 'value': 'Hi' }, { 'name': 'sleeptime', 'value': 5 }] } r = client.post(url, json=body, headers=headers) assert r.status_code == 201 run_id = r.json['id'] # -- Cancel and delete run ------------------------------------------------ url = RUN_CANCEL.format(config.API_PATH(), run_id) r = client.put(url, json={rlbls.CANCEL_REASON: 'Test'}, headers=headers) assert r.status_code == 200 url = RUN_CANCEL.format(config.API_PATH(), run_id) # Error when cancelling inactive run or providing invalid body. r = client.put(url, headers=headers) assert r.status_code == 400 r = client.put(url, json={'messgae': 'invalid'}, headers=headers) assert r.status_code == 400
def test_get_benchmark(client, benchmark_id): """Test getting a benchmark handle.""" url = config.API_PATH() + '/workflows/{}'.format(benchmark_id) r = client.get(url) assert r.status_code == 200 # Attempt to access unknown benchmark url = config.API_PATH() + '/workflows/undefined' r = client.get(url) assert r.status_code == 404
def test_reset_password(client): """Test password reset requests.""" # Create user and login USER1 = dict(USER) USER1[LABELS['VERIFY']] = False r = client.post(config.API_PATH() + '/users/register', json=USER1) data = {LABELS['NAME']: 'user1', LABELS['PASSWORD']: '******'} r = client.post(config.API_PATH() + '/users/login', json=data) token = json.loads(r.data)[LABELS['TOKEN']] headers = {HEADER_TOKEN: token} r = client.get(config.API_PATH() + '/users/whoami', headers=headers) assert r.status_code == 200 # Reset password for user1. This should also invalidate the access token. data = {LABELS['NAME']: 'user1'} url = config.API_PATH() + '/users/password/request' r = client.post(url, json=data, headers=headers) assert r.status_code == 200 req_id = json.loads(r.data)[LABELS['REQUEST_ID']] data = {LABELS['REQUEST_ID']: req_id, LABELS['PASSWORD']: '******'} url = config.API_PATH() + '/users/password/reset' r = client.post(url, json=data, headers=headers) assert r.status_code == 200 r = client.get(config.API_PATH() + '/users', headers=headers) assert r.status_code == 200 # The old password is invalid data = {LABELS['NAME']: 'user1', LABELS['PASSWORD']: '******'} r = client.post(config.API_PATH() + '/users/login', json=data) assert r.status_code == 404 data = {LABELS['NAME']: 'user1', LABELS['PASSWORD']: '******'} r = client.post(config.API_PATH() + '/users/login', json=data) assert r.status_code == 200
def test_submission_uploads(client, benchmark_id): """Tests uploading and retrieving a file.""" # -- Setup ---------------------------------------------------------------- # Create new user and submission. user_1, token_1 = create_user(client, '0000') headers = {HEADER_TOKEN: token_1} url = CREATE_SUBMISSION.format(config.API_PATH(), benchmark_id) r = client.post(url, json={labels.GROUP_NAME: 'S1'}, headers=headers) submission_id = r.json[labels.GROUP_ID] # -- Upload files --------------------------------------------------------- data = dict() with open(SMALL_FILE, 'rb') as f: data['file'] = (io.BytesIO(f.read()), 'names.txt') url = SUBMISSION_FILES.format(config.API_PATH(), submission_id) r = client.post( url, data=data, content_type='multipart/form-data', headers=headers ) assert r.status_code == 201 file_id = r.json[flbls.FILE_ID] # Attempt to upload a file that is too large data = dict() with open(LARGE_FILE, 'rb') as f: data['file'] = (io.BytesIO(f.read()), 'names.txt') r = client.post( url, data=data, content_type='multipart/form-data', headers=headers ) assert r.status_code == 413 # Download the file. url = SUBMISSION_FILE.format(config.API_PATH(), submission_id, file_id) r = client.get(url, headers=headers) assert r.status_code == 200 assert b'Alice' in r.data assert b'Bob' in r.data # -- Error cases ---------------------------------------------------------- data = {'file': (io.BytesIO(b'Alice'), '')} url = SUBMISSION_FILES.format(config.API_PATH(), submission_id) r = client.post(url, data=data, content_type='multipart/form-data', headers=headers) assert r.status_code == 400 data = {} url = SUBMISSION_FILES.format(config.API_PATH(), submission_id) r = client.post(url, data=data, content_type='multipart/form-data', headers=headers) assert r.status_code == 400
def test_list_benchmarks(client): """Test getting a list of available workflows.""" # The benchmark listing contains one element (independently of whether the # user is logged in or not). r = client.get(config.API_PATH() + '/workflows') assert r.status_code == 200 doc = r.json assert len(doc[labels.WORKFLOW_LIST]) == 1 # Create user and the request header that contains the API key for the # logged in user. _, token = create_user(client, '0000') headers = {HEADER_TOKEN: token} r = client.get(config.API_PATH() + '/workflows', headers=headers) assert r.status_code == 200 doc = r.json assert len(doc[labels.WORKFLOW_LIST]) == 1
def test_create_submission(client, benchmark_id): """Unit tests that create a new submission.""" # Create user and the request header that contains the API key for the # logged in user. user_1, token_1 = create_user(client, '0000') headers_1 = {HEADER_TOKEN: token_1} # -- Create submission ---------------------------------------------------- url = CREATE_SUBMISSION.format(config.API_PATH(), benchmark_id) body = {labels.GROUP_NAME: 'First submission'} r = client.post(url, json=body, headers=headers_1) assert r.status_code == 201 # Creating a submission with the same name will return a BAD REQUEST r = client.post(url, json=body, headers=headers_1) assert r.status_code == 400 # Creating a submission with an unknown member will return a BAD REQUEST err_body = { labels.GROUP_NAME: 'Error submission', labels.GROUP_MEMBERS: ['unknown'] } r = client.post(url, json=err_body, headers=headers_1) assert r.status_code == 400 # Invalid request when group members is not a list. err_body = { labels.GROUP_NAME: 'Error submission', labels.GROUP_MEMBERS: 'unknown' } r = client.post(url, json=err_body, headers=headers_1) assert r.status_code == 400
def prepare_submission(client, benchmark_id): """Prepare submission and return the client, user token (header), the benahcmark identifier , submission identifier and uploaded file identifier. """ _, token = create_user(client, '0000') headers = {HEADER_TOKEN: token} url = SUBMISSION_CREATE.format(config.API_PATH(), benchmark_id) r = client.post(url, json={glbls.GROUP_NAME: 'S1'}, headers=headers) submission_id = r.json[glbls.GROUP_ID] data = {'file': (io.BytesIO(b'Alice\nBob'), 'names.txt')} url = SUBMISSION_FILES.format(config.API_PATH(), submission_id) r = client.post(url, data=data, content_type='multipart/form-data', headers=headers) file_id = r.json[flbls.FILE_ID] return client, headers, benchmark_id, submission_id, file_id
def test_list_users(client): """Test user listings.""" # Create an inactive user r = client.post(config.API_PATH() + '/users/register', json=USER) user_id = json.loads(r.data)[LABELS['ID']] # The list of users is still empty. However, since the user is not logged # in we also cannot access the listing r = client.get(config.API_PATH() + '/users') assert r.status_code == 403 # Activate and authenticate the user data = {LABELS['ID']: user_id} r = client.post(config.API_PATH() + '/users/activate', json=data) data = {LABELS['NAME']: 'user1', LABELS['PASSWORD']: '******'} r = client.post(config.API_PATH() + '/users/login', json=data) token = json.loads(r.data)[LABELS['TOKEN']] headers = {HEADER_TOKEN: token} r = client.get(config.API_PATH() + '/users', headers=headers) assert r.status_code == 200 assert len(json.loads(r.data)[LABELS['USERS']]) == 1 # Create user that is active and get listing USER2 = { LABELS['NAME']: 'user2', LABELS['PASSWORD']: '******', LABELS['VERIFY']: False } r = client.post(config.API_PATH() + '/users/register', json=USER2) assert r.status_code == 201 r = client.get(config.API_PATH() + '/users', headers=headers) assert r.status_code == 200 assert len(json.loads(r.data)[LABELS['USERS']]) == 2
def test_register_user(client): """Test creating and activating a user.""" # Create an inactive user r = client.post(config.API_PATH() + '/users/register', json=USER) assert r.status_code == 201 obj = json.loads(r.data) user_id = obj[LABELS['ID']] user_name = obj[LABELS['NAME']] assert user_name == 'user1' # Ensure that the user cannot login until they have been activated data = {LABELS['NAME']: 'user1', LABELS['PASSWORD']: '******'} r = client.post(config.API_PATH() + '/users/login', json=data) # Inactive users are unknown users assert r.status_code == 404 # User can login after activation and the user list has one element data = {LABELS['ID']: user_id} r = client.post(config.API_PATH() + '/users/activate', json=data) assert r.status_code == 200 data = {LABELS['NAME']: 'user1', LABELS['PASSWORD']: '******'} r = client.post(config.API_PATH() + '/users/login', json=data) assert r.status_code == 200 token = json.loads(r.data)[LABELS['TOKEN']] headers = {HEADER_TOKEN: token} # Whoami (user1) r = client.get(config.API_PATH() + '/users/whoami', headers=headers) assert r.status_code == 200 obj = json.loads(r.data) assert obj[LABELS['ID']] == user_id assert obj[LABELS['NAME']] == 'user1' assert obj[LABELS['TOKEN']] == token # Create a user with an existing user name will return a BAD REQUEST r = client.post(config.API_PATH() + '/users/register', json=USER) assert r.status_code == 400 # Logout user without access token causes error r = client.post(config.API_PATH() + '/users/logout') assert r.status_code == 403 # Logout user that is currently logged in r = client.post(config.API_PATH() + '/users/logout', headers=headers) assert r.status_code == 200 # After logout the token is invalid but we can still get a list of users. r = client.get(config.API_PATH() + '/users', headers=headers) assert r.status_code == 200 # Whoami will fail with an invalid token. r = client.get(config.API_PATH() + '/users/whoami', headers=headers) assert r.status_code == 403
def test_delete_run(prepare_submission): """Test deleting a submission run.""" # Create user, submission and upload the run file. client, headers, benchmark_id, submission_id, file_id = prepare_submission # -- Start run ------------------------------------------------------------ url = SUBMISSION_RUN.format(config.API_PATH(), submission_id) body = { rlbls.RUN_ARGUMENTS: [{ 'name': 'names', 'value': serialize_fh(file_id) }, { 'name': 'greeting', 'value': 'Hi' }, { 'name': 'sleeptime', 'value': 0 }] } r = client.post(url, json=body, headers=headers) assert r.status_code == 201 run_id = r.json['id'] url = RUN_GET.format(config.API_PATH(), run_id) r = client.get(url, headers=headers) assert r.status_code == 200 obj = r.json while obj['state'] == st.STATE_RUNNING: time.sleep(1) r = client.get(url, headers=headers) assert r.status_code == 200 obj = r.json assert obj['state'] == st.STATE_SUCCESS # -- Delete run ----------------------------------------------------------- url = RUNS_LIST.format(config.API_PATH(), submission_id) r = client.get(url, headers=headers) doc = r.json assert len(doc[rlbls.RUN_LIST]) == 1 url = RUN_DELETE.format(config.API_PATH(), run_id) r = client.delete(url, headers=headers) assert r.status_code == 204 url = RUNS_LIST.format(config.API_PATH(), submission_id) r = client.get(url, headers=headers) doc = r.json assert len(doc[rlbls.RUN_LIST]) == 0
def test_get_submission(client, benchmark_id): """Unit test to retrieve the handle for a user submission.""" # -- Setup ---------------------------------------------------------------- # Create user and the request header that contains the API key for the # logged in user. user_1, token_1 = create_user(client, '0000') headers_1 = {HEADER_TOKEN: token_1} # Create submission and get the submission identifier. url = CREATE_SUBMISSION.format(config.API_PATH(), benchmark_id) r = client.post(url, json={labels.GROUP_NAME: 'My submission'}, headers=headers_1) submission_id = r.json[labels.GROUP_ID] # -- Retrieve submission handles ------------------------------------------ url = ACCESS_SUBMISSION.format(config.API_PATH(), submission_id) r = client.get(url, headers=headers_1) assert r.status_code == 200 # Retureving an unknown submission results in stats 404. url = ACCESS_SUBMISSION.format(config.API_PATH(), 'unknown') r = client.get(url, headers=headers_1) assert r.status_code == 404
# Platform (ROB). # # Copyright (C) 2019-2021 NYU. # # ROB is free software; you can redistribute it and/or modify it under the # terms of the MIT License; see LICENSE file for more details. """Blueprint for user authentication and the user manager service.""" from flask import Blueprint, jsonify, make_response, request from robflask.api.util import ACCESS_TOKEN, jsonbody import flowserv.view.user as labels import robflask.config as config bp = Blueprint('users', __name__, url_prefix=config.API_PATH()) @bp.route('/users', methods=['GET']) def list_users(): """Get listing of registered users. Only users that are registered and currently logged in are allowed to query the database. Returns ------- flask.response_class Raises ------ flowserv.error.UnauthenticatedAccessError """
# Copyright (C) 2019-2021 NYU. # # ROB is free software; you can redistribute it and/or modify it under the # terms of the MIT License; see LICENSE file for more details. """Blueprint for benchmark resources and benchmark leader boards.""" from flask import Blueprint, jsonify, make_response, request, send_file from flowserv.model.template.schema import SortColumn from robflask.api.util import ACCESS_TOKEN import robflask.config as config bp = Blueprint('workflows', __name__, url_prefix=config.API_PATH()) @bp.route('/workflows', methods=['GET']) def list_benchmarks(): """Get listing of available benchmarks. The benchmark listing is available to everyone, independent of whether they are currently authenticated or not. """ from robflask.service import service with service() as api: r = api.workflows().list_workflows() return make_response(jsonify(r), 200) @bp.route('/workflows/<string:workflow_id>', methods=['GET'])
# This file is part of the Reproducible Open Benchmarks for Data Analysis # Platform (ROB). # # Copyright (C) 2019-2021 NYU. # # ROB is free software; you can redistribute it and/or modify it under the # terms of the MIT License; see LICENSE file for more details. """Blueprint for the service descriptor.""" from flask import Blueprint, jsonify, request from robflask.api.util import ACCESS_TOKEN import robflask.config as config bp = Blueprint('service', __name__, url_prefix=config.API_PATH()) @bp.route('/', methods=['GET']) def service_descriptor(): """Get the API service descriptor.""" # If the request contains an access token we validate that the token is # still active. The access token is optional for the service descriptor. # Make sure not to raise an error if no token is present. from robflask.service import service with service(access_token=ACCESS_TOKEN(request, raise_error=False)) as api: return jsonify(api.server().to_dict()), 200
def test_service_descriptor(client): """Get service descriptor and ensure that the version is set correclty.""" r = client.get(config.API_PATH() + '/') assert r.status_code == 200
def test_apipath(): """Test getting the API path from the service environment.""" assert config.API_PATH() is not None
# # ROB is free software; you can redistribute it and/or modify it under the # terms of the MIT License; see LICENSE file for more details. """Blueprint for submission resources.""" from flask import Blueprint, jsonify, make_response, request from flowserv.error import UnknownUserError from robflask.api.util import ACCESS_TOKEN, jsonbody import flowserv.view.group as labels import robflask.config as config import robflask.error as err bp = Blueprint('submissions', __name__, url_prefix=config.API_PATH()) @bp.route('/workflows/<string:workflow_id>/groups', methods=['POST']) def create_submission(workflow_id): """Create a new submission for a given benchmark. The user has to be authenticated in order to be able to create a new submission. """ # Get the access token first to raise an error immediately if no token is # present (to avoid unnecessarily instantiating the service API). token = ACCESS_TOKEN(request) # Verify that the request contains a valid Json object that contains the # submission name and an optional list of member identifier. obj = jsonbody(request, mandatory=[labels.GROUP_NAME], optional=[labels.GROUP_MEMBERS])
def test_submission_run(prepare_submission): """Tests start and monitor a run and access run resources.""" # Create user, submission and upload the run file. client, headers, benchmark_id, submission_id, file_id = prepare_submission # -- Start run ------------------------------------------------------------ url = SUBMISSION_RUN.format(config.API_PATH(), submission_id) body = { rlbls.RUN_ARGUMENTS: [{ 'name': 'names', 'value': serialize_fh(file_id) }, { 'name': 'greeting', 'value': 'Hi' }, { 'name': 'sleeptime', 'value': 2 }] } r = client.post(url, json=body, headers=headers) assert r.status_code == 201 run_id = r.json['id'] # -- Monitor run state ---------------------------------------------------- url = RUN_GET.format(config.API_PATH(), run_id) r = client.get(url, headers=headers) assert r.status_code == 200 obj = r.json while obj['state'] == st.STATE_RUNNING: time.sleep(1) r = client.get(url, headers=headers) assert r.status_code == 200 obj = r.json assert obj['state'] == st.STATE_SUCCESS # -- Run resources -------------------------------------------------------- resources = {r['name']: r for r in obj['files']} assert len(resources) == 2 assert 'results/greetings.txt' in resources assert 'results/analytics.json' in resources result_file_id = resources['results/greetings.txt']['id'] res_url = RUN_FILE.format(config.API_PATH(), run_id, result_file_id) r = client.get(res_url, headers=headers) assert r.status_code == 200 data = str(r.data) assert 'Hi Alice' in data assert 'Hi Bob' in data # Run archive url = RUN_ARCHIVE.format(config.API_PATH(), run_id) r = client.get(url, headers=headers) assert r.status_code == 200 # -- Workflow resources --------------------------------------------------- url = BENCHMARK_GET.format(config.API_PATH(), benchmark_id) b = client.get(url).json counter = 0 while 'postproc' not in b: counter += 1 if counter == 10: break time.sleep(1) b = client.get(url).json assert counter < 10 counter = 0 while b['postproc']['state'] != st.STATE_SUCCESS: counter += 1 if counter == 10: break time.sleep(1) b = client.get(url).json assert counter < 10 url = BENCHMARK_ARCHIVE.format(config.API_PATH(), benchmark_id) r = client.get(url) assert r.status_code == 200 assert 'results.tar.gz' in r.headers['Content-Disposition'] resource_id = b['postproc']['files'][0]['id'] url = BENCHMARK_FILE.format(config.API_PATH(), benchmark_id, resource_id) r = client.get(url) assert r.status_code == 200 assert 'results/compare.json' in r.headers['Content-Disposition'] # -- Leaderboard ---------------------------------------------------------- url = BENCHMARK_LEADERBOARD.format(config.API_PATH(), benchmark_id) r = client.get(url) assert r.status_code == 200 url += '?includeAll' r = client.get(url) assert r.status_code == 200 url += '=true' r = client.get(url) assert r.status_code == 200 url += '&orderBy=max_len:asc,max_line:desc,avg_count' r = client.get(url) assert r.status_code == 200 # Error for runs with invalid arguments. url = SUBMISSION_RUN.format(config.API_PATH(), submission_id) body = { rlbls.RUN_ARGUMENTS: [{ 'name': 'names', 'value': serialize_fh(file_id) }, { 'name': 'greeting', 'value': 'Hi' }, { 'name': 'sleepfor', 'value': 2 }] } r = client.post(url, json=body, headers=headers) assert r.status_code == 400