def test_post_command_json_mime_type():
rw = RunwayModel()
@rw.command('times_two',
inputs={'input': number},
outputs={'output': number})
def times_two(model, args):
return args['input'] * 2
rw.run(debug=True)
client = get_test_client(rw)
response = client.post('/times_two', json={'input': 5})
assert response.is_json
assert json.loads(response.data) == {'output': 10}
def test_inference_error():
rw = RunwayModel()
client = get_test_client(rw)
@rw.command('test_command',
inputs={'input': number},
outputs={'output': text})
def test_command(model, inputs):
raise Exception(
'test exception, thrown from inside a wrapped command() function')
rw.run(debug=True)
response = client.post('test_command', json={'input': 5})
assert response.is_json
assert 'InferenceError' in str(response.data)
def test_500_internal_server_error():
rw = RunwayModel()
@rw.app.route('/test/internal_server_error')
def unauthorized():
abort(500)
rw.run(debug=True)
client = get_test_client(rw)
response = client.get('/test/internal_server_error')
assert response.is_json
assert response.status_code == 500
expect = { 'error': 'Internal server error.' }
assert response.json == expect
def test_403_forbidden():
rw = RunwayModel()
@rw.app.route('/test/forbidden')
def unauthorized():
abort(403)
rw.run(debug=True)
client = get_test_client(rw)
response = client.get('/test/forbidden')
assert response.is_json
assert response.status_code == 403
expect = { 'error': 'Forbidden.' }
assert response.json == expect
def test_401_unauthorized():
rw = RunwayModel()
@rw.app.route('/test/unauthorized')
def unauthorized():
abort(401)
rw.run(debug=True)
client = get_test_client(rw)
response = client.get('/test/unauthorized')
assert response.is_json
assert response.status_code == 401
expect = { 'error': 'Unauthorized (well... really unauthenticated but hey I didn\'t write the spec).' }
assert response.json == expect
def test_post_setup_invalid_json_no_mime_type():
rw = RunwayModel()
@rw.setup(options={'input': text})
def setup(opts):
pass
rw.run(debug=True)
client = get_test_client(rw)
response = client.post('/setup', data='{"input": test input"}')
assert response.is_json
assert response.status_code == 400
expect = { 'error': 'The body of all POST requests must contain JSON' }
assert json.loads(response.data) == expect
def test_post_command_form_encoding():
rw = RunwayModel()
@rw.command('times_two', inputs={ 'input': number }, outputs={ 'output': number })
def times_two(model, args):
return args['input'] * 2
rw.run(debug=True)
client = get_test_client(rw)
content_type='application/x-www-form-urlencoded'
response = client.post('/times_two', data='input=5', content_type=content_type)
assert response.is_json
assert response.status_code == 400
expect = { 'error': 'The body of all POST requests must contain JSON' }
assert json.loads(response.data) == expect
def test_post_setup_form_encoding():
rw = RunwayModel()
@rw.setup(options={'input': text})
def setup(opts):
pass
rw.run(debug=True)
client = get_test_client(rw)
content_type='application/x-www-form-urlencoded'
response = client.post('/setup', data='input=test', content_type=content_type)
assert response.is_json
assert response.status_code == 400
expect = { 'error': 'The body of all POST requests must contain JSON' }
assert json.loads(response.data) == expect
def test_setup_invalid_category():
rw = RunwayModel()
@rw.setup(options={'category': category(choices=['Starks', 'Lannisters'])})
def setup(opts):
pass
rw.run(debug=True)
client = get_test_client(rw)
response = client.post('/setup', json={ 'category': 'Tyrells' })
assert response.status_code == 400
json_response = json.loads(response.data)
assert 'error' in json_response
# ensure the user is displayed an error that indicates the category option
# is problematic
assert 'Invalid argument: category' in json_response['error']
# ensure the user is displayed an error that indicates the problematic value
assert 'Tyrells' in json_response['error']
def test_command_invalid_category():
rw = RunwayModel()
inputs = {'category': category(choices=['Starks', 'Lannisters'])}
outputs = {'reflect': text }
@rw.command('test_command', inputs=inputs, outputs=outputs)
def test_command(opts):
return opts['category']
rw.run(debug=True)
client = get_test_client(rw)
response = client.post('/test_command', json={ 'category': 'Targaryen' })
assert response.status_code == 400
json_response = json.loads(response.data)
assert 'error' in json_response
# ensure the user is displayed an error that indicates the category option
# is problematic
assert 'Invalid argument: category' in json_response['error']
# ensure the user is displayed an error that indicates the problematic value
assert 'Targaryen' in json_response['error']
def test_millis_since_last_command_resets_each_command():
rw = RunwayModel()
@rw.command('test_command', inputs={ 'input': number }, outputs = { 'output': text })
def test_command(model, inputs):
pass
rw.run(debug=True)
client = get_test_client(rw)
assert get_manifest(client)['millisSinceLastCommand'] is None
client.post('test_command', json={ 'input': 5 })
first_time = get_manifest(client)['millisSinceLastCommand']
assert type(first_time) == int
for i in range(5):
sleep(0.02)
millis_since_last_command = get_manifest(client)['millisSinceLastCommand']
assert millis_since_last_command > first_time
client.post('test_command', json={ 'input': 5 })
assert get_manifest(client)['millisSinceLastCommand'] < millis_since_last_command
def test_model_setup_and_command():
# use a dict to share state across function scopes. This makes up for the
# fact that Python 2.x doesn't have support for the 'nonlocal' keyword.
closure = dict(setup_ran = False, command_ran = False)
expected_manifest = {
'modelSDKVersion': model_sdk_version,
'millisRunning': None,
'millisSinceLastCommand': None,
'GPU': os.environ.get('GPU', False),
'options': [{
'type': 'category',
'name': 'size',
'oneOf': ['big', 'small'],
'default': 'big',
'description': 'The size of the model. Bigger is better but also slower.',
}],
'commands': [{
'name': 'test_command',
'description': None,
'inputs': [{
'type': 'text',
'name': 'input',
'description': 'Some input text.',
'default': '',
'minLength': 0
}],
'outputs': [{
'type': 'number',
'name': 'output',
'description': 'An output number.',
'default': 0,
'min': 0,
'max': 1,
'step': 1
}]
}]
}
rw = RunwayModel()
description = 'The size of the model. Bigger is better but also slower.'
@rw.setup(options={ 'size': category(choices=['big', 'small'], description=description) })
def setup(opts):
closure['setup_ran'] = True
return {}
inputs = { 'input': text(description='Some input text.') }
outputs = { 'output': number(description='An output number.') }
# Python 2.7 doesn't seem to handle emoji serialization correctly in JSON,
# so we will only test emoji serialization/deserialization in Python 3
if sys.version_info[0] < 3:
description = 'Sorry, Python 2 doesn\'t support emoji very well'
else:
description = 'A test command whose description contains emoji 🕳'
expected_manifest['commands'][0]['description'] = description
@rw.command('test_command', inputs=inputs, outputs=outputs, description=description)
def test_command(model, opts):
closure['command_ran'] = True
return 100
rw.run(debug=True)
client = get_test_client(rw)
response = client.get('/meta')
assert response.is_json
manifest = json.loads(response.data)
# unset millisRunning as we can't reliably predict this value.
# testing that it is an int should be good enough.
assert type(manifest['millisRunning']) == int
manifest['millisRunning'] = None
assert manifest == expected_manifest
# TEMPORARILY CHECK / PATH IN ADDITION TO /meta ----------------------------
# ... sorry for the gross dupe code ;)
response = client.get('/')
assert response.is_json
manifest = json.loads(response.data)
# unset millisRunning as we can't reliably predict this value.
# testing that it is an int should be good enough.
assert type(manifest['millisRunning']) == int
manifest['millisRunning'] = None
assert manifest == expected_manifest
# --------------------------------------------------------------------------
# check the input/output manifest for GET /test_command
response = client.get('/test_command')
assert response.is_json
command_manifest = json.loads(response.data)
assert command_manifest == expected_manifest['commands'][0]
post_data = {
'input': 'test input'
}
response = client.post('/test_command', json=post_data)
assert response.is_json
assert json.loads(response.data) == { 'output' : 100 }
# now that we've run a command lets make sure millis since last command is
# a number
manifest_after_command = get_manifest(client)
assert type(manifest_after_command['millisSinceLastCommand']) == int
assert closure['command_ran'] == True
assert closure['setup_ran'] == True
