def test_odo_kwargs(test, serial):
expr = t.dumb
bad_query = {'expr': to_tree(expr)}
result = test.post(
'/compute',
headers=mimetype(serial),
data=serial.dumps(bad_query),
)
assert result.status_code == 500
assert b'return_df must be passed' in result.data
good_query = {
'expr': to_tree(expr),
'odo_kwargs': {
'return_df': odo(DumbResource.df, list),
},
}
result = test.post(
'/compute',
headers=mimetype(serial),
data=serial.dumps(good_query)
)
assert result.status_code == 200
data = serial.loads(result.data)
dshape = discover(DumbResource.df)
assert_dshape_equal(
datashape.dshape(data['datashape']),
dshape,
)
assert_frame_equal(
odo(data['data'], DataFrame, dshape=dshape),
DumbResource.df,
)
def test_compute_kwargs(test, serial):
expr = t.dumb.sort()
bad_query = {'expr': to_tree(expr)}
result = test.post(
'/compute',
headers=mimetype(serial),
data=serial.dumps(bad_query),
)
assert result.status_code == 500
assert b'return_df must be passed' in result.data
good_query = {
'expr': to_tree(expr),
'compute_kwargs': {
'return_df': odo(DumbResource.df, list),
},
}
result = test.post(
'/compute',
headers=mimetype(serial),
data=serial.dumps(good_query)
)
assert result.status_code == 200
data = serial.loads(result.data)
dshape = discover(DumbResource.df)
assert_dshape_equal(
datashape.dshape(data['datashape']),
dshape,
)
assert_frame_equal(
odo(data['data'], DataFrame, dshape=dshape),
DumbResource.df,
)
def test_add_data_to_empty_server(empty_server, serial):
# add data
with temp_server() as test:
iris_path = example('iris.csv')
blob = serial.dumps({'iris': iris_path})
response1 = empty_server.post(
'/add',
headers=mimetype(serial),
data=blob,
)
assert 'OK' in response1.status
assert response1.status_code == 200
# check for expected server datashape
response2 = empty_server.get('/datashape')
expected2 = str(discover({'iris': resource(iris_path)}))
assert response2.data.decode('utf-8') == expected2
# compute on added data
t = Data({'iris': resource(iris_path)})
expr = t.iris.petal_length.sum()
response3 = empty_server.post(
'/compute',
data=serial.dumps({'expr': to_tree(expr)}),
headers=mimetype(serial)
)
result3 = serial.loads(response3.data)['data']
expected3 = compute(expr, {'iris': resource(iris_path)})
assert result3 == expected3
def test_add_expanded_payload_has_effect(temp_add_server, serial):
# Ensure that the expanded payload format actually passes the arguments
# through to the resource constructor
iris_path = example('iris-latin1.tsv')
csv_kwargs = {'delimiter': '\t', 'encoding': 'iso-8859-1'}
blob = serial.dumps({'iris': {'source': iris_path, 'kwargs': csv_kwargs}})
response1 = temp_add_server.post('/add',
headers=mimetype(serial),
data=blob)
assert 'CREATED' in response1.status
assert response1.status_code == RC.CREATED
# check for expected server datashape
response2 = temp_add_server.get('/datashape')
expected2 = discover({'iris': data(iris_path, **csv_kwargs)})
response_dshape = datashape.dshape(response2.data.decode('utf-8'))
assert_dshape_equal(response_dshape.measure.dict['iris'],
expected2.measure.dict['iris'])
# compute on added data
t = data({'iris': data(iris_path, **csv_kwargs)})
expr = t.iris.petal_length.sum()
response3 = temp_add_server.post('/compute',
data=serial.dumps({'expr':
to_tree(expr)}),
headers=mimetype(serial))
result3 = serial.data_loads(serial.loads(response3.data)['data'])
expected3 = compute(expr, {'iris': data(iris_path, **csv_kwargs)})
assert result3 == expected3
def test_add_data_to_server(temp_add_server, serial):
# add data
iris_path = example('iris.csv')
blob = serial.dumps({'iris': iris_path})
response1 = temp_add_server.post('/add',
headers=mimetype(serial),
data=blob)
assert 'CREATED' in response1.status
assert response1.status_code == RC.CREATED
# check for expected server datashape
response2 = temp_add_server.get('/datashape')
expected2 = discover({'iris': data(iris_path)})
response_dshape = datashape.dshape(response2.data.decode('utf-8'))
assert_dshape_equal(response_dshape.measure.dict['iris'],
expected2.measure.dict['iris'])
# compute on added data
t = data({'iris': data(iris_path)})
expr = t.iris.petal_length.sum()
response3 = temp_add_server.post('/compute',
data=serial.dumps({'expr': to_tree(expr)}),
headers=mimetype(serial))
result3 = serial.data_loads(serial.loads(response3.data)['data'])
expected3 = compute(expr, {'iris': data(iris_path)})
assert result3 == expected3
def test_add_expanded_payload_has_effect(temp_add_server, serial):
# Ensure that the expanded payload format actually passes the arguments
# through to the resource constructor
iris_path = example('iris-latin1.tsv')
csv_kwargs = {'delimiter': '\t', 'encoding': 'iso-8859-1'}
blob = serial.dumps({'iris': {'source': iris_path,
'kwargs': csv_kwargs}})
response1 = temp_add_server.post('/add',
headers=mimetype(serial),
data=blob)
assert 'CREATED' in response1.status
assert response1.status_code == RC.CREATED
# check for expected server datashape
response2 = temp_add_server.get('/datashape')
expected2 = discover({'iris': data(iris_path, **csv_kwargs)})
response_dshape = datashape.dshape(response2.data.decode('utf-8'))
assert_dshape_equal(response_dshape.measure.dict['iris'],
expected2.measure.dict['iris'])
# compute on added data
t = data({'iris': data(iris_path, **csv_kwargs)})
expr = t.iris.petal_length.sum()
response3 = temp_add_server.post('/compute',
data=serial.dumps({'expr': to_tree(expr)}),
headers=mimetype(serial))
result3 = serial.data_loads(serial.loads(response3.data)['data'])
expected3 = compute(expr, {'iris': data(iris_path, **csv_kwargs)})
assert result3 == expected3
def test_compute_kwargs(test, serial):
expr = t.dumb.sort()
bad_query = {'expr': to_tree(expr)}
result = test.post('/compute',
headers=mimetype(serial),
data=serial.dumps(bad_query))
assert result.status_code == RC.INTERNAL_SERVER_ERROR
assert b'return_df must be passed' in result.data
good_query = {
'expr': to_tree(expr),
'compute_kwargs': {
'return_df': odo(DumbResource.df, list)
}
}
result = test.post('/compute',
headers=mimetype(serial),
data=serial.dumps(good_query))
assert result.status_code == RC.OK
tdata = serial.loads(result.data)
dshape = discover(DumbResource.df)
assert_dshape_equal(datashape.dshape(tdata['datashape']), dshape)
assert_frame_equal(
odo(serial.data_loads(tdata['data']), DataFrame, dshape=dshape),
DumbResource.df)
def test_add_data_to_empty_server(empty_server, serial):
# add data
with temp_server() as test:
iris_path = example('iris.csv')
blob = serial.dumps({'iris': iris_path})
response1 = empty_server.post(
'/add',
headers=mimetype(serial),
data=blob,
)
assert 'OK' in response1.status
assert response1.status_code == 200
# check for expected server datashape
response2 = empty_server.get('/datashape')
expected2 = str(discover({'iris': resource(iris_path)}))
assert response2.data.decode('utf-8') == expected2
# compute on added data
t = Data({'iris': resource(iris_path)})
expr = t.iris.petal_length.sum()
response3 = empty_server.post(
'/compute',
data=serial.dumps({'expr': to_tree(expr)}),
headers=mimetype(serial)
)
result3 = serial.loads(response3.data)['data']
expected3 = compute(expr, {'iris': resource(iris_path)})
assert result3 == expected3
def test_add_data_to_server(temp_add_server, serial):
# add data
iris_path = example('iris.csv')
blob = serial.dumps({'iris': iris_path})
response1 = temp_add_server.post('/add',
headers=mimetype(serial),
data=blob)
assert 'CREATED' in response1.status
assert response1.status_code == RC.CREATED
# check for expected server datashape
response2 = temp_add_server.get('/datashape')
expected2 = discover({'iris': data(iris_path)})
response_dshape = datashape.dshape(response2.data.decode('utf-8'))
assert_dshape_equal(response_dshape.measure.dict['iris'],
expected2.measure.dict['iris'])
# compute on added data
t = data({'iris': data(iris_path)})
expr = t.iris.petal_length.sum()
response3 = temp_add_server.post('/compute',
data=serial.dumps({'expr':
to_tree(expr)}),
headers=mimetype(serial))
result3 = serial.data_loads(serial.loads(response3.data)['data'])
expected3 = compute(expr, {'iris': data(iris_path)})
assert result3 == expected3
def test_bad_add_payload(temp_add_server, serial):
# try adding more data to server
blob = serial.dumps('This is not a mutable mapping.')
response1 = temp_add_server.post('/add',
headers=mimetype(serial),
data=blob)
assert response1.status_code == RC.UNPROCESSABLE_ENTITY
def test_bad_add_payload(temp_add_server, serial):
# try adding more data to server
blob = serial.dumps('This is not a mutable mapping.')
response1 = temp_add_server.post('/add',
headers=mimetype(serial),
data=blob)
assert response1.status_code == RC.UNPROCESSABLE_ENTITY
def test_sqlalchemy_result(test, serial):
expr = t.db.iris.head(5)
query = {'expr': to_tree(expr)}
response = test.post('/compute',
data=serial.dumps(query),
headers=mimetype(serial))
assert 'OK' in response.status
tdata = serial.loads(response.data)
result = serial.data_loads(tdata['data'])
if isinstance(result, list):
assert all(isinstance(item, (tuple, list)) for item in result)
elif isinstance(result, DataFrame):
expected = DataFrame([[5.1, 3.5, 1.4, 0.2, 'Iris-setosa'],
[4.9, 3.0, 1.4, 0.2, 'Iris-setosa'],
[4.7, 3.2, 1.3, 0.2, 'Iris-setosa'],
[4.6, 3.1, 1.5, 0.2, 'Iris-setosa'],
[5.0, 3.6, 1.4, 0.2, 'Iris-setosa']],
columns=[
'sepal_length', 'sepal_width', 'petal_length',
'petal_width', 'species'
])
assert_frame_equal(expected, result)
assert list(tdata['names']) == t.db.iris.fields
def test_add_bunk_data_error(temp_add_server, serial):
# Try to add bunk data
payload = serial.dumps({'foo': None})
resp = temp_add_server.post('/add',
headers=mimetype(serial),
data=payload)
assert resp.status_code == RC.UNPROCESSABLE_ENTITY
def test_sqlalchemy_result(test, serial):
expr = t.db.iris.head(5)
query = {'expr': to_tree(expr)}
response = test.post('/compute',
data=serial.dumps(query),
headers=mimetype(serial))
assert 'OK' in response.status
tdata = serial.loads(response.data)
result = serial.data_loads(tdata['data'])
if isinstance(result, list):
assert all(isinstance(item, (tuple, list)) for item in result)
elif isinstance(result, DataFrame):
expected = DataFrame([[5.1, 3.5, 1.4, 0.2, 'Iris-setosa'],
[4.9, 3.0, 1.4, 0.2, 'Iris-setosa'],
[4.7, 3.2, 1.3, 0.2, 'Iris-setosa'],
[4.6, 3.1, 1.5, 0.2, 'Iris-setosa'],
[5.0, 3.6, 1.4, 0.2, 'Iris-setosa']],
columns=['sepal_length',
'sepal_width',
'petal_length',
'petal_width',
'species'])
assert_frame_equal(expected, result)
assert list(tdata['names']) == t.db.iris.fields
def test_add_bunk_data_error(temp_add_server, serial):
# Try to add bunk data
payload = serial.dumps({'foo': None})
resp = temp_add_server.post('/add',
headers=mimetype(serial),
data=payload)
assert resp.status_code == RC.UNPROCESSABLE_ENTITY
def test_auth(test_with_auth, username, password, serial):
expr = t.accounts.amount.sum()
query = {'expr': to_tree(expr)}
r = test_with_auth.get(
'/datashape',
headers={'authorization': basic_auth(username, password)})
assert r.status_code == RC.OK
headers = mimetype(serial)
headers['authorization'] = basic_auth(username, password)
s = test_with_auth.post('/compute',
data=serial.dumps(query),
headers=headers)
assert s.status_code == RC.OK
u = test_with_auth.get(
'/datashape',
headers={'authorization': basic_auth(username + 'a', password + 'a')})
assert u.status_code == RC.UNAUTHORIZED
headers['authorization'] = basic_auth(username + 'a', password + 'a')
v = test_with_auth.post('/compute',
data=serial.dumps(query),
headers=headers)
assert v.status_code == RC.UNAUTHORIZED
def test_auth(test_with_auth, username, password, serial):
expr = t.accounts.amount.sum()
query = {'expr': to_tree(expr)}
r = test_with_auth.get(
'/datashape',
headers={'authorization': basic_auth(username, password)},
)
assert r.status_code == 200
headers = mimetype(serial)
headers['authorization'] = basic_auth(username, password)
s = test_with_auth.post(
'/compute',
data=serial.dumps(query),
headers=headers,
)
assert s.status_code == 200
u = test_with_auth.get(
'/datashape',
headers={'authorization': basic_auth(username + 'a', password + 'a')},
)
assert u.status_code == 401
headers['authorization'] = basic_auth(username + 'a', password + 'a')
v = test_with_auth.post(
'/compute',
data=serial.dumps(query),
headers=headers,
)
assert v.status_code == 401
def test_add_two_data_sets_at_once_error(temp_add_server, serial):
# Try to add two things at once
payload = serial.dumps({'foo': 'iris.csv',
'bar': 'iris.csv'})
resp = temp_add_server.post('/add',
headers=mimetype(serial),
data=payload)
assert resp.status_code == RC.UNPROCESSABLE_ENTITY
def test_cant_add_data_to_server(iris_server, serial):
# try adding more data to server
iris_path = example('iris.csv')
blob = serial.dumps({'iris': iris_path})
response1 = iris_server.post('/add',
headers=mimetype(serial),
data=blob)
assert response1.status_code == RC.UNPROCESSABLE_ENTITY
def test_add_default_not_allowed(temp_server, serial):
iris_path = example('iris.csv')
blob = serial.dumps({'iris': iris_path})
response1 = temp_server.post('/add',
headers=mimetype(serial),
data=blob)
assert 'NOT FOUND' in response1.status
assert response1.status_code == RC.NOT_FOUND
def test_add_two_data_sets_at_once_error(temp_add_server, serial):
# Try to add two things at once
payload = serial.dumps({'foo': 'iris.csv',
'bar': 'iris.csv'})
resp = temp_add_server.post('/add',
headers=mimetype(serial),
data=payload)
assert resp.status_code == RC.UNPROCESSABLE_ENTITY
def test_add_default_not_allowed(temp_server, serial):
iris_path = example('iris.csv')
blob = serial.dumps({'iris': iris_path})
response1 = temp_server.post('/add',
headers=mimetype(serial),
data=blob)
assert 'NOT FOUND' in response1.status
assert response1.status_code == RC.NOT_FOUND
def test_cant_add_data_to_server(iris_server, serial):
# try adding more data to server
iris_path = example('iris.csv')
blob = serial.dumps({'iris': iris_path})
response1 = iris_server.post('/add',
headers=mimetype(serial),
data=blob)
assert response1.status_code == RC.UNPROCESSABLE_ENTITY
def test_bad_add_payload(empty_server, serial):
# try adding more data to server
blob = serial.dumps('This is not a mutable mapping.')
response1 = empty_server.post(
'/add',
headers=mimetype(serial),
data=blob,
)
assert response1.status_code == 422
def test_bad_add_payload(empty_server, serial):
# try adding more data to server
blob = serial.dumps('This is not a mutable mapping.')
response1 = empty_server.post(
'/add',
headers=mimetype(serial),
data=blob,
)
assert response1.status_code == 422
def test_log_format_exc(temp_server_with_excfmt, serial):
expr = t.dumb.sort()
bad_query = {'expr': to_tree(expr)}
server, log_stream = temp_server_with_excfmt
result = server.post('/compute', headers=mimetype(serial),
data=serial.dumps(bad_query))
assert 'CUSTOM TRACEBACK' in log_stream.getvalue()
def test_add_data_twice_error(temp_server, serial):
# add iris
iris_path = example('iris.csv')
payload = serial.dumps({'iris': iris_path})
temp_server.post('/add', headers=mimetype(serial), data=payload)
# Try to add to existing 'iris'
resp = temp_server.post('/add', headers=mimetype(serial), data=payload)
assert resp.status_code == RC.CONFLICT
# Verify the server still serves the original 'iris'.
ds = datashape.dshape(temp_server.get('/datashape').data.decode('utf-8'))
t = symbol('t', ds)
query = {'expr': to_tree(t.iris)}
resp = temp_server.post('/compute',
data=serial.dumps(query),
headers=mimetype(serial))
assert resp.status_code == RC.OK
def test_add_expanded_payload(temp_add_server, serial):
# Ensure that the expanded payload format is accepted by the server
iris_path = example('iris.csv')
blob = serial.dumps({'iris': {'source': iris_path,
'kwargs': {'delimiter': ','}}})
response1 = temp_add_server.post('/add',
headers=mimetype(serial),
data=blob)
assert 'CREATED' in response1.status
assert response1.status_code == RC.CREATED
def test_log_format_exc(temp_server_with_excfmt, serial):
expr = t.dumb.sort()
bad_query = {'expr': to_tree(expr)}
server, log_stream = temp_server_with_excfmt
result = server.post('/compute',
headers=mimetype(serial),
data=serial.dumps(bad_query))
assert 'CUSTOM TRACEBACK' in log_stream.getvalue()
def test_add_expanded_payload(temp_add_server, serial):
# Ensure that the expanded payload format is accepted by the server
iris_path = example('iris.csv')
blob = serial.dumps({'iris': {'source': iris_path,
'kwargs': {'delimiter': ','}}})
response1 = temp_add_server.post('/add',
headers=mimetype(serial),
data=blob)
assert 'CREATED' in response1.status
assert response1.status_code == RC.CREATED
def test_builtin_501_exception(iris_server, serial):
t = symbol('t', discover(iris))
for name in ('map', 'apply'):
func = getattr(t.species, name)
expr = func(copy, 'int')
query = {'expr': to_tree(expr)}
response = iris_server.post('/compute',
data=serial.dumps(query),
headers=mimetype(serial))
assert '501 Not Implemented'.lower() in response.status.lower()
def test_serialization_endpoints(test, serial):
expr = t.db.iris.petal_length.sum()
query = {'expr': to_tree(expr)}
response = test.post('/compute',
data=serial.dumps(query),
headers=mimetype(serial))
assert 'OK' in response.status
respdata = serial.loads(response.data)
result = serial.data_loads(respdata['data'])
assert result == into(int, compute(expr, {t: tdata}))
assert list(respdata['names']) == ['petal_length_sum']
def test_leaf_symbol(test, serial):
query = {'expr': {'op': 'Field', 'args': [':leaf', 'cities']}}
resp = test.post('/compute',
data=serial.dumps(query),
headers=mimetype(serial))
tdata = serial.loads(resp.data)
a = serial.data_loads(tdata['data'])
b = into(list, cities)
assert list(map(tuple, into(list, a))) == b
assert list(tdata['names']) == cities.columns.tolist()
def test_apply_client_server(iris_server, serial):
test = iris_server
t = symbol('t', discover(iris))
expr = t.species.apply(id, 'int') # Very dumb example...
query = {'expr': to_tree(expr)}
response = test.post('/compute',
data=serial.dumps(query),
headers=mimetype(serial))
assert 'OK' in response.status
respdata = serial.loads(response.data)
result = serial.data_loads(respdata['data'])
assert type(result) == type(compute(expr, {t: iris}, return_type=int))
def test_apply_client_server(iris_server, serial):
test = iris_server
t = symbol('t', discover(iris))
expr = t.species.apply(id, 'int') # Very dumb example...
query = {'expr': to_tree(expr)}
response = test.post('/compute',
data=serial.dumps(query),
headers=mimetype(serial))
assert 'OK' in response.status
respdata = serial.loads(response.data)
result = serial.data_loads(respdata['data'])
assert type(result) == type(compute(expr, {t: iris}, return_type=int))
def test_serialization_endpoints(test, serial):
expr = t.db.iris.petal_length.sum()
query = {'expr': to_tree(expr)}
response = test.post('/compute',
data=serial.dumps(query),
headers=mimetype(serial))
assert 'OK' in response.status
respdata = serial.loads(response.data)
result = serial.data_loads(respdata['data'])
assert result == into(int, compute(expr, {t: tdata}))
assert list(respdata['names']) == ['petal_length_sum']
def test_server_can_compute_sqlalchemy_reductions(test, serial):
expr = t.db.iris.petal_length.sum()
query = {'expr': to_tree(expr)}
response = test.post('/compute',
data=serial.dumps(query),
headers=mimetype(serial))
assert 'OK' in response.status
respdata = serial.loads(response.data)
result = respdata['data']
assert result == odo(compute(expr, {t: data}), int)
assert respdata['names'] == ['petal_length_sum']
def test_fastmsgmpack_mutable_dataframe(test):
expr = t.events # just get back the dataframe
query = {'expr': to_tree(expr)}
result = test.post('/compute',
headers=mimetype(fastmsgpack),
data=fastmsgpack.dumps(query))
assert result.status_code == RC.OK
data = fastmsgpack.data_loads(fastmsgpack.loads(result.data)['data'])
for block in data._data.blocks:
# make sure all the blocks are mutable
assert block.values.flags.writeable
def test_builtin_501_exception(iris_server, serial):
t = symbol('t', discover(iris))
for name in ('map', 'apply'):
func = getattr(t.species, name)
expr = func(copy, 'int')
query = {'expr': to_tree(expr)}
response = iris_server.post('/compute',
data=serial.dumps(query),
headers=mimetype(serial))
assert '501 Not Implemented'.lower() in response.status.lower()
def test_leaf_symbol(test, serial):
query = {'expr': {'op': 'Field', 'args': [':leaf', 'cities']}}
resp = test.post('/compute',
data=serial.dumps(query),
headers=mimetype(serial))
tdata = serial.loads(resp.data)
a = serial.data_loads(tdata['data'])
b = into(list, cities)
assert list(map(tuple, into(list, a))) == b
assert list(tdata['names']) == cities.columns.tolist()
def dont_test_compute_with_namespace(test, serial):
query = {'expr': {'op': 'Field', 'args': ['accounts', 'name']}}
expected = ['Alice', 'Bob']
response = test.post('/compute',
data=serial.dumps(query),
headers=mimetype(serial))
assert 'OK' in response.status
tdata = serial.loads(response.data)
assert serial.data_loads(tdata['data']) == expected
assert tdata['names'] == ['name']
def test_fastmsgmpack_mutable_dataframe(test):
expr = t.events # just get back the dataframe
query = {'expr': to_tree(expr)}
result = test.post('/compute',
headers=mimetype(fastmsgpack),
data=fastmsgpack.dumps(query))
assert result.status_code == RC.OK
data = fastmsgpack.data_loads(fastmsgpack.loads(result.data)['data'])
for block in data._data.blocks:
# make sure all the blocks are mutable
assert block.values.flags.writeable
def test_minute_query(test, serial):
expr = t.events.when.minute
query = {'expr': to_tree(expr)}
result = test.post('/compute',
headers=mimetype(serial),
data=serial.dumps(query))
expected = {
'data': [0, 0],
'names': ['when_minute'],
'datashape': '2 * int64'
}
assert result.status_code == 200
assert expected == serial.loads(result.data)
def test_compute(test, serial):
expr = t.accounts.amount.sum()
query = {'expr': to_tree(expr)}
expected = 300
response = test.post('/compute',
data=serial.dumps(query),
headers=mimetype(serial))
assert 'OK' in response.status
tdata = serial.loads(response.data)
assert serial.data_loads(tdata['data']) == expected
assert list(tdata['names']) == ['amount_sum']
def dont_test_compute_with_namespace(test, serial):
query = {'expr': {'op': 'Field',
'args': ['accounts', 'name']}}
expected = ['Alice', 'Bob']
response = test.post('/compute',
data=serial.dumps(query),
headers=mimetype(serial))
assert 'OK' in response.status
tdata = serial.loads(response.data)
assert serial.data_loads(tdata['data']) == expected
assert tdata['names'] == ['name']
def test_isin(test, serial):
expr = t.events.value.isin(frozenset([1]))
query = {'expr': to_tree(expr)}
result = test.post('/compute',
headers=mimetype(serial),
data=serial.dumps(query))
expected = {
'data': [True, False],
'names': ['value'],
'datashape': '2 * bool',
}
assert result.status_code == 200
assert expected == serial.loads(result.data)
def test_bad_payload_keys(test, serial):
result = test.post('/compute',
headers=mimetype(serial),
data=serial.dumps({u'ayy': 'lmao',
u'nah': 'fam',
u'profile': True,
u'profiler_output': ':response'}))
assert result.status_code == RC.BAD_REQUEST
assert (result.data ==
"unexpected keys in payload: [{u}'ayy', {u}'nah']".format(
u='u' if PY2 else '',
).encode('ascii'))
def test_compute(test, serial):
expr = t.accounts.amount.sum()
query = {'expr': to_tree(expr)}
expected = 300
response = test.post('/compute',
data=serial.dumps(query),
headers=mimetype(serial))
assert 'OK' in response.status
tdata = serial.loads(response.data)
assert serial.data_loads(tdata['data']) == expected
assert list(tdata['names']) == ['amount_sum']
def test_sqlalchemy_result(test, serial):
expr = t.db.iris.head(5)
query = {'expr': to_tree(expr)}
response = test.post('/compute',
data=serial.dumps(query),
headers=mimetype(serial))
assert 'OK' in response.status
data = serial.loads(response.data)
result = data['data']
assert all(isinstance(item, (tuple, list)) for item in result)
assert data['names'] == t.db.iris.fields
def test_add_data_to_server(serial):
with temp_server(data) as test:
# add data
initial_datashape = test.get('/datashape').data.decode('utf-8')
iris_path = example('iris.csv')
blob = serial.dumps({'iris': iris_path})
response1 = test.post(
'/add',
headers=mimetype(serial),
data=blob,
)
assert 'OK' in response1.status
assert response1.status_code == 200
# check for expected server datashape
new_datashape = test.get('/datashape').data.decode('utf-8')
data2 = data.copy()
data2.update({'iris': resource(iris_path)})
expected2 = str(discover(data2))
from pprint import pprint as pp
#import ipdb; ipdb.set_trace()
assert new_datashape == expected2
a = new_datashape != initial_datashape
assert new_datashape != initial_datashape
# compute on added data
t = Data({'iris': resource(iris_path)})
expr = t.iris.petal_length.sum()
response3 = test.post(
'/compute',
data=serial.dumps({'expr': to_tree(expr)}),
headers=mimetype(serial)
)
result3 = serial.loads(response3.data)['data']
expected3 = compute(expr, {'iris': resource(iris_path)})
assert result3 == expected3
def test_add_data_twice_error(temp_add_server, serial):
# add iris
iris_path = example('iris.csv')
payload = serial.dumps({'iris': iris_path})
temp_add_server.post('/add',
headers=mimetype(serial),
data=payload)
# Try to add to existing 'iris'
resp = temp_add_server.post('/add',
headers=mimetype(serial),
data=payload)
assert resp.status_code == RC.CONFLICT
# Verify the server still serves the original 'iris'.
response_ds = temp_add_server.get('/datashape').data.decode('utf-8')
ds = datashape.dshape(response_ds)
t = symbol('t', ds)
query = {'expr': to_tree(t.iris)}
resp = temp_add_server.post('/compute',
data=serial.dumps(query),
headers=mimetype(serial))
assert resp.status_code == RC.OK
def test_custom_compute_hook(temp_server_with_custom_hook, serial):
expr = t.accounts.amount.sum()
query = {
'expr': to_tree(expr),
'compute_hook_kwargs': {
'status': RC.OK
},
}
response = temp_server_with_custom_hook.post('/compute',
data=serial.dumps(query),
headers=mimetype(serial))
assert response.status_code == RC.OK
query = {
'expr': to_tree(expr),
'compute_hook_kwargs': {
'status': RC.BAD_REQUEST
},
}
response = temp_server_with_custom_hook.post('/compute',
data=serial.dumps(query),
headers=mimetype(serial))
assert response.status_code == RC.BAD_REQUEST
def test_server_can_compute_sqlalchemy_reductions(test, serial, server):
client.requests = server.app.test_client()
t = data(bz.Client('localhost:6363'))
expr = t.db.iris.petal_length.sum()
query = {'expr': to_tree(expr)}
response = test.post('/compute',
data=serial.dumps(query),
headers=mimetype(serial))
assert 'OK' in response.status
respdata = serial.loads(response.data)
result = serial.data_loads(respdata['data'])
assert result == compute(expr, {t: tdata}, return_type='core')
assert list(respdata['names']) == ['petal_length_sum']
def test_serialization_endpoints(test, serial, server):
client.requests = server.app.test_client()
t = data(bz.Client('localhost:6363'))
expr = t.db.iris.petal_length.sum()
query = {'expr': to_tree(expr)}
response = test.post('/compute',
data=serial.dumps(query),
headers=mimetype(serial))
assert 'OK' in response.status
respdata = serial.loads(response.data)
result = serial.data_loads(respdata['data'])
assert result == compute(expr, {t: tdata}, return_type='core')
assert list(respdata['names']) == ['petal_length_sum']
def test_minute_query(test, serial):
expr = t.events.when.minute
query = {'expr': to_tree(expr)}
result = test.post('/compute',
headers=mimetype(serial),
data=serial.dumps(query))
expected = {'data': [0, 0],
'names': ['when_minute'],
'datashape': '2 * int64'}
assert result.status_code == RC.OK
resp = serial.loads(result.data)
assert list(serial.data_loads(resp['data'])) == expected['data']
assert list(resp['names']) == expected['names']
assert resp['datashape'] == expected['datashape']
def test_compute_by_with_summary(iris_server, serial):
test = iris_server
t = symbol('t', discover(iris))
expr = by(t.species, max=t.petal_length.max(), sum=t.petal_width.sum())
tree = to_tree(expr)
blob = serial.dumps({'expr': tree})
resp = test.post('/compute', data=blob, headers=mimetype(serial))
assert 'OK' in resp.status
tdata = serial.loads(resp.data)
result = DataFrame(serial.data_loads(tdata['data'])).values
expected = compute(expr, iris).values
np.testing.assert_array_equal(result[:, 0], expected[:, 0])
np.testing.assert_array_almost_equal(result[:, 1:], expected[:, 1:])
assert list(tdata['names']) == ['species', 'max', 'sum']
def test_isin(test, serial):
expr = t.events.value.isin(frozenset([1]))
query = {'expr': to_tree(expr)}
result = test.post('/compute',
headers=mimetype(serial),
data=serial.dumps(query))
expected = {'data': [True, False],
'names': ['value'],
'datashape': '2 * bool'}
assert result.status_code == RC.OK
resp = serial.loads(result.data)
assert list(serial.data_loads(resp['data'])) == expected['data']
assert list(resp['names']) == expected['names']
assert resp['datashape'] == expected['datashape']
def test_server_can_compute_sqlalchemy_reductions(test, serial):
expr = t.db.iris.petal_length.sum()
query = {'expr': to_tree(expr)}
response = test.post(
'/compute',
data=serial.dumps(query),
headers=mimetype(serial)
)
assert 'OK' in response.status
respdata = serial.loads(response.data)
result = respdata['data']
assert result == odo(compute(expr, {t: data}), int)
assert respdata['names'] == ['petal_length_sum']
def test_odo_kwargs(test, serial):
expr = t.dumb
bad_query = {'expr': to_tree(expr)}
result = test.post('/compute',
headers=mimetype(serial),
data=serial.dumps(bad_query))
assert result.status_code == RC.INTERNAL_SERVER_ERROR
assert b'return_df must be passed' in result.data
good_query = {'expr': to_tree(expr),
'odo_kwargs': {'return_df': odo(DumbResource.df, list)}}
result = test.post('/compute',
headers=mimetype(serial),
data=serial.dumps(good_query))
assert result.status_code == RC.OK
tdata = serial.loads(result.data)
dshape = discover(DumbResource.df)
assert_dshape_equal(datashape.dshape(tdata['datashape']),
dshape)
assert_frame_equal(odo(serial.data_loads(tdata['data']),
DataFrame,
dshape=dshape),
DumbResource.df)
def test_sqlalchemy_result(test, serial):
expr = t.db.iris.head(5)
query = {'expr': to_tree(expr)}
response = test.post(
'/compute',
data=serial.dumps(query),
headers=mimetype(serial)
)
assert 'OK' in response.status
data = serial.loads(response.data)
result = data['data']
assert all(isinstance(item, (tuple, list)) for item in result)
assert data['names'] == t.db.iris.fields