def test_compute_kwargs(test, serial):
expr = t.dumb.sort()
bad_query = {'expr': to_tree(expr)}
result = test.post(
'/compute',
headers=mimetype(serial),
data=serial.dumps(bad_query),
)
assert result.status_code == 500
assert b'return_df must be passed' in result.data
good_query = {
'expr': to_tree(expr),
'compute_kwargs': {
'return_df': odo(DumbResource.df, list),
},
}
result = test.post(
'/compute',
headers=mimetype(serial),
data=serial.dumps(good_query)
)
assert result.status_code == 200
data = serial.loads(result.data)
dshape = discover(DumbResource.df)
assert_dshape_equal(
datashape.dshape(data['datashape']),
dshape,
)
assert_frame_equal(
odo(data['data'], DataFrame, dshape=dshape),
DumbResource.df,
)
def test_add_expanded_payload_has_effect(temp_add_server, serial):
# Ensure that the expanded payload format actually passes the arguments
# through to the resource constructor
iris_path = example('iris-latin1.tsv')
csv_kwargs = {'delimiter': '\t', 'encoding': 'iso-8859-1'}
blob = serial.dumps({'iris': {'source': iris_path,
'kwargs': csv_kwargs}})
response1 = temp_add_server.post('/add',
headers=mimetype(serial),
data=blob)
assert 'CREATED' in response1.status
assert response1.status_code == RC.CREATED
# check for expected server datashape
response2 = temp_add_server.get('/datashape')
expected2 = discover({'iris': data(iris_path, **csv_kwargs)})
response_dshape = datashape.dshape(response2.data.decode('utf-8'))
assert_dshape_equal(response_dshape.measure.dict['iris'],
expected2.measure.dict['iris'])
# compute on added data
t = data({'iris': data(iris_path, **csv_kwargs)})
expr = t.iris.petal_length.sum()
response3 = temp_add_server.post('/compute',
data=serial.dumps({'expr': to_tree(expr)}),
headers=mimetype(serial))
result3 = serial.data_loads(serial.loads(response3.data)['data'])
expected3 = compute(expr, {'iris': data(iris_path, **csv_kwargs)})
assert result3 == expected3
def test_sqlalchemy_result(test, serial):
expr = t.db.iris.head(5)
query = {'expr': to_tree(expr)}
response = test.post('/compute',
data=serial.dumps(query),
headers=mimetype(serial))
assert 'OK' in response.status
tdata = serial.loads(response.data)
result = serial.data_loads(tdata['data'])
if isinstance(result, list):
assert all(isinstance(item, (tuple, list)) for item in result)
elif isinstance(result, DataFrame):
expected = DataFrame([[5.1, 3.5, 1.4, 0.2, 'Iris-setosa'],
[4.9, 3.0, 1.4, 0.2, 'Iris-setosa'],
[4.7, 3.2, 1.3, 0.2, 'Iris-setosa'],
[4.6, 3.1, 1.5, 0.2, 'Iris-setosa'],
[5.0, 3.6, 1.4, 0.2, 'Iris-setosa']],
columns=['sepal_length',
'sepal_width',
'petal_length',
'petal_width',
'species'])
assert_frame_equal(expected, result)
assert list(tdata['names']) == t.db.iris.fields
def test_add_data_to_empty_server(empty_server, serial):
# add data
with temp_server() as test:
iris_path = example('iris.csv')
blob = serial.dumps({'iris': iris_path})
response1 = empty_server.post(
'/add',
headers=mimetype(serial),
data=blob,
)
assert 'OK' in response1.status
assert response1.status_code == 200
# check for expected server datashape
response2 = empty_server.get('/datashape')
expected2 = str(discover({'iris': resource(iris_path)}))
assert response2.data.decode('utf-8') == expected2
# compute on added data
t = Data({'iris': resource(iris_path)})
expr = t.iris.petal_length.sum()
response3 = empty_server.post(
'/compute',
data=serial.dumps({'expr': to_tree(expr)}),
headers=mimetype(serial)
)
result3 = serial.loads(response3.data)['data']
expected3 = compute(expr, {'iris': resource(iris_path)})
assert result3 == expected3
def test_from_tree_is_robust_to_unnecessary_namespace():
t = symbol('t', 'var * {name: string, amount: int32}')
expr = t.amount + 1
tree = to_tree(expr) # don't use namespace
assert from_tree(tree, {'t': t}).isidentical(expr)
def test_auth(test_with_auth, username, password, serial):
expr = t.accounts.amount.sum()
query = {'expr': to_tree(expr)}
r = test_with_auth.get(
'/datashape',
headers={'authorization': basic_auth(username, password)},
)
assert r.status_code == 200
headers = mimetype(serial)
headers['authorization'] = basic_auth(username, password)
s = test_with_auth.post(
'/compute',
data=serial.dumps(query),
headers=headers,
)
assert s.status_code == 200
u = test_with_auth.get(
'/datashape',
headers={'authorization': basic_auth(username + 'a', password + 'a')},
)
assert u.status_code == 401
headers['authorization'] = basic_auth(username + 'a', password + 'a')
v = test_with_auth.post(
'/compute',
data=serial.dumps(query),
headers=headers,
)
assert v.status_code == 401
def test_add_data_to_server(temp_add_server, serial):
# add data
iris_path = example('iris.csv')
blob = serial.dumps({'iris': iris_path})
response1 = temp_add_server.post('/add',
headers=mimetype(serial),
data=blob)
assert 'CREATED' in response1.status
assert response1.status_code == RC.CREATED
# check for expected server datashape
response2 = temp_add_server.get('/datashape')
expected2 = discover({'iris': data(iris_path)})
response_dshape = datashape.dshape(response2.data.decode('utf-8'))
assert_dshape_equal(response_dshape.measure.dict['iris'],
expected2.measure.dict['iris'])
# compute on added data
t = data({'iris': data(iris_path)})
expr = t.iris.petal_length.sum()
response3 = temp_add_server.post('/compute',
data=serial.dumps({'expr': to_tree(expr)}),
headers=mimetype(serial))
result3 = serial.data_loads(serial.loads(response3.data)['data'])
expected3 = compute(expr, {'iris': data(iris_path)})
assert result3 == expected3
def test_to_tree():
t = symbol('t', 'var * {name: string, amount: int32}')
expr = t.amount.sum()
expected = {
'op': 'sum',
'args': [
{
'op': 'Field',
'args': [
{
'op': 'Symbol',
'args': [
't',
'var * {name: string, amount: int32}',
0,
]
},
'amount'
]
},
[0],
False,
],
}
assert to_tree(expr) == expected
def test_to_from_tree_namespace():
t = symbol('t', 'var * {name: string, amount: int32}')
expr = t.name
tree = to_tree(expr, names={t: 't'})
assert tree == {'op': 'Field', 'args': ['t', 'name']}
new = from_tree(tree, namespace={'t': t})
assert new.isidentical(expr)
def test_log_format_exc(temp_server_with_excfmt, serial):
expr = t.dumb.sort()
bad_query = {'expr': to_tree(expr)}
server, log_stream = temp_server_with_excfmt
result = server.post('/compute', headers=mimetype(serial),
data=serial.dumps(bad_query))
assert 'CUSTOM TRACEBACK' in log_stream.getvalue()
def test_server_can_compute_sqlalchemy_reductions(app_context):
expr = t.db.iris.petal_length.sum()
query = {'expr': to_tree(expr)}
response = test.post('/compute.json',
data=json.dumps(query),
content_type='application/json')
assert 'OK' in response.status
result = json.loads(response.data.decode('utf-8'))['data']
assert result == odo(compute(expr, {t: data}), int)
def test_cors_compute(test, serial, has_bokeh):
expr = t.db.iris.petal_length.sum()
res = test.post('/compute.{name}'.format(name=serial.name),
data=serial.dumps(dict(expr=to_tree(expr))))
assert res.status_code == 200
assert res.headers['Access-Control-Allow-Origin'] == '*'
assert 'HEAD' in res.headers['Access-Control-Allow-Methods']
assert 'OPTIONS' in res.headers['Access-Control-Allow-Methods']
assert 'GET' in res.headers['Access-Control-Allow-Methods']
def test_serialization_endpoints(serial):
expr = t.db.iris.petal_length.sum()
query = {'expr': to_tree(expr)}
response = test.post(
'/compute.{name}'.format(name=serial.name),
data=serial.dumps(query),
)
assert 'OK' in response.status
result = serial.loads(response.data)['data']
assert result == odo(compute(expr, {t: data}), int)
def test_compute(app_context):
expr = t.accounts.amount.sum()
query = {'expr': to_tree(expr)}
expected = 300
response = test.post('/compute.json',
data = json.dumps(query),
content_type='application/json')
assert 'OK' in response.status
assert json.loads(response.data.decode('utf-8'))['data'] == expected
def test_sqlalchemy_result(app_context):
expr = t.db.iris.head(5)
query = {'expr': to_tree(expr)}
response = test.post('/compute.json',
data = json.dumps(query),
content_type='application/json')
assert 'OK' in response.status
result = json.loads(response.data.decode('utf-8'))['data']
assert all(isinstance(item, (tuple, list)) for item in result)
def test_compute_by_with_summary(iris_server, iris):
test = iris_server
t = TableSymbol('t', iris.dshape)
expr = by(t.species, max=t.petal_length.max(), sum=t.petal_width.sum())
tree = to_tree(expr)
blob = json.dumps({'expr': tree})
resp = test.post('/compute/iris.json', data=blob,
content_type='application/json')
assert 'OK' in resp.status
result = json.loads(resp.data)['data']
expected = compute(expr, iris)
assert result == list(map(list, expected))
def test_compute(serial):
expr = t.accounts.amount.sum()
query = {'expr': to_tree(expr)}
expected = 300
response = test.post(
'/compute.{name}'.format(name=serial.name),
data=serial.dumps(query),
)
assert 'OK' in response.status
assert serial.loads(response.data)['data'] == expected
def test_sqlalchemy_result(serial):
expr = t.db.iris.head(5)
query = {'expr': to_tree(expr)}
response = test.post(
'/compute.{name}'.format(name=serial.name),
data=serial.dumps(query),
)
assert 'OK' in response.status
result = serial.loads(response.data)['data']
assert all(isinstance(item, (tuple, list)) for item in result)
def test_fastmsgmpack_mutable_dataframe(test):
expr = t.events # just get back the dataframe
query = {'expr': to_tree(expr)}
result = test.post('/compute',
headers=mimetype(fastmsgpack),
data=fastmsgpack.dumps(query))
assert result.status_code == RC.OK
data = fastmsgpack.data_loads(fastmsgpack.loads(result.data)['data'])
for block in data._data.blocks:
# make sure all the blocks are mutable
assert block.values.flags.writeable
def test_compute():
t = TableSymbol('t', '{name: string, amount: int}')
expr = t.amount.sum()
query = {'expr': to_tree(expr)}
expected = 300
response = test.post('/compute/accounts_df.json',
data = json.dumps(query),
content_type='application/json')
assert 'OK' in response.status
assert json.loads(response.data)['data'] == expected
def test_server_can_compute_sqlalchemy_reductions(test, serial):
expr = t.db.iris.petal_length.sum()
query = {'expr': to_tree(expr)}
response = test.post(
'/compute.{name}'.format(name=serial.name),
data=serial.dumps(query),
)
assert 'OK' in response.status
respdata = serial.loads(response.data)
result = respdata['data']
assert result == odo(compute(expr, {t: data}), int)
assert respdata['names'] == ['petal_length_sum']
def test_builtin_501_exception(iris_server, serial):
t = symbol('t', discover(iris))
for name in ('map', 'apply'):
func = getattr(t.species, name)
expr = func(copy, 'int')
query = {'expr': to_tree(expr)}
response = iris_server.post('/compute',
data=serial.dumps(query),
headers=mimetype(serial))
assert '501 Not Implemented'.lower() in response.status.lower()
def test_to_tree():
t = symbol('t', 'var * {name: string, amount: int32}')
expr = t.amount.sum()
dshape = datashape.dshape('var * {name: string, amount: int32}')
sum_args = [{'op': 'Field',
'args': [{'op': 'Symbol',
'args': ['t', dshape, 0]},
'amount']},
[0],
False]
expected = {'op': 'sum', 'args': sum_args}
assert to_tree(expr) == expected
def test_serialization_endpoints(test, serial):
expr = t.db.iris.petal_length.sum()
query = {'expr': to_tree(expr)}
response = test.post('/compute',
data=serial.dumps(query),
headers=mimetype(serial))
assert 'OK' in response.status
respdata = serial.loads(response.data)
result = serial.data_loads(respdata['data'])
assert result == into(int, compute(expr, {t: tdata}))
assert list(respdata['names']) == ['petal_length_sum']
def test_apply_client_server(iris_server, serial):
test = iris_server
t = symbol('t', discover(iris))
expr = t.species.apply(id, 'int') # Very dumb example...
query = {'expr': to_tree(expr)}
response = test.post('/compute',
data=serial.dumps(query),
headers=mimetype(serial))
assert 'OK' in response.status
respdata = serial.loads(response.data)
result = serial.data_loads(respdata['data'])
assert type(result) == type(compute(expr, {t: iris}, return_type=int))
def test_compute(test, serial):
expr = t.accounts.amount.sum()
query = {'expr': to_tree(expr)}
expected = 300
response = test.post('/compute',
data=serial.dumps(query),
headers=mimetype(serial))
assert 'OK' in response.status
tdata = serial.loads(response.data)
assert serial.data_loads(tdata['data']) == expected
assert list(tdata['names']) == ['amount_sum']
def test_compute(test, serial):
expr = t.accounts.amount.sum()
query = {'expr': to_tree(expr)}
expected = 300
response = test.post('/compute',
data=serial.dumps(query),
headers=mimetype(serial))
assert 'OK' in response.status
data = serial.loads(response.data)
assert data['data'] == expected
assert data['names'] == ['amount_sum']
def test_get_datetimes(app_context):
expr = t.events
query = {'expr': to_tree(expr)}
response = test.post('/compute.json',
data=json.dumps(query),
content_type='application/json')
assert 'OK' in response.status
data = json.loads(response.data.decode('utf-8'))
ds = datashape.dshape(data['datashape'])
result = into(np.ndarray, data['data'], dshape=ds)
assert into(list, result) == into(list, events)
def test_sqlalchemy_result(test, serial):
expr = t.db.iris.head(5)
query = {'expr': to_tree(expr)}
response = test.post(
'/compute.{name}'.format(name=serial.name),
data=serial.dumps(query),
)
assert 'OK' in response.status
data = serial.loads(response.data)
result = data['data']
assert all(isinstance(item, (tuple, list)) for item in result)
assert data['names'] == t.db.iris.fields
def test_multi_expression_compute():
s = symbol('s', discover(data))
expr = join(s.accounts, s.cities)
resp = test.post('/compute.json',
data=json.dumps({'expr': to_tree(expr)}),
content_type='application/json')
assert 'OK' in resp.status
result = json.loads(resp.data.decode('utf-8'))['data']
expected = compute(expr, {s: data})
assert list(map(tuple, result)) == into(list, expected)
def test_serialization_endpoints(test, serial):
expr = t.db.iris.petal_length.sum()
query = {'expr': to_tree(expr)}
response = test.post(
'/compute',
data=serial.dumps(query),
headers=mimetype(serial)
)
assert 'OK' in response.status
respdata = serial.loads(response.data)
result = respdata['data']
assert result == odo(compute(expr, {t: data}), int)
assert respdata['names'] == ['petal_length_sum']
def test_multi_expression_compute(app_context):
s = symbol('s', discover(data))
expr = join(s.accounts, s.cities)
resp = test.post('/compute.json',
data=json.dumps({'expr': to_tree(expr)}),
content_type='application/json')
assert 'OK' in resp.status
result = json.loads(resp.data.decode('utf-8'))['data']
expected = compute(expr, {s: data})
assert list(map(tuple, result))== into(list, expected)
def test_cors_compute(test, serial, has_bokeh):
expr = t.db.iris.petal_length.sum()
res = test.post('/compute',
data=serial.dumps(dict(expr=to_tree(expr))),
headers=mimetype(serial))
assert res.status_code == 200
assert res.headers['Access-Control-Allow-Origin'] == '*'
assert 'HEAD' in res.headers['Access-Control-Allow-Methods']
assert 'OPTIONS' in res.headers['Access-Control-Allow-Methods']
assert 'POST' in res.headers['Access-Control-Allow-Methods']
# we don't allow gets because we're always sending data
assert 'GET' not in res.headers['Access-Control-Allow-Methods']
def test_minute_query(test, serial):
expr = t.events.when.minute
query = {'expr': to_tree(expr)}
result = test.post('/compute',
headers=mimetype(serial),
data=serial.dumps(query))
expected = {'data': [0, 0],
'names': ['when_minute'],
'datashape': '2 * int64'}
assert result.status_code == RC.OK
resp = serial.loads(result.data)
assert list(serial.data_loads(resp['data'])) == expected['data']
assert list(resp['names']) == expected['names']
assert resp['datashape'] == expected['datashape']
def test_isin(test, serial):
expr = t.events.value.isin(frozenset([1]))
query = {'expr': to_tree(expr)}
result = test.post('/compute',
headers=mimetype(serial),
data=serial.dumps(query))
expected = {'data': [True, False],
'names': ['value'],
'datashape': '2 * bool'}
assert result.status_code == RC.OK
resp = serial.loads(result.data)
assert list(serial.data_loads(resp['data'])) == expected['data']
assert list(resp['names']) == expected['names']
assert resp['datashape'] == expected['datashape']
def test_custom_compute_hook(temp_server_with_custom_hook, serial):
expr = t.accounts.amount.sum()
query = {
'expr': to_tree(expr),
'compute_hook_kwargs': {
'status': RC.OK
},
}
response = temp_server_with_custom_hook.post('/compute',
data=serial.dumps(query),
headers=mimetype(serial))
assert response.status_code == RC.OK
query = {
'expr': to_tree(expr),
'compute_hook_kwargs': {
'status': RC.BAD_REQUEST
},
}
response = temp_server_with_custom_hook.post('/compute',
data=serial.dumps(query),
headers=mimetype(serial))
assert response.status_code == RC.BAD_REQUEST
def test_compute_by_with_summary(iris_server, serial):
test = iris_server
t = symbol('t', discover(iris))
expr = by(t.species, max=t.petal_length.max(), sum=t.petal_width.sum())
tree = to_tree(expr)
blob = serial.dumps({'expr': tree})
resp = test.post('/compute', data=blob, headers=mimetype(serial))
assert 'OK' in resp.status
tdata = serial.loads(resp.data)
result = DataFrame(serial.data_loads(tdata['data'])).values
expected = compute(expr, iris).values
np.testing.assert_array_equal(result[:, 0], expected[:, 0])
np.testing.assert_array_almost_equal(result[:, 1:], expected[:, 1:])
assert list(tdata['names']) == ['species', 'max', 'sum']
def test_get_datetimes(test, serial):
expr = t.events
query = {'expr': to_tree(expr)}
response = test.post('/compute',
data=serial.dumps(query),
headers=mimetype(serial))
assert 'OK' in response.status
tdata = serial.loads(response.data)
ds = datashape.dshape(tdata['datashape'])
result = into(np.ndarray, serial.data_loads(tdata['data']), dshape=ds)
assert into(list, result) == into(list, events)
assert list(tdata['names']) == events.columns.tolist()
def test_serialization_endpoints(test, serial, server):
client.requests = server.app.test_client()
t = data(bz.Client('localhost:6363'))
expr = t.db.iris.petal_length.sum()
query = {'expr': to_tree(expr)}
response = test.post('/compute',
data=serial.dumps(query),
headers=mimetype(serial))
assert 'OK' in response.status
respdata = serial.loads(response.data)
result = serial.data_loads(respdata['data'])
assert result == compute(expr, {t: tdata}, return_type='core')
assert list(respdata['names']) == ['petal_length_sum']
def test_isin(test, serial):
expr = t.events.value.isin(frozenset([1]))
query = {'expr': to_tree(expr)}
result = test.post('/compute',
headers=mimetype(serial),
data=serial.dumps(query))
expected = {'data': [True, False],
'names': ['value'],
'datashape': '2 * bool'}
assert result.status_code == RC.OK
resp = serial.loads(result.data)
assert list(serial.data_loads(resp['data'])) == expected['data']
assert list(resp['names']) == expected['names']
assert resp['datashape'] == expected['datashape']
def test_minute_query(test, serial):
expr = t.events.when.minute
query = {'expr': to_tree(expr)}
result = test.post('/compute',
headers=mimetype(serial),
data=serial.dumps(query))
expected = {'data': [0, 0],
'names': ['when_minute'],
'datashape': '2 * int64'}
assert result.status_code == RC.OK
resp = serial.loads(result.data)
assert list(serial.data_loads(resp['data'])) == expected['data']
assert list(resp['names']) == expected['names']
assert resp['datashape'] == expected['datashape']
def test_to_and_from_tree_with_literal():
data = frozenset([1, 2])
expr = literal(data)
as_tree = {
'args': [
frozenset({1, 2}),
datashape.dshape("2 * int64"),
None,
],
'op': 'Literal',
}
assert to_tree(expr) == as_tree
assert from_tree(as_tree).isidentical(expr)
def test_compute(test, serial):
expr = t.accounts.amount.sum()
query = {'expr': to_tree(expr)}
expected = 300
response = test.post(
'/compute.{name}'.format(name=serial.name),
data=serial.dumps(query),
)
assert 'OK' in response.status
data = serial.loads(response.data)
assert data['data'] == expected
assert data['names'] == ['amount_sum']
def test_compute_with_variable_in_namespace(iris_server):
test = iris_server
t = symbol('t', discover(iris))
pl = symbol('pl', 'float32')
expr = t[t.petal_length > pl].species
tree = to_tree(expr, {pl: 'pl'})
blob = json.dumps({'expr': tree, 'namespace': {'pl': 5}})
resp = test.post('/compute.json', data=blob,
content_type='application/json')
assert 'OK' in resp.status
result = json.loads(resp.data.decode('utf-8'))['data']
expected = list(compute(expr._subs({pl: 5}), {t: iris}))
assert result == expected
def test_multi_expression_compute():
a = Symbol('accounts', discover(accounts))
c = Symbol('cities', discover(cities))
expr = join(a, c)
resp = test.post('/compute.json',
data=json.dumps({'expr': to_tree(expr)}),
content_type='application/json')
assert 'OK' in resp.status
result = json.loads(resp.data)['data']
expected = compute(expr, {a: accounts, c: cities})
assert list(map(tuple, result)) == into(list, expected)
def test_compute_column_wise(iris_server):
test = iris_server
t = symbol('t', discover(iris))
subexpr = ((t.petal_width / 2 > 0.5) & (t.petal_length / 2 > 0.5))
expr = t[subexpr]
tree = to_tree(expr)
blob = json.dumps({'expr': tree})
resp = test.post('/compute.json',
data=blob,
content_type='application/json')
assert 'OK' in resp.status
result = json.loads(resp.data.decode('utf-8'))['data']
expected = compute(expr, iris)
assert list(map(tuple, result)) == into(list, expected)
def test_compute_kwargs(test, serial):
expr = t.dumb.sort()
bad_query = {'expr': to_tree(expr)}
result = test.post('/compute',
headers=mimetype(serial),
data=serial.dumps(bad_query))
assert result.status_code == RC.INTERNAL_SERVER_ERROR
assert b'return_df must be passed' in result.data
good_query = {'expr': to_tree(expr),
'compute_kwargs': {'return_df': odo(DumbResource.df, list)}}
result = test.post('/compute',
headers=mimetype(serial),
data=serial.dumps(good_query))
assert result.status_code == RC.OK
tdata = serial.loads(result.data)
dshape = discover(DumbResource.df)
assert_dshape_equal(datashape.dshape(tdata['datashape']),
dshape)
assert_frame_equal(odo(serial.data_loads(tdata['data']),
DataFrame,
dshape=dshape),
DumbResource.df)
def test_get_datetimes(test, serial):
expr = t.events
query = {'expr': to_tree(expr)}
response = test.post(
'/compute.{name}'.format(name=serial.name),
data=serial.dumps(query),
)
assert 'OK' in response.status
data = serial.loads(response.data)
ds = datashape.dshape(data['datashape'])
result = into(np.ndarray, data['data'], dshape=ds)
assert into(list, result) == into(list, events)
assert data['names'] == events.columns.tolist()
def test_compute_column_wise(iris_server, serial):
test = iris_server
t = symbol('t', discover(iris))
subexpr = ((t.petal_width / 2 > 0.5) & (t.petal_length / 2 > 0.5))
expr = t[subexpr]
tree = to_tree(expr)
blob = serial.dumps({'expr': tree})
resp = test.post('/compute', data=blob, headers=mimetype(serial))
assert 'OK' in resp.status
tdata = serial.loads(resp.data)
result = serial.data_loads(tdata['data'])
expected = compute(expr, iris)
assert list(map(tuple, into(list, result))) == into(list, expected)
assert list(tdata['names']) == t.fields
def test_to_tree():
t = symbol('t', 'var * {name: string, amount: int32}')
expr = t.amount.sum()
expected = {
'op':
'sum',
'args': [{
'op':
'Field',
'args': [{
'op': 'Symbol',
'args': ['t', 'var * {name: string, amount: int32}', None]
}, 'amount']
}, [0], False]
}
assert to_tree(expr) == expected
def test_multi_expression_compute(test, serial):
s = symbol('s', discover(tdata))
expr = join(s.accounts, s.cities)
resp = test.post('/compute',
data=serial.dumps({'expr': to_tree(expr)}),
headers=mimetype(serial))
assert 'OK' in resp.status
respdata = serial.loads(resp.data)
result = serial.data_loads(respdata['data'])
expected = compute(expr, {s: tdata})
assert list(map(tuple, odo(result, list))) == into(list, expected)
assert list(respdata['names']) == expr.fields
def test_compute_with_variable_in_namespace(iris_server, serial):
test = iris_server
t = symbol('t', discover(iris))
pl = symbol('pl', 'float32')
expr = t[t.petal_length > pl].species
tree = to_tree(expr, {pl: 'pl'})
blob = serial.dumps({'expr': tree, 'namespace': {'pl': 5}})
resp = test.post('/compute', data=blob, headers=mimetype(serial))
assert 'OK' in resp.status
tdata = serial.loads(resp.data)
result = serial.data_loads(tdata['data'])
expected = list(compute(expr._subs({pl: 5}), {t: iris}))
assert odo(result, list) == expected
assert list(tdata['names']) == ['species']
def test_isin_expr(test, serial):
name_filter = t.accounts[t.accounts.amount > 100].name
expr = t.cities.name.isin(name_filter)
query = {'expr': to_tree(expr)}
result = test.post('/compute',
headers=mimetype(serial),
data=serial.dumps(query))
expected = {
'data': [False, True],
'names': ['name'],
'datashape': '2 * bool'
}
assert result.status_code == RC.OK
resp = serial.loads(result.data)
assert list(serial.data_loads(resp['data'])) == expected['data']
assert list(resp['names']) == expected['names']
assert_dshape_equal(resp['datashape'], expected['datashape'])
def test_builtin_403_exception(iris_server, serial):
'''
ensure exception is raised when both map and apply are invoked.
exception is raised in check_request() when object_hook is invoked;
this is when the payload is loaded from the bytes object in reqeust.data
'''
t = symbol('t', discover(iris))
for name in ('map', 'apply'):
func = getattr(t.species, name)
expr = func(eval, 'int')
query = {'expr': to_tree(expr)}
response = iris_server.post('/compute',
data=serial.dumps(query),
headers=mimetype(serial))
assert '403 FORBIDDEN'.lower() in response.status.lower()
def serialize_query_with_map_builtin_function(test, serial, fcn):
"""
serialize a query that invokes the 'map' operation using a builtin function
return the result of the post operation along with expected result
"""
t = symbol('t', discover(iris))
expr = t.species.map(fcn, 'int')
query = {'expr': to_tree(expr)}
response = test.post('/compute',
data=serial.dumps(query),
headers=mimetype(serial))
assert 'OK' in response.status
respdata = serial.loads(response.data)
result = serial.data_loads(respdata['data'])
exp_res = compute(expr, {t: iris}, return_type=list)
return (exp_res, result)
def test_multi_expression_compute(test, serial):
s = symbol('s', discover(data))
expr = join(s.accounts, s.cities)
resp = test.post(
'/compute.{name}'.format(name=serial.name),
data=serial.dumps({'expr': to_tree(expr)}),
)
assert 'OK' in resp.status
respdata = serial.loads(resp.data)
result = respdata['data']
expected = compute(expr, {s: data})
assert list(map(tuple, result)) == into(list, expected)
assert respdata['names'] == expr.fields
def test_compute_column_wise(iris_server, serial):
test = iris_server
t = symbol('t', discover(iris))
subexpr = ((t.petal_width / 2 > 0.5) & (t.petal_length / 2 > 0.5))
expr = t[subexpr]
tree = to_tree(expr)
blob = serial.dumps({'expr': tree})
resp = test.post(
'/compute.{name}'.format(name=serial.name),
data=blob,
)
assert 'OK' in resp.status
data = serial.loads(resp.data)
result = data['data']
expected = compute(expr, iris)
assert list(map(tuple, result)) == into(list, expected)
assert data['names'] == t.fields
def test_add_data_twice_error(temp_server, serial):
# add iris
iris_path = example('iris.csv')
payload = serial.dumps({'iris': iris_path})
temp_server.post('/add', headers=mimetype(serial), data=payload)
# Try to add to existing 'iris'
resp = temp_server.post('/add', headers=mimetype(serial), data=payload)
assert resp.status_code == RC.CONFLICT
# Verify the server still serves the original 'iris'.
ds = datashape.dshape(temp_server.get('/datashape').data.decode('utf-8'))
t = symbol('t', ds)
query = {'expr': to_tree(t.iris)}
resp = temp_server.post('/compute',
data=serial.dumps(query),
headers=mimetype(serial))
assert resp.status_code == RC.OK
def test_compute_with_variable_in_namespace(iris_server, serial):
test = iris_server
t = symbol('t', discover(iris))
pl = symbol('pl', 'float32')
expr = t[t.petal_length > pl].species
tree = to_tree(expr, {pl: 'pl'})
blob = serial.dumps({'expr': tree, 'namespace': {'pl': 5}})
resp = test.post(
'/compute.{name}'.format(name=serial.name),
data=blob,
)
assert 'OK' in resp.status
data = serial.loads(resp.data)
result = data['data']
expected = list(compute(expr._subs({pl: 5}), {t: iris}))
assert result == expected
assert data['names'] == ['species']
def test_to_tree():
t = symbol('t', 'var * {name: string, amount: int32}')
expr = t.amount.sum()
dshape = datashape.dshape('var * {name: string, amount: int32}')
sum_args = [
{
'op': 'Field',
'args': [
{
'op': 'Symbol',
'args': ['t', dshape, 0],
},
'amount',
],
},
[0],
False,
]
expected = {'op': 'sum', 'args': sum_args}
assert to_tree(expr) == expected
