def test_filter_df_for_grid(test_data):
req = build_req_tuple({
'filters': json.dumps({
'security_id': {'type': 'NumericFilter', 'value': [{'value': 1, 'type': 1}]},
'bar': {'type': 'NumericFilter', 'value': [{'value': 1, 'type': 3}]},
'baz': {'type': 'StringFilter', 'value': 'baz'}
})
})
results = utils.filter_df_for_grid(test_data, utils.retrieve_grid_params(req), dict())
pdt.assert_frame_equal(results, test_data[test_data.security_id == 1])
req = build_req_tuple({
'filters': json.dumps({
'security_id': {'type': 'NumericFilter', 'value': [{'value': 1, 'type': 1}]},
'bar': {'type': 'NumericFilter', 'value': [{'begin': 1, 'end': 2, 'type': 2}]},
'baz': {'type': 'StringFilter', 'value': '=baz'}
})
})
results = utils.filter_df_for_grid(test_data, utils.retrieve_grid_params(req), dict())
pdt.assert_frame_equal(results, test_data[test_data.security_id == 1])
req = build_req_tuple({
'filters': json.dumps({
'security_id': {'type': 'NumericFilter', 'value': [{'value': 1, 'type': 1}]},
'bar': {'type': 'NumericFilter', 'value': [{'value': 2, 'type': 4}]},
'date': {'type': 'StringFilter', 'value': '2000-01-01'}
})
})
results = utils.filter_df_for_grid(test_data, utils.retrieve_grid_params(req), dict())
pdt.assert_frame_equal(results, test_data[test_data.security_id == 1])
req = build_req_tuple({'query': 'security_id == 1'})
results = utils.filter_df_for_grid(test_data, utils.retrieve_grid_params(req), dict())
pdt.assert_frame_equal(results, test_data[test_data.security_id == 1])
req = build_req_tuple({'page': 1, 'page_size': 50})
page, page_size = utils.retrieve_grid_params(req, props=['page', 'page_size'])
assert page == 1
assert page_size == 50
def get_data(data_id):
"""
:class:`flask:flask.Flask` route which returns current rows from DATA (based on scrollbar specs and saved settings)
to front-end as JSON
:param data_id: integer string identifier for a D-Tale process's data
:type data_id: str
:param ids: required dash separated string "START-END" stating a range of row indexes to be returned to the screen
:param query: string from flask.request.args['query'] which is applied to DATA using the query() function
:param sort: JSON string from flask.request.args['sort'] which is applied to DATA using the sort_values() or
sort_index() function. Here is the JSON structure: [col1,dir1],[col2,dir2],....[coln,dirn]
:return: JSON {
results: [
{dtale_index: 1, col1: val1_1, ...,colN: valN_1},
...,
{dtale_index: N2, col1: val1_N2, ...,colN: valN_N2}
],
columns: [{name: col1, dtype: 'int64'},...,{name: colN, dtype: 'datetime'}],
total: N2,
success: True/False
}
"""
try:
global SETTINGS, DATA, DTYPES
data = DATA[data_id]
# this will check for when someone instantiates D-Tale programatically and directly alters the internal
# state of the dataframe (EX: d.data['new_col'] = 'foo')
curr_dtypes = [c['name'] for c in DTYPES[data_id]]
if any(c not in curr_dtypes for c in data.columns):
data, _ = format_data(data)
DATA[data_id] = data
DTYPES[data_id] = build_dtypes_state(data)
params = retrieve_grid_params(request)
ids = get_json_arg(request, 'ids')
if ids is None:
return jsonify({})
col_types = DTYPES[data_id]
f = grid_formatter(col_types)
curr_settings = SETTINGS.get(data_id, {})
if curr_settings.get('sort') != params.get('sort'):
data = sort_df_for_grid(data, params)
DATA[data_id] = data
if params.get('sort') is not None:
curr_settings = dict_merge(curr_settings, dict(sort=params['sort']))
else:
curr_settings = {k: v for k, v in curr_settings.items() if k != 'sort'}
data = filter_df_for_grid(data, params)
if params.get('query') is not None:
curr_settings = dict_merge(curr_settings, dict(query=params['query']))
else:
curr_settings = {k: v for k, v in curr_settings.items() if k != 'query'}
SETTINGS[data_id] = curr_settings
total = len(data)
results = {}
for sub_range in ids:
sub_range = list(map(int, sub_range.split('-')))
if len(sub_range) == 1:
sub_df = data.iloc[sub_range[0]:sub_range[0] + 1]
sub_df = f.format_dicts(sub_df.itertuples())
results[sub_range[0]] = dict_merge({IDX_COL: sub_range[0]}, sub_df[0])
else:
[start, end] = sub_range
sub_df = data.iloc[start:] if end >= len(data) - 1 else data.iloc[start:end + 1]
sub_df = f.format_dicts(sub_df.itertuples())
for i, d in zip(range(start, end + 1), sub_df):
results[i] = dict_merge({IDX_COL: i}, d)
return_data = dict(results=results, columns=[dict(name=IDX_COL, dtype='int64')] + DTYPES[data_id], total=total)
return jsonify(return_data)
except BaseException as e:
return jsonify(dict(error=str(e), traceback=str(traceback.format_exc())))
def get_data():
"""
Flask route which returns current rows from DATA (based on scrollbar specs and saved settings) to front-end as
JSON
:param ids: required dash separated string "START-END" stating a range of row indexes to be returned to the screen
:param query: string from flask.request.args['query'] which is applied to DATA using the query() function
:param sort: JSON string from flask.request.args['sort'] which is applied to DATA using the sort_values() or
sort_index() function. Here is the JSON structure: [col1,dir1],[col2,dir2],....[coln,dirn]
:param port: number string from flask.request.environ['SERVER_PORT'] for retrieving saved settings
:return: JSON {
results: [
{dtale_index: 1, col1: val1_1, ...,colN: valN_1},
...,
{dtale_index: N2, col1: val1_N2, ...,colN: valN_N2}.
],
columns: [{name: col1, dtype: 'int64'},...,{name: colN, dtype: 'datetime'}],
total: N2,
success: True/False
}
"""
try:
global SETTINGS, DATA
params = retrieve_grid_params(request)
ids = get_str_arg(request, 'ids')
if ids:
ids = json.loads(ids)
else:
return jsonify({})
col_types = grid_columns(DATA)
f = grid_formatter(col_types)
curr_settings = SETTINGS.get(
request.environ.get('SERVER_PORT', 'curr'), {})
if curr_settings.get('sort') != params.get('sort'):
DATA = sort_df_for_grid(DATA, params)
df = DATA
if params.get('sort') is not None:
curr_settings = dict_merge(curr_settings,
dict(sort=params['sort']))
else:
curr_settings = {
k: v
for k, v in curr_settings.items() if k != 'sort'
}
df = filter_df_for_grid(df, params)
if params.get('query') is not None:
curr_settings = dict_merge(curr_settings,
dict(query=params['query']))
else:
curr_settings = {
k: v
for k, v in curr_settings.items() if k != 'query'
}
SETTINGS[request.environ.get('SERVER_PORT', 'curr')] = curr_settings
total = len(df)
results = {}
for sub_range in ids:
sub_range = list(map(int, sub_range.split('-')))
if len(sub_range) == 1:
sub_df = df.iloc[sub_range[0]:sub_range[0] + 1]
sub_df = f.format_dicts(sub_df.itertuples())
results[sub_range[0]] = dict_merge(
dict(dtale_index=sub_range[0]), sub_df[0])
else:
[start, end] = sub_range
sub_df = df.iloc[start:] if end >= len(
df) - 1 else df.iloc[start:end + 1]
sub_df = f.format_dicts(sub_df.itertuples())
for i, d in zip(range(start, end + 1), sub_df):
results[i] = dict_merge(dict(dtale_index=i), d)
return_data = dict(results=results,
columns=[dict(name='dtale_index', dtype='int64')] +
col_types,
total=total)
return jsonify(return_data)
except BaseException as e:
return jsonify(
dict(error=str(e), traceback=str(traceback.format_exc())))
