def test_filter_df_for_grid(test_data): req = build_req_tuple({ 'filters': json.dumps({ 'security_id': {'type': 'NumericFilter', 'value': [{'value': 1, 'type': 1}]}, 'bar': {'type': 'NumericFilter', 'value': [{'value': 1, 'type': 3}]}, 'baz': {'type': 'StringFilter', 'value': 'baz'} }) }) results = utils.filter_df_for_grid(test_data, utils.retrieve_grid_params(req), dict()) pdt.assert_frame_equal(results, test_data[test_data.security_id == 1]) req = build_req_tuple({ 'filters': json.dumps({ 'security_id': {'type': 'NumericFilter', 'value': [{'value': 1, 'type': 1}]}, 'bar': {'type': 'NumericFilter', 'value': [{'begin': 1, 'end': 2, 'type': 2}]}, 'baz': {'type': 'StringFilter', 'value': '=baz'} }) }) results = utils.filter_df_for_grid(test_data, utils.retrieve_grid_params(req), dict()) pdt.assert_frame_equal(results, test_data[test_data.security_id == 1]) req = build_req_tuple({ 'filters': json.dumps({ 'security_id': {'type': 'NumericFilter', 'value': [{'value': 1, 'type': 1}]}, 'bar': {'type': 'NumericFilter', 'value': [{'value': 2, 'type': 4}]}, 'date': {'type': 'StringFilter', 'value': '2000-01-01'} }) }) results = utils.filter_df_for_grid(test_data, utils.retrieve_grid_params(req), dict()) pdt.assert_frame_equal(results, test_data[test_data.security_id == 1]) req = build_req_tuple({'query': 'security_id == 1'}) results = utils.filter_df_for_grid(test_data, utils.retrieve_grid_params(req), dict()) pdt.assert_frame_equal(results, test_data[test_data.security_id == 1]) req = build_req_tuple({'page': 1, 'page_size': 50}) page, page_size = utils.retrieve_grid_params(req, props=['page', 'page_size']) assert page == 1 assert page_size == 50
def get_data(data_id): """ :class:`flask:flask.Flask` route which returns current rows from DATA (based on scrollbar specs and saved settings) to front-end as JSON :param data_id: integer string identifier for a D-Tale process's data :type data_id: str :param ids: required dash separated string "START-END" stating a range of row indexes to be returned to the screen :param query: string from flask.request.args['query'] which is applied to DATA using the query() function :param sort: JSON string from flask.request.args['sort'] which is applied to DATA using the sort_values() or sort_index() function. Here is the JSON structure: [col1,dir1],[col2,dir2],....[coln,dirn] :return: JSON { results: [ {dtale_index: 1, col1: val1_1, ...,colN: valN_1}, ..., {dtale_index: N2, col1: val1_N2, ...,colN: valN_N2} ], columns: [{name: col1, dtype: 'int64'},...,{name: colN, dtype: 'datetime'}], total: N2, success: True/False } """ try: global SETTINGS, DATA, DTYPES data = DATA[data_id] # this will check for when someone instantiates D-Tale programatically and directly alters the internal # state of the dataframe (EX: d.data['new_col'] = 'foo') curr_dtypes = [c['name'] for c in DTYPES[data_id]] if any(c not in curr_dtypes for c in data.columns): data, _ = format_data(data) DATA[data_id] = data DTYPES[data_id] = build_dtypes_state(data) params = retrieve_grid_params(request) ids = get_json_arg(request, 'ids') if ids is None: return jsonify({}) col_types = DTYPES[data_id] f = grid_formatter(col_types) curr_settings = SETTINGS.get(data_id, {}) if curr_settings.get('sort') != params.get('sort'): data = sort_df_for_grid(data, params) DATA[data_id] = data if params.get('sort') is not None: curr_settings = dict_merge(curr_settings, dict(sort=params['sort'])) else: curr_settings = {k: v for k, v in curr_settings.items() if k != 'sort'} data = filter_df_for_grid(data, params) if params.get('query') is not None: curr_settings = dict_merge(curr_settings, dict(query=params['query'])) else: curr_settings = {k: v for k, v in curr_settings.items() if k != 'query'} SETTINGS[data_id] = curr_settings total = len(data) results = {} for sub_range in ids: sub_range = list(map(int, sub_range.split('-'))) if len(sub_range) == 1: sub_df = data.iloc[sub_range[0]:sub_range[0] + 1] sub_df = f.format_dicts(sub_df.itertuples()) results[sub_range[0]] = dict_merge({IDX_COL: sub_range[0]}, sub_df[0]) else: [start, end] = sub_range sub_df = data.iloc[start:] if end >= len(data) - 1 else data.iloc[start:end + 1] sub_df = f.format_dicts(sub_df.itertuples()) for i, d in zip(range(start, end + 1), sub_df): results[i] = dict_merge({IDX_COL: i}, d) return_data = dict(results=results, columns=[dict(name=IDX_COL, dtype='int64')] + DTYPES[data_id], total=total) return jsonify(return_data) except BaseException as e: return jsonify(dict(error=str(e), traceback=str(traceback.format_exc())))
def get_data(): """ Flask route which returns current rows from DATA (based on scrollbar specs and saved settings) to front-end as JSON :param ids: required dash separated string "START-END" stating a range of row indexes to be returned to the screen :param query: string from flask.request.args['query'] which is applied to DATA using the query() function :param sort: JSON string from flask.request.args['sort'] which is applied to DATA using the sort_values() or sort_index() function. Here is the JSON structure: [col1,dir1],[col2,dir2],....[coln,dirn] :param port: number string from flask.request.environ['SERVER_PORT'] for retrieving saved settings :return: JSON { results: [ {dtale_index: 1, col1: val1_1, ...,colN: valN_1}, ..., {dtale_index: N2, col1: val1_N2, ...,colN: valN_N2}. ], columns: [{name: col1, dtype: 'int64'},...,{name: colN, dtype: 'datetime'}], total: N2, success: True/False } """ try: global SETTINGS, DATA params = retrieve_grid_params(request) ids = get_str_arg(request, 'ids') if ids: ids = json.loads(ids) else: return jsonify({}) col_types = grid_columns(DATA) f = grid_formatter(col_types) curr_settings = SETTINGS.get( request.environ.get('SERVER_PORT', 'curr'), {}) if curr_settings.get('sort') != params.get('sort'): DATA = sort_df_for_grid(DATA, params) df = DATA if params.get('sort') is not None: curr_settings = dict_merge(curr_settings, dict(sort=params['sort'])) else: curr_settings = { k: v for k, v in curr_settings.items() if k != 'sort' } df = filter_df_for_grid(df, params) if params.get('query') is not None: curr_settings = dict_merge(curr_settings, dict(query=params['query'])) else: curr_settings = { k: v for k, v in curr_settings.items() if k != 'query' } SETTINGS[request.environ.get('SERVER_PORT', 'curr')] = curr_settings total = len(df) results = {} for sub_range in ids: sub_range = list(map(int, sub_range.split('-'))) if len(sub_range) == 1: sub_df = df.iloc[sub_range[0]:sub_range[0] + 1] sub_df = f.format_dicts(sub_df.itertuples()) results[sub_range[0]] = dict_merge( dict(dtale_index=sub_range[0]), sub_df[0]) else: [start, end] = sub_range sub_df = df.iloc[start:] if end >= len( df) - 1 else df.iloc[start:end + 1] sub_df = f.format_dicts(sub_df.itertuples()) for i, d in zip(range(start, end + 1), sub_df): results[i] = dict_merge(dict(dtale_index=i), d) return_data = dict(results=results, columns=[dict(name='dtale_index', dtype='int64')] + col_types, total=total) return jsonify(return_data) except BaseException as e: return jsonify( dict(error=str(e), traceback=str(traceback.format_exc())))