def input_data(_ts, chart_type, x, y_multi, y_single, z, group, agg, window, rolling_comp, pathname, query): """ dash callback for maintaining chart input state and column-based dropdown options. This will guard against users selecting the same column for multiple axes. """ y_val = make_list(y_single if chart_type in ZAXIS_CHARTS else y_multi) inputs = dict(query=query, chart_type=chart_type, x=x, y=y_val, z=z, group=group, agg=agg, window=window, rolling_comp=rolling_comp) data_id = get_data_id(pathname) cols = DATA[data_id].columns dtypes = get_dtypes(DATA[data_id]) def build_selections(*args): return flatten_lists([[] if a is None else make_list(a) for a in args]) def build_cols(): for c in cols: if classify_type(dtypes[c]) == 'D': for freq in FREQS: if freq in FREQ_LABELS: yield '{}|{}'.format(c, freq), '{} ({})'.format(c, FREQ_LABELS[freq]) else: yield c, c else: yield c, c col_opts = list(build_cols()) group_val, z_val = (None, z) if chart_type in ZAXIS_CHARTS else (group, None) x_options = [build_option(c, l) for c, l in col_opts if c not in build_selections(y_val, z_val, group_val)] y_filter = build_selections(x, group_val, z_val) y_multi_options = [build_option(c, l) for c, l in col_opts if c not in y_filter] y_single_options = [build_option(c, l) for c, l in col_opts if c not in y_filter] z_options = [build_option(c) for c in cols if c not in build_selections(x, y_val, group_val)] group_options = [build_option(c, l) for c, l in col_opts if c not in build_selections(x, y_val, z_val)] return inputs, x_options, y_single_options, y_multi_options, z_options, group_options
def describe(column): """ Flask route which returns standard details about column data using :meth:`pandas:pandas.DataFrame.describe` to the front-end as JSON :param column: required dash separated string "START-END" stating a range of row indexes to be returned to the screen :return: JSON { describe: object representing output from :meth:`pandas:pandas.Series.describe`, unique_data: array of unique values when data has <= 100 unique values success: True/False } """ try: data = DATA[get_port()] desc = load_describe(data[column]) return_data = dict(describe=desc, success=True) uniq_vals = data[column].unique() if 'unique' not in return_data['describe']: return_data['describe']['unique'] = json_int(len(uniq_vals), as_string=True) if len(uniq_vals) <= 100: uniq_f = find_dtype_formatter(get_dtypes(data)[column]) return_data['uniques'] = [ uniq_f(u, nan_display='N/A') for u in uniq_vals ] return jsonify(return_data) except BaseException as e: return jsonify( dict(error=str(e), traceback=str(traceback.format_exc())))
def build_input_options(df, **inputs): """ Builds dropdown options for (X, Y, Z, Group, Barsort & Y-Axis Ranges) with filtering based on currently selected values for the following inputs: x, y, z, group. """ [chart_type, x, y, z, group] = [inputs.get(p) for p in ['chart_type', 'x', 'y', 'z', 'group']] col_opts = list(build_cols(df.columns, get_dtypes(df))) group_val, z_val = (None, z) if chart_type in ZAXIS_CHARTS else (group, None) x_options = [ build_option(c, l) for c, l in col_opts if c not in build_selections(y, z_val, group_val) ] y_filter = build_selections(x, group_val, z_val) y_multi_options = [ build_option(c, l) for c, l in col_opts if c not in y_filter ] y_single_options = [ build_option(c, l) for c, l in col_opts if c not in y_filter ] z_options = [ build_option(c) for c in df.columns if c not in build_selections(x, y, group_val) ] group_options = [ build_option(c, l) for c, l in col_opts if c not in build_selections(x, y, z_val) ] barsort_options = [build_option(o) for o in build_selections(x, y)] yaxis_options = [build_option(y2) for y2 in y or []] return x_options, y_multi_options, y_single_options, z_options, group_options, barsort_options, yaxis_options
def date_freq_handler(df): """ This returns a column definition handler which returns a series based on the specs from the front-end. Column definitions can be a column name 'Col1' or a column name with a frequency 'Col1|M' for columns which are of type datetime. :Example: Col1 -> returns series for Col1 Col1|M -> returns series for Col1 in monthly format with name 'Col1|M' :param df: dataframe whose data needs to be checked :type df: :class:`pandas:pandas.DataFrame` :return: handler function :rtype: func """ dtypes = get_dtypes(df) orig_idx = df.index def _handler(col_def): col_def_segs = col_def.split('|') if len(col_def_segs) > 1 and classify_type( dtypes[col_def_segs[0]]) == 'D': col, freq = col_def_segs if freq == 'WD': freq_grp = df.set_index(col).index.dayofweek.values elif freq == 'H2': freq_grp = df.set_index(col).index.hour.values else: freq_grp = df.set_index(col).index.to_period( freq).to_timestamp(how='end').values freq_grp = pd.Series(freq_grp, index=orig_idx, name=col_def) return freq_grp return df[col_def] return _handler
def animate_styles(df, **inputs): chart_type, agg, cpg = (inputs.get(p) for p in ['chart_type', 'agg', 'cpg']) opts = [] if cpg or agg in ['pctsum', 'pctct']: return dict(display='none'), dict(display='none'), opts if chart_type in ANIMATION_CHARTS: return dict(display='block'), dict(display='none'), opts if chart_type in ANIMATE_BY_CHARTS: opts = [build_option(v, l) for v, l in build_cols(df.columns, get_dtypes(df))] if len(opts): return dict(display='none'), dict(display='block'), opts return dict(display='none'), dict(display='none'), []
def build_group_inputs_filter(df, group_inputs): dtypes = get_dtypes(df) def _group_filter(group_val): for gc, gv in group_val.items(): classifier = classify_type(dtypes[gc]) yield group_filter_handler(gc, gv, classifier) def _full_filter(): for group_val in group_inputs: group_filter = " and ".join(list(_group_filter(group_val))) yield group_filter filters = list(_full_filter()) return "({})".format(") or (".join(filters))
def animate_by_style(df, **inputs): chart_type, cpg = (inputs.get(p) for p in ['chart_type', 'cpg']) if cpg: return dict(display='none'), [] opts = [] if chart_type in ANIMATE_BY_CHARTS: opts = [ build_option(v, l) for v, l in build_cols(df.columns, get_dtypes(df)) ] if chart_type in ANIMATION_CHARTS: opts = [build_option('chart_values', 'Values in Chart')] if len(opts): return dict(display='block'), opts return dict(display='none'), []
def describe(data_id, column): """ :class:`flask:flask.Flask` route which returns standard details about column data using :meth:`pandas:pandas.DataFrame.describe` to the front-end as JSON :param data_id: integer string identifier for a D-Tale process's data :type data_id: str :param column: required dash separated string "START-END" stating a range of row indexes to be returned to the screen :return: JSON { describe: object representing output from :meth:`pandas:pandas.Series.describe`, unique_data: array of unique values when data has <= 100 unique values success: True/False } """ try: data = DATA[data_id][[column]] additional_aggs = None dtype = next((dtype_info['dtype'] for dtype_info in DTYPES[data_id] if dtype_info['name'] == column), None) if classify_type(dtype) in ['I', 'F']: additional_aggs = [ 'sum', 'median', 'mode', 'var', 'sem', 'skew', 'kurt' ] desc = load_describe(data[column], additional_aggs=additional_aggs) return_data = dict(describe=desc, success=True) uniq_vals = data[column].unique() if 'unique' not in return_data['describe']: return_data['describe']['unique'] = json_int(len(uniq_vals), as_string=True) uniq_f = find_dtype_formatter(get_dtypes(data)[column]) if len(uniq_vals) <= 100: return_data['uniques'] = dict(data=[uniq_f(u) for u in uniq_vals], top=False) else: # get top 100 most common values uniq_vals = data[column].value_counts().sort_values( ascending=False).head(100).index.values return_data['uniques'] = dict(data=[uniq_f(u) for u in uniq_vals], top=True) return jsonify(return_data) except BaseException as e: return jsonify( dict(error=str(e), traceback=str(traceback.format_exc())))
def date_freq_handler(df): """ This returns a column definition handler which returns a series based on the specs from the front-end. Column definitions can be a column name 'Col1' or a column name with a frequency 'Col1|M' for columns which are of type datetime. :Example: Col1 -> returns series for Col1 Col1|M -> returns series for Col1 in monthly format with name 'Col1|M' :param df: dataframe whose data needs to be checked :type df: :class:`pandas:pandas.DataFrame` :return: handler function :rtype: func """ dtypes = get_dtypes(df) orig_idx = df.index def _handler(col_def): col_def_segs = col_def.split("|") if len(col_def_segs) > 1 and classify_type(dtypes[col_def_segs[0]]) == "D": col, freq = col_def_segs if freq == "WD": code = "df.set_index('{col}').index.dayofweek.values" freq_grp = df.set_index(col).index.dayofweek.values elif freq == "H2": code = "df.set_index('{col}').index.hour.values" freq_grp = df.set_index(col).index.hour.values else: code = "df.set_index('{col}').index.to_period('{freq}').to_timestamp(how='end').values" freq_grp = ( df.set_index(col) .index.to_period(freq) .to_timestamp(how="end") .values ) code = "\tpd.Series(" + code + ", index=df.index, name='{col_def}')," freq_grp = pd.Series(freq_grp, index=orig_idx, name=col_def) return freq_grp, code.format(col=col, freq=freq, col_def=col_def) else: return df[col_def], "\tdf['{col_def}'],".format(col_def=col_def) return _handler
def retrieve_chart_data(df, *args, **kwargs): """ Retrieves data from a dataframe for x, y, z & group inputs complete with date frequency formatting (:meth:`dtale.charts.utils.date_freq_handler`) if specified :param df: dataframe that contains data for chart :type df: :class:`pandas:pandas.DataFrame` :param args: columns to use :type args: iterable of str :return: dataframe of data required for chart construction :rtype: :class:`pandas:pandas.DataFrame` """ freq_handler = date_freq_handler(df) cols = flatten_lists([make_list(a) for a in args]) all_code = [] all_data = [] for col in cols: if col is not None: s, code = freq_handler(col) all_data.append(s) if code is not None: all_code.append(code) all_data = pd.concat(all_data, axis=1) all_code = ["chart_data = pd.concat(["] + all_code + ["], axis=1)"] if len(make_list(kwargs.get('group_val'))): dtypes = get_dtypes(all_data) def _group_filter(group_val): for gc, gv in group_val.items(): classifier = classify_type(dtypes[gc]) yield group_filter_handler(gc, gv, classifier) def _full_filter(): for group_val in kwargs['group_val']: group_filter = ' and '.join(list(_group_filter(group_val))) yield group_filter filters = list(_full_filter()) filters = '({})'.format(') or ('.join(filters)) all_data = all_data.query(filters) all_code.append('chart_data = chart_data.query({})'.format(filters)) return all_data, all_code
def build_map_options(df, type='choropleth', loc=None, lat=None, lon=None, map_val=None): dtypes = get_dtypes(df) cols = sorted(dtypes.keys()) float_cols, str_cols, num_cols = [], [], [] for c in cols: dtype = dtypes[c] classification = classify_type(dtype) if classification == 'S': str_cols.append(c) continue if classification in ['F', 'I']: num_cols.append(c) if classification == 'F': float_cols.append(c) lat_options = [ build_option(c) for c in float_cols if c not in build_selections(lon, map_val) ] lon_options = [ build_option(c) for c in float_cols if c not in build_selections(lat, map_val) ] loc_options = [ build_option(c) for c in str_cols if c not in build_selections(map_val) ] if type == 'choropleth': val_options = [ build_option(c) for c in num_cols if c not in build_selections(loc) ] else: val_options = [ build_option(c) for c in num_cols if c not in build_selections(lon, lat) ] return loc_options, lat_options, lon_options, val_options
def build_dtypes_state(data): """ Helper function to build globally managed state pertaining to a D-Tale instances columns & data types :param data: dataframe to build data type information for :type data: :class:`pandas:pandas.DataFrame` :return: a list of dictionaries containing column names, indexes and data types """ dtypes = get_dtypes(data) mins = data.min().to_dict() maxs = data.max().to_dict() def _format_dtype(col_index, col): dtype = dtypes[col] dtype_data = dict(name=col, dtype=dtype, index=col_index) if classify_type(dtype) == 'F': # floats dtype_data['min'] = mins[col] dtype_data['max'] = maxs[col] return dtype_data return [_format_dtype(i, c) for i, c in enumerate(data.columns)]
def build_dtypes_state(data): """ Helper function to build globally managed state pertaining to a D-Tale instances columns & data types :param data: dataframe to build data type information for :type data: :class:`pandas:pandas.DataFrame` :return: a list of dictionaries containing column names, indexes and data types """ dtypes = get_dtypes(data) ranges = data.agg([min, max]).to_dict() def _format_dtype(col_index, col): dtype = dtypes[col] dtype_data = dict(name=col, dtype=dtype, index=col_index) if classify_type(dtype) == 'F' and not data[col].isnull().all(): # floats col_ranges = ranges[col] if not any((np.isnan(v) or np.isinf(v) for v in col_ranges.values())): dtype_data = dict_merge(ranges[col], dtype_data) return dtype_data return [_format_dtype(i, c) for i, c in enumerate(data.columns)]
def build_group_inputs_filter(df, group_inputs): dtypes = get_dtypes(df) def _group_filter(group_val): for gc, gv in group_val.items(): classifier = classify_type(dtypes[gc]) yield group_filter_handler(gc, gv, classifier) def _full_filter(): for group_val in group_inputs: filter_vals, label_vals = [], [] for fv, lv in _group_filter(group_val): filter_vals.append(fv) label_vals.append(lv) yield " and ".join(filter_vals), ", ".join(label_vals) full_filters, full_labels = [], [] for ff, fl in _full_filter(): full_filters.append(ff) full_labels.append(fl) return ("({})".format(") or (".join(full_filters)), ", ".join(full_labels))
def build_x_dropdown(is_open, pathname, inputs, chart_inputs, yaxis_data, map_data): if not is_open: raise PreventUpdate df = global_state.get_data(get_data_id(pathname)) all_inputs = combine_inputs( dash_app, inputs, chart_inputs, yaxis_data, map_data ) chart_type, x, y, z, group, map_val, animate_by = ( all_inputs.get(p) for p in ["chart_type", "x", "y", "z", "group", "map_val", "animate_by"] ) if chart_type == "maps": if all_inputs.get("map_type") == "choropleth": x = all_inputs["loc"] else: x = "lat_lon" x_options = build_selections(map_val, animate_by) else: x_options = build_selections(z or y, animate_by) col_opts = list(build_cols(df.columns, get_dtypes(df))) x_options = [build_option(c, l) for c, l in col_opts if c not in x_options] return x_options, x
def build_base_chart(raw_data, x, y, group_col=None, group_val=None, agg=None, allow_duplicates=False, return_raw=False, unlimited_data=False, animate_by=None, **kwargs): """ Helper function to return data for 'chart-data' & 'correlations-ts' endpoints. Will return a dictionary of dictionaries (one for each series) which contain the data for the x & y axes of the chart as well as the minimum & maximum of all the series for the y-axis. If there is only one series (no group_col specified) the only key in the dictionary of series data will be 'all' otherwise the keys will be the values of the groups. :param raw_data: dataframe to be used for chart :type raw_data: :class:`pandas:pandas.DataFrame` :param x: column to be used as x-axis of chart :type x: str :param y: column to be used as y-axis of chart :type y: list of strings :param group: comma-separated string of columns to group chart data by :type group: str, optional :param agg: points to a specific function that can be applied to :func: pandas.core.groupby.DataFrameGroupBy. Possible values are: count, first, last mean, median, min, max, std, var, mad, prod, sum :type agg: str, optional :param allow_duplicates: flag to allow duplicates to be ignored (usually for scatter plots) :type allow_duplicates: bool, optional :return: dict """ group_fmt_overrides = { "I": lambda v, as_string: json_int(v, as_string=as_string, fmt="{}") } data, code = retrieve_chart_data(raw_data, x, y, kwargs.get("z"), group_col, animate_by, group_val=group_val) x_col = str("x") if x is None: x = x_col data.loc[:, x_col] = range(1, len(data) + 1) # sequential integers: 1, 2, ..., N y_cols = make_list(y) z_col = kwargs.get("z") z_cols = make_list(z_col) sort_cols = y_cols if len(z_cols) else [] if group_col is not None and len(group_col): main_group = group_col if animate_by is not None: main_group = [animate_by] + main_group sort_cols = main_group + [x] + sort_cols data = data.sort_values(sort_cols) code.append("chart_data = chart_data.sort_values(['{cols}'])".format( cols="', '".join(sort_cols))) check_all_nan(data) data = data.rename(columns={x: x_col}) code.append("chart_data = chart_data.rename(columns={'" + x + "': '" + x_col + "'})") if agg is not None: data, agg_code = build_agg_data( data, x_col, y_cols, kwargs, agg, z=z_col, group_col=group_col, animate_by=animate_by, ) code += agg_code MAX_GROUPS = 30 group_vals = data[group_col].drop_duplicates() if len(group_vals) > MAX_GROUPS: dtypes = get_dtypes(group_vals) group_fmts = { c: find_dtype_formatter(dtypes[c], overrides=group_fmt_overrides) for c in group_col } group_f, _ = build_formatters(group_vals) group_vals = group_f.format_lists(group_vals) group_vals = pd.DataFrame(group_vals, columns=group_col) msg = ( "Group ({}) contains more than {} unique values, more groups than that will make the chart unreadable. " 'You can choose specific groups to display from then "Group(s)" dropdown above. The available group(s) ' "are listed below:\n\n{}").format( ", ".join(group_col), MAX_GROUPS, group_vals.to_string(index=False)) raise ChartBuildingError(msg, group_vals.to_string(index=False)) data = data.dropna() if return_raw: return data.rename(columns={x_col: x}) code.append("chart_data = chart_data.dropna()") data_f, range_f = build_formatters(data) ret_data = dict( data={}, min={ col: fmt(data[col].min(), None) for _, col, fmt in range_f.fmts if col in [x_col] + y_cols + z_cols }, max={ col: fmt(data[col].max(), None) for _, col, fmt in range_f.fmts if col in [x_col] + y_cols + z_cols }, ) dtypes = get_dtypes(data) group_fmts = { c: find_dtype_formatter(dtypes[c], overrides=group_fmt_overrides) for c in group_col } def _load_groups(df): for group_val, grp in df.groupby(group_col): def _group_filter(): for gv, gc in zip(make_list(group_val), group_col): classifier = classify_type(dtypes[gc]) yield group_filter_handler( gc, group_fmts[gc](gv, as_string=True), classifier) group_filter = " and ".join(list(_group_filter())) yield group_filter, data_f.format_lists(grp) if animate_by is not None: frame_fmt = find_dtype_formatter(dtypes[animate_by], overrides=group_fmt_overrides) ret_data["frames"] = [] for frame_key, frame in data.sort_values(animate_by).groupby( animate_by): ret_data["frames"].append( dict( data=dict(_load_groups(frame)), name=frame_fmt(frame_key, as_string=True), )) ret_data["data"] = copy.deepcopy(ret_data["frames"][-1]["data"]) else: ret_data["data"] = dict(_load_groups(data)) return ret_data, code main_group = [x] if animate_by is not None: main_group = [animate_by] + main_group sort_cols = main_group + sort_cols data = data.sort_values(sort_cols) code.append("chart_data = chart_data.sort_values(['{cols}'])".format( cols="', '".join(sort_cols))) check_all_nan(data) y_cols = [str(y_col) for y_col in y_cols] data = data[main_group + y_cols + z_cols] data = data.rename(columns={x: x_col}) main_group = [x_col if c == x else c for c in main_group] code.append("chart_data = chart_data.rename(columns={'" + x + "': '" + x_col + "'})") if agg is not None: data, agg_code = build_agg_data(data, x_col, y_cols, kwargs, agg, z=z_col, animate_by=animate_by) code += agg_code data = data.dropna() if return_raw: return data.rename(columns={x_col: x}) code.append("chart_data = chart_data.dropna()") dupe_cols = main_group + (y_cols if len(z_cols) else []) check_exceptions( data[dupe_cols].rename(columns={x_col: x}), allow_duplicates or agg in ["raw", "drop_duplicates"], unlimited_data=unlimited_data, data_limit=40000 if len(z_cols) or animate_by is not None else 15000, ) data_f, range_f = build_formatters(data) ret_data = dict( min={ col: fmt(data[col].min(), None) for _, col, fmt in range_f.fmts if col in [x_col] + y_cols + z_cols }, max={ col: fmt(data[col].max(), None) for _, col, fmt in range_f.fmts if col in [x_col] + y_cols + z_cols }, ) if animate_by is not None: frame_fmt = find_dtype_formatter(find_dtype(data[animate_by]), overrides=group_fmt_overrides) ret_data["frames"] = [] for frame_key, frame in data.sort_values(animate_by).groupby( animate_by): ret_data["frames"].append( dict( data={str("all"): data_f.format_lists(frame)}, name=frame_fmt(frame_key, as_string=True), )) ret_data["data"] = copy.deepcopy(ret_data["frames"][-1]["data"]) else: ret_data["data"] = {str("all"): data_f.format_lists(data)} return ret_data, code
def build_base_chart(raw_data, x, y, group_col=None, group_val=None, agg=None, allow_duplicates=False, return_raw=False, unlimited_data=False, **kwargs): """ Helper function to return data for 'chart-data' & 'correlations-ts' endpoints. Will return a dictionary of dictionaries (one for each series) which contain the data for the x & y axes of the chart as well as the minimum & maximum of all the series for the y-axis. If there is only one series (no group_col specified) the only key in the dictionary of series data will be 'all' otherwise the keys will be the values of the groups. :param raw_data: dataframe to be used for chart :type raw_data: :class:`pandas:pandas.DataFrame` :param x: column to be used as x-axis of chart :type x: str :param y: column to be used as y-axis of chart :type y: list of strings :param group: comma-separated string of columns to group chart data by :type group: str, optional :param agg: points to a specific function that can be applied to :func: pandas.core.groupby.DataFrameGroupBy. Possible values are: count, first, last mean, median, min, max, std, var, mad, prod, sum :type agg: str, optional :param allow_duplicates: flag to allow duplicates to be ignored (usually for scatter plots) :type allow_duplicates: bool, optional :return: dict """ data, code = retrieve_chart_data(raw_data, x, y, kwargs.get('z'), group_col, group_val=group_val) x_col = str('x') y_cols = make_list(y) z_col = kwargs.get('z') z_cols = make_list(z_col) if group_col is not None and len(group_col): data = data.sort_values(group_col + [x]) code.append("chart_data = chart_data.sort_values(['{cols}'])".format( cols="', '".join(group_col + [x]))) check_all_nan(data, [x] + y_cols) data = data.rename(columns={x: x_col}) code.append("chart_data = chart_data.rename(columns={'" + x + "': '" + x_col + "'})") if agg is not None and agg != 'raw': data = data.groupby(group_col + [x_col]) data = getattr(data, agg)().reset_index() code.append( "chart_data = chart_data.groupby(['{cols}']).{agg}().reset_index()" .format(cols="', '".join(group_col + [x]), agg=agg)) MAX_GROUPS = 30 group_vals = data[group_col].drop_duplicates() if len(group_vals) > MAX_GROUPS: dtypes = get_dtypes(group_vals) group_fmt_overrides = { 'I': lambda v, as_string: json_int(v, as_string=as_string, fmt='{}') } group_fmts = { c: find_dtype_formatter(dtypes[c], overrides=group_fmt_overrides) for c in group_col } group_f, _ = build_formatters(group_vals) group_vals = group_f.format_lists(group_vals) group_vals = pd.DataFrame(group_vals, columns=group_col) msg = ( 'Group ({}) contains more than {} unique values, more groups than that will make the chart unreadable. ' 'You can choose specific groups to display from then "Group(s)" dropdown above. The available group(s) ' 'are listed below:').format(', '.join(group_col), MAX_GROUPS, group_vals.to_string(index=False)) raise ChartBuildingError(msg, group_vals.to_string(index=False)) data = data.dropna() if return_raw: return data.rename(columns={x_col: x}) code.append("chart_data = chart_data.dropna()") data_f, range_f = build_formatters(data) ret_data = dict( data={}, min={ col: fmt(data[col].min(), None) for _, col, fmt in range_f.fmts if col in [x_col] + y_cols }, max={ col: fmt(data[col].max(), None) for _, col, fmt in range_f.fmts if col in [x_col] + y_cols }, ) dtypes = get_dtypes(data) group_fmt_overrides = { 'I': lambda v, as_string: json_int(v, as_string=as_string, fmt='{}') } group_fmts = { c: find_dtype_formatter(dtypes[c], overrides=group_fmt_overrides) for c in group_col } for group_val, grp in data.groupby(group_col): def _group_filter(): for gv, gc in zip(make_list(group_val), group_col): classifier = classify_type(dtypes[gc]) yield group_filter_handler( gc, group_fmts[gc](gv, as_string=True), classifier) group_filter = ' and '.join(list(_group_filter())) ret_data['data'][group_filter] = data_f.format_lists(grp) return ret_data, code sort_cols = [x] + (y_cols if len(z_cols) else []) data = data.sort_values(sort_cols) code.append("chart_data = chart_data.sort_values(['{cols}'])".format( cols="', '".join(sort_cols))) check_all_nan(data, [x] + y_cols + z_cols) y_cols = [str(y_col) for y_col in y_cols] data.columns = [x_col] + y_cols + z_cols code.append("chart_data.columns = ['{cols}']".format( cols="', '".join([x_col] + y_cols + z_cols))) if agg is not None: data, agg_code = build_agg_data(data, x_col, y_cols, kwargs, agg, z=z_col) code += agg_code data = data.dropna() if return_raw: return data.rename(columns={x_col: x}) code.append("chart_data = chart_data.dropna()") dupe_cols = [x_col] + (y_cols if len(z_cols) else []) check_exceptions(data[dupe_cols].rename(columns={'x': x}), allow_duplicates or agg == 'raw', unlimited_data=unlimited_data, data_limit=40000 if len(z_cols) else 15000) data_f, range_f = build_formatters(data) ret_data = dict(data={str('all'): data_f.format_lists(data)}, min={ col: fmt(data[col].min(), None) for _, col, fmt in range_f.fmts if col in [x_col] + y_cols + z_cols }, max={ col: fmt(data[col].max(), None) for _, col, fmt in range_f.fmts if col in [x_col] + y_cols + z_cols }) return ret_data, code
def heatmap_builder(data_id, **inputs): """ Builder function for :plotly:`plotly.graph_objects.Heatmap <plotly.graph_objects.Heatmap>` :param data_id: integer string identifier for a D-Tale process's data :type data_id: str :param inputs: Optional keyword arguments containing the following information: - x: column to be used as x-axis of chart - y: column to be used as y-axis of chart - z: column to use for the Z-Axis - agg: points to a specific function that can be applied to :func: pandas.core.groupby.DataFrameGroupBy :type inputs: dict :return: heatmap :rtype: :plotly:`plotly.graph_objects.Heatmap <plotly.graph_objects.Heatmap>` """ try: if not valid_chart(**inputs): return None raw_data = global_state.get_data(data_id) wrapper = chart_wrapper(data_id, raw_data, inputs) hm_kwargs = dict(hoverongaps=False, colorscale='Greens', showscale=True, hoverinfo='x+y+z') x, y, z, agg = (inputs.get(p) for p in ['x', 'y', 'z', 'agg']) y = y[0] data = retrieve_chart_data(raw_data, x, y, z) x_title = update_label_for_freq(x) y_title = update_label_for_freq(y) z_title = z data = data.sort_values([x, y]) check_all_nan(data) dupe_cols = [x, y] if agg is not None: z_title = '{} ({})'.format(z_title, AGGS[agg]) if agg == 'corr': data = data.dropna() data = data.set_index([x, y]).unstack().corr() data = data.stack().reset_index(0, drop=True) y_title = x_title dupe_cols = [ '{}{}'.format(col, i) for i, col in enumerate(data.index.names) ] [x, y] = dupe_cols data.index.names = dupe_cols data = data.reset_index() data.loc[data[x] == data[y], z] = np.nan hm_kwargs = dict_merge( hm_kwargs, dict(colorscale=[[0, 'red'], [0.5, 'yellow'], [1.0, 'green']], zmin=-1, zmax=1)) else: data = build_agg_data(data, x, y, inputs, agg, z=z) if not len(data): raise Exception('No data returned for this computation!') check_exceptions( data[dupe_cols], agg != 'corr', data_limit=40000, limit_msg= 'Heatmap exceeds {} cells, cannot render. Please apply filter...') dtypes = { c: classify_type(dtype) for c, dtype in get_dtypes(data).items() } data_f, _ = chart_formatters(data) data = data_f.format_df(data) data = data.sort_values([x, y]) data = data.set_index([x, y]) data = data.unstack(0)[z] x_data = weekday_tick_handler(data.columns, x) y_data = weekday_tick_handler(data.index.values, y) heat_data = data.values x_axis = dict_merge({ 'title': x_title, 'tickangle': -20 }, build_spaced_ticks(x_data)) if dtypes.get(x) == 'I': x_axis['tickformat'] = '.0f' y_axis = dict_merge({ 'title': y_title, 'tickangle': -20 }, build_spaced_ticks(y_data)) if dtypes.get(y) == 'I': y_axis['tickformat'] = '.0f' hovertemplate = ''.join([ x_title, ': %{customdata[0]}<br>', y_title, ': %{customdata[1]}<br>', z_title, ': %{z}<extra></extra>' ]) hm_kwargs = dict_merge( hm_kwargs, dict(z=heat_data, colorbar={'title': z_title}, hoverinfo='x+y+z', hovertemplate=hovertemplate, customdata=[[[xd, yd] for xd in x_data] for yd in y_data])) return wrapper( dcc.Graph(id='heatmap-graph-{}'.format(y), style={ 'margin-right': 'auto', 'margin-left': 'auto', 'height': 600 }, figure=dict(data=[go.Heatmap(**hm_kwargs)], layout=build_layout( dict_merge( dict(xaxis=x_axis, yaxis=y_axis, xaxis_zeroline=False, yaxis_zeroline=False), build_title(x, y, z=z, agg=agg)))))) except BaseException as e: return build_error(str(e), str(traceback.format_exc()))
def build_axes(data_id, x, axis_inputs, mins, maxs, z=None, agg=None): """ Returns helper function for building axis configurations against a specific y-axis. :param data_id: identifier of data to build axis configurations against :type data_id: str :param x: column to be used as x-axis of chart :type x: str :param axis_inputs: current settings for y-axis limits :type axis_inputs: dict :param mins: minimums for all columns involved in chart :type mins: dict :param maxs: maximums for all columns invloved in chart :param maxs: dict :param z: column to use for the Z-Axis :type z: str, optional :param agg: specific aggregation that can be applied to y or z axes. Possible values are: count, first, last mean, median, min, max, std, var, mad, prod, sum. This is included in label of axis it is being applied to. :type agg: str, optional :return: handler function to be applied against each y-axis used in chart :rtype: func """ data = global_state.get_data(data_id) dtypes = get_dtypes(data) def _build_axes(y): axes = {'xaxis': dict(title=update_label_for_freq(x))} positions = [] for i, y2 in enumerate(y, 0): right = i % 2 == 1 axis_ct = int(i / 2) title = update_label_for_freq(y2) if z is None and agg is not None: title = '{} ({})'.format(title, AGGS[agg]) value = dict(title=title) if i == 0: key = 'yaxis' else: key = 'yaxis{}'.format(i + 1) value = dict_merge( value, dict(overlaying='y', side='right' if right else 'left')) value['anchor'] = 'free' if axis_ct > 0 else 'x' if axis_ct > 0: pos = axis_ct / 20.0 value['position'] = (1 - pos) if right else pos positions.append(value['position']) if y2 in axis_inputs and not (axis_inputs[y2]['min'], axis_inputs[y2]['max']) == ( mins[y2], maxs[y2]): value['range'] = [ axis_inputs[y2]['min'], axis_inputs[y2]['max'] ] if classify_type(dtypes.get(y2)) == 'I': value['tickformat'] = '.0f' axes[key] = value if len(positions): if len(positions) == 1: domain = [positions[0] + 0.05, 1] elif len(positions) == 2: domain = sorted(positions) domain = [domain[0] + 0.05, domain[1] - 0.05] else: lower, upper = divide_chunks(sorted(positions), 2) domain = [lower[-1] + 0.05, upper[0] - 0.05] axes['xaxis']['domain'] = domain if classify_type(dtypes.get(x)) == 'I': axes['xaxis']['tickformat'] = '.0f' if z is not None: axes['zaxis'] = dict( title=z if agg is None else '{} ({})'.format(z, AGGS[agg])) if classify_type(dtypes.get(z)) == 'I': axes['zaxis']['tickformat'] = '.0f' return axes return _build_axes
def build_chart(data, x, y, group_col=None, agg=None, allow_duplicates=False, **kwargs): """ Helper function to return data for 'chart-data' & 'correlations-ts' endpoints. Will return a dictionary of dictionaries (one for each series) which contain the data for the x & y axes of the chart as well as the minimum & maximum of all the series for the y-axis. If there is only one series (no group_col specified) the only key in the dictionary of series data will be 'all' otherwise the keys will be the values of the groups. :param data: dataframe to be used for chart :type data: :class:`pandas:pandas.DataFrame` :param x: column to be used as x-axis of chart :type x: str :param y: column to be used as y-axis of chart :type y: list of strings :param group: comma-separated string of columns to group chart data by :type group: str, optional :param aggregation: points to a specific function that can be applied to :func: pandas.core.groupby.DataFrameGroupBy. Possible values are: count, first, last mean, median, min, max, std, var, mad, prod, sum :type aggregation: str, optional :return: dict """ def build_formatters(df): cols = grid_columns(df) data_f = grid_formatter(cols, nan_display=None) overrides = {'F': lambda f, i, c: f.add_float(i, c, precision=2)} range_f = grid_formatter(cols, overrides=overrides, nan_display=None) return data_f, range_f def check_all_nan(df, cols): for col in cols: if df[col].isnull().all(): raise Exception('All data for column "{}" is NaN!'.format(col)) x_col = str('x') y_cols = make_list(y) if group_col is not None: data = data[group_col + [x] + y_cols].sort_values(group_col + [x]) check_all_nan(data, [x] + y_cols) y_cols = [str(y_col) for y_col in y_cols] data.columns = group_col + [x_col] + y_cols if agg is not None: data = data.groupby(group_col + [x_col]) data = getattr(data, agg)().reset_index() max_groups = 15 if len(data[group_col].drop_duplicates()) > max_groups: msg = ( 'Group ({}) contains more than {} unique values, please add additional filter' ' or else chart will be unreadable' ).format(', '.join(group_col), max_groups) raise Exception(msg) data_f, range_f = build_formatters(data[[x_col] + y_cols]) ret_data = dict( data={}, min={col: fmt(data[col].min(), None) for _, col, fmt in range_f.fmts if col in [x_col] + y_cols}, max={col: fmt(data[col].max(), None) for _, col, fmt in range_f.fmts if col in [x_col] + y_cols}, ) dtypes = get_dtypes(data) group_fmts = {c: find_dtype_formatter(dtypes[c]) for c in group_col} for group_val, grp in data.groupby(group_col): group_val = '/'.join([ group_fmts[gc](gv, as_string=True) for gv, gc in zip(make_list(group_val), group_col) ]) ret_data['data'][group_val] = data_f.format_lists(grp) return ret_data data = data[[x] + y_cols].sort_values(x) check_all_nan(data, [x] + y_cols) y_cols = [str(y_col) for y_col in y_cols] data.columns = [x_col] + y_cols if agg is not None: if agg == 'rolling': window, comp = map(kwargs.get, ['rolling_win', 'rolling_comp']) data = data.set_index(x_col).rolling(window=window) data = pd.DataFrame({c: getattr(data[c], comp)() for c in y_cols}) data = data.reset_index() else: data = data.groupby(x_col) data = getattr(data[y_cols], agg)().reset_index() if not allow_duplicates and any(data[x_col].duplicated()): raise Exception('{} contains duplicates, please specify group or additional filtering'.format(x)) if len(data) > 15000: raise Exception('Dataset exceeds 15,000 records, cannot render. Please apply filter...') data_f, range_f = build_formatters(data) ret_data = dict( data={str('all'): data_f.format_lists(data)}, min={col: fmt(data[col].min(), None) for _, col, fmt in range_f.fmts if col in [x_col] + y_cols}, max={col: fmt(data[col].max(), None) for _, col, fmt in range_f.fmts if col in [x_col] + y_cols}, ) return ret_data
def build_base_chart(raw_data, x, y, group_col=None, group_type=None, group_val=None, bins_val=None, bin_type=None, agg=None, extended_aggregation=[], allow_duplicates=False, return_raw=False, unlimited_data=False, animate_by=None, cleaners=[], **kwargs): """ Helper function to return data for 'chart-data' & 'correlations-ts' endpoints. Will return a dictionary of dictionaries (one for each series) which contain the data for the x & y axes of the chart as well as the minimum & maximum of all the series for the y-axis. If there is only one series (no group_col specified) the only key in the dictionary of series data will be 'all' otherwise the keys will be the values of the groups. :param raw_data: dataframe to be used for chart :type raw_data: :class:`pandas:pandas.DataFrame` :param x: column to be used as x-axis of chart :type x: str :param y: column to be used as y-axis of chart :type y: list of strings :param group: comma-separated string of columns to group chart data by :type group: str, optional :param agg: points to a specific function that can be applied to :func: pandas.core.groupby.DataFrameGroupBy. Possible values are: count, first, last mean, median, min, max, std, var, mad, prod, sum :type agg: str, optional :param extended_aggregation: list of configurations that point to a specific function that can be applied to :func: pandas.core.groupby.DataFrameGroupBy. Possible values are: count, first, last mean, median, min, max, std, var, mad, prod, sum :type agg: list, optional :param allow_duplicates: flag to allow duplicates to be ignored (usually for scatter plots) :type allow_duplicates: bool, optional :return: dict """ group_fmt_overrides = { "I": lambda v, as_string: json_int(v, as_string=as_string, fmt="{}") } data, code = retrieve_chart_data(raw_data, x, y, kwargs.get("z"), group_col, animate_by, group_val=group_val) cleaners = cleaners or [] if len(cleaners): for col in data.columns: if classify_type(find_dtype(data[col])) == "S": code.append("s = chart_data['{}']".format(col)) cleaned_col, cleaned_code = handle_cleaners( data[col], ",".join(cleaners)) data.loc[:, col] = cleaned_col code += cleaned_code code.append("chart_data.loc[:, '{}'] = s".format(col)) x_col = str("x") if x is None: x = x_col data.loc[:, x_col] = range(1, len(data) + 1) # sequential integers: 1, 2, ..., N y_cols = make_list(y) z_col = kwargs.get("z") z_cols = make_list(z_col) y_cols = [str(col) for col in y_cols] is_z = len(z_cols) > 0 y_group_cols = y_cols if is_z else [] sort_cols = y_group_cols final_cols = y_cols + z_cols if group_col is not None and len(group_col): for col in make_list(group_col): classifier = classify_type(find_dtype(data[col])) if classifier == "F" or (classifier == "I" and group_type == "bins"): if bin_type == "width": data.loc[:, col] = pd.qcut(data[col], q=bins_val, duplicates="drop").astype("str") code.append(( "chart_data.loc[:, '{col}'] = pd.qcut(chart_data['{col}'], q={bins}, duplicates=\"drop\")" ).format(col=col, bins=bins_val)) else: bins_data = data[col].dropna() npt = len(bins_data) equal_freq_bins = np.interp( np.linspace(0, npt, bins_val + 1), np.arange(npt), np.sort(bins_data), ) data.loc[:, col] = pd.cut(data[col], bins=equal_freq_bins, duplicates="drop").astype("str") code.append(( "bins_data = data['{col}'].dropna()\n" "npt = len(bins_data)\n" "equal_freq_bins = np.interp(np.linspace(0, npt, {bins}), np.arange(npt), " "np.sort(bins_data))\n" "chart_data.loc[:, '{col}'] = pd.cut(chart_data['{col}'], bins=equal_freq_bins, " 'duplicates="drop")').format(col=col, bins=bins_val + 1)) main_group = group_col if animate_by is not None: main_group = [animate_by] + main_group sort_cols = main_group + [x] + sort_cols data = data.sort_values(sort_cols) code.append("chart_data = chart_data.sort_values(['{cols}'])".format( cols="', '".join(sort_cols))) check_all_nan(data) data = data.rename(columns={x: x_col}) code.append("chart_data = chart_data.rename(columns={'" + x + "': '" + x_col + "'})") if agg is not None or len(extended_aggregation): data, agg_code, final_cols = build_agg_data( data, x_col, y_cols, kwargs, agg, z=z_col, group_col=group_col, animate_by=animate_by, extended_aggregation=extended_aggregation, ) code += agg_code group_vals = data[group_col].drop_duplicates() if len(group_vals) > MAX_GROUPS: dtypes = get_dtypes(group_vals) group_fmts = { c: find_dtype_formatter(dtypes[c], overrides=group_fmt_overrides) for c in group_col } group_f, _ = build_formatters(group_vals) group_vals = group_f.format_lists(group_vals) group_vals = pd.DataFrame(group_vals, columns=group_col) msg = ( "Group ({}) contains more than {} unique values, more groups than that will make the chart unreadable. " 'You can choose specific groups to display from then "Group(s)" dropdown above. The available group(s) ' "are listed below:\n\n{}").format( ", ".join(group_col), MAX_GROUPS, group_vals.to_string(index=False)) raise ChartBuildingError(msg, group_vals.to_string(index=False)) data = data.dropna() if return_raw: return data.rename(columns={x_col: x}) code.append("chart_data = chart_data.dropna()") data_f, range_f = build_formatters(data) ret_data = dict( data={}, min={ col: fmt(data[col].min(), None) for _, col, fmt in range_f.fmts if col in [x_col] + final_cols }, max={ col: fmt(data[col].max(), None) for _, col, fmt in range_f.fmts if col in [x_col] + final_cols }, ) dtypes = get_dtypes(data) group_fmts = { c: find_dtype_formatter(dtypes[c], overrides=group_fmt_overrides) for c in group_col } def _load_groups(df): for group_val, grp in df.groupby(group_col): def _group_filter(): for gv, gc in zip(make_list(group_val), group_col): classifier = classify_type(dtypes[gc]) yield group_filter_handler( gc, group_fmts[gc](gv, as_string=True), classifier) final_group_filter, final_group_label = [], [] for gf, gl in _group_filter(): final_group_filter.append(gf) final_group_label.append(gl) group_filter = " and ".join(final_group_filter) group_label = "({})".format(", ".join(final_group_label)) data = data_f.format_lists(grp) data["_filter_"] = group_filter yield group_label, data if animate_by is not None: frame_fmt = find_dtype_formatter(dtypes[animate_by], overrides=group_fmt_overrides) ret_data["frames"] = [] for frame_key, frame in data.sort_values(animate_by).groupby( animate_by): ret_data["frames"].append( dict( data=dict(_load_groups(frame)), name=frame_fmt(frame_key, as_string=True), )) ret_data["data"] = copy.deepcopy(ret_data["frames"][-1]["data"]) else: ret_data["data"] = dict(_load_groups(data)) return ret_data, code main_group = [x] if animate_by is not None: main_group = [animate_by] + main_group sort_cols = main_group + sort_cols data = data.sort_values(sort_cols) code.append("chart_data = chart_data.sort_values(['{cols}'])".format( cols="', '".join(sort_cols))) check_all_nan(data) data = data[main_group + final_cols] data = data.rename(columns={x: x_col}) main_group = [x_col if c == x else c for c in main_group] code.append("chart_data = chart_data.rename(columns={'" + x + "': '" + x_col + "'})") # convert booleans into integers for aggregation for col in z_cols or y_cols: classifier = classify_type(find_dtype(data[col])) if classifier == "B": data.loc[:, col] = data[col].astype("int") if agg is not None or len(extended_aggregation): data, agg_code, final_cols = build_agg_data( data, x_col, y_cols, kwargs, agg, z=z_col, animate_by=animate_by, extended_aggregation=extended_aggregation, ) code += agg_code data = data.dropna() if return_raw: return data.rename(columns={x_col: x}) code.append("chart_data = chart_data.dropna()") dupe_cols = main_group + y_group_cols data_limit = global_state.get_chart_settings( )["3d_points" if is_z or animate_by is not None else "scatter_points"] check_exceptions( data[dupe_cols].rename(columns={x_col: x}), allow_duplicates or agg in ["raw", "drop_duplicates"], unlimited_data=unlimited_data, data_limit=data_limit, ) data_f, range_f = build_formatters(data) ret_data = dict( min={ col: fmt(data[col].min(), None) for _, col, fmt in range_f.fmts if col in [x_col] + y_group_cols + final_cols }, max={ col: fmt(data[col].max(), None) for _, col, fmt in range_f.fmts if col in [x_col] + y_group_cols + final_cols }, ) if animate_by is not None: frame_fmt = find_dtype_formatter(find_dtype(data[animate_by]), overrides=group_fmt_overrides) ret_data["frames"] = [] for frame_key, frame in data.sort_values(animate_by).groupby( animate_by): ret_data["frames"].append( dict( data={str("all"): data_f.format_lists(frame)}, name=frame_fmt(frame_key, as_string=True), )) ret_data["data"] = copy.deepcopy(ret_data["frames"][-1]["data"]) else: ret_data["data"] = {str("all"): data_f.format_lists(data)} return ret_data, code
def build_chart(raw_data, x, y, group_col=None, agg=None, allow_duplicates=False, **kwargs): """ Helper function to return data for 'chart-data' & 'correlations-ts' endpoints. Will return a dictionary of dictionaries (one for each series) which contain the data for the x & y axes of the chart as well as the minimum & maximum of all the series for the y-axis. If there is only one series (no group_col specified) the only key in the dictionary of series data will be 'all' otherwise the keys will be the values of the groups. :param raw_data: dataframe to be used for chart :type raw_data: :class:`pandas:pandas.DataFrame` :param x: column to be used as x-axis of chart :type x: str :param y: column to be used as y-axis of chart :type y: list of strings :param group: comma-separated string of columns to group chart data by :type group: str, optional :param agg: points to a specific function that can be applied to :func: pandas.core.groupby.DataFrameGroupBy. Possible values are: count, first, last mean, median, min, max, std, var, mad, prod, sum :type agg: str, optional :param allow_duplicates: flag to allow duplicates to be ignored (usually for scatter plots) :type allow_duplicates: bool, optional :return: dict """ data, code = retrieve_chart_data(raw_data, x, y, kwargs.get('z'), group_col) x_col = str('x') y_cols = make_list(y) z_col = kwargs.get('z') z_cols = [] if z_col is not None: z_cols = [z_col] if group_col is not None and len(group_col): data = data.sort_values(group_col + [x]) code.append("chart_data = chart_data.sort_values(['{cols}'])".format(cols="', '".join(group_col + [x]))) check_all_nan(data, [x] + y_cols) data = data.rename(columns={x: x_col}) code.append("chart_data = chart_data.rename(columns={'" + x + "': '" + x_col + "'})") if agg is not None: data = data.groupby(group_col + [x_col]) data = getattr(data, agg)().reset_index() code.append("chart_data = chart_data.groupby(['{cols}']).{agg}().reset_index()".format( cols="', '".join(group_col + [x]), agg=agg )) max_groups = 15 if len(data[group_col].drop_duplicates()) > max_groups: msg = ( 'Group ({}) contains more than {} unique values, please add additional filter' ' or else chart will be unreadable' ).format(', '.join(group_col), max_groups) raise Exception(msg) data = data.dropna() code.append("chart_data = chart_data.dropna()") data_f, range_f = build_formatters(data) ret_data = dict( data={}, min={col: fmt(data[col].min(), None) for _, col, fmt in range_f.fmts if col in [x_col] + y_cols}, max={col: fmt(data[col].max(), None) for _, col, fmt in range_f.fmts if col in [x_col] + y_cols}, ) dtypes = get_dtypes(data) group_fmt_overrides = {'I': lambda v, as_string: json_int(v, as_string=as_string, fmt='{}')} group_fmts = {c: find_dtype_formatter(dtypes[c], overrides=group_fmt_overrides) for c in group_col} for group_val, grp in data.groupby(group_col): group_val = '/'.join([ group_fmts[gc](gv, as_string=True) for gv, gc in zip(make_list(group_val), group_col) ]) ret_data['data'][group_val] = data_f.format_lists(grp) ret_data['dtypes'] = {c: classify_type(dtype) for c, dtype in dtypes.items()} return ret_data, code sort_cols = [x] + (y_cols if len(z_cols) else []) data = data.sort_values(sort_cols) code.append("chart_data = chart_data.sort_values(['{cols}'])".format(cols="', '".join(sort_cols))) check_all_nan(data, [x] + y_cols + z_cols) y_cols = [str(y_col) for y_col in y_cols] data.columns = [x_col] + y_cols + z_cols code.append("chart_data.columns = ['{cols}']".format(cols="', '".join([x_col] + y_cols + z_cols))) if agg is not None: data, agg_code = build_agg_data(data, x_col, y_cols, kwargs, agg, z=z_col) code += agg_code data = data.dropna() code.append("chart_data = chart_data.dropna()") dupe_cols = [x_col] + (y_cols if len(z_cols) else []) check_exceptions(data[dupe_cols].rename(columns={'x': x}), allow_duplicates, data_limit=40000 if len(z_cols) else 15000) data_f, range_f = build_formatters(data) ret_data = dict( data={str('all'): data_f.format_lists(data)}, min={col: fmt(data[col].min(), None) for _, col, fmt in range_f.fmts if col in [x_col] + y_cols + z_cols}, max={col: fmt(data[col].max(), None) for _, col, fmt in range_f.fmts if col in [x_col] + y_cols + z_cols} ) return ret_data, code