def test_getters(builtin_pkg):
req = build_req_tuple(
{'int': '1', 'empty_int': '', 'str': 'hello', 'empty_str': '', 'bool': 'true', 'float': '1.1'}
)
val = utils.get_str_arg(req, 'str')
assert isinstance(val, str) and val == 'hello'
val = utils.get_str_arg(req, 'str_def', default='def')
assert val == 'def'
val = utils.get_str_arg(req, 'empty_str')
assert val is None
with mock.patch('{}.str'.format(builtin_pkg), mock.Mock(side_effect=Exception)):
val = utils.get_str_arg(req, 'str', default='def')
assert val == 'def'
val = utils.get_int_arg(req, 'int')
assert isinstance(val, int) and val == 1
val = utils.get_int_arg(req, 'int_def', default=2)
assert val == 2
val = utils.get_int_arg(req, 'empty_int')
assert val is None
with mock.patch('{}.int'.format(builtin_pkg), mock.Mock(side_effect=Exception)):
val = utils.get_int_arg(req, 'int', default=2)
assert val == 2
val = utils.get_bool_arg(req, 'bool')
assert isinstance(val, bool) and val
val = utils.get_float_arg(req, 'float')
assert isinstance(val, float) and val == 1.1
val = utils.get_float_arg(req, 'int_def', default=2.0)
assert val == 2.0
val = utils.get_float_arg(req, 'empty_float')
assert val is None
with mock.patch('{}.float'.format(builtin_pkg), mock.Mock(side_effect=Exception)):
val = utils.get_float_arg(req, 'float', default=2.0)
assert val == 2
def get_chart_data(data_id):
"""
:class:`flask:flask.Flask` route which builds data associated with a chart.js chart
:param data_id: integer string identifier for a D-Tale process's data
:type data_id: str
:param query: string from flask.request.args['query'] which is applied to DATA using the query() function
:param x: string from flask.request.args['x'] column to be used as x-axis of chart
:param y: string from flask.request.args['y'] column to be used as y-axis of chart
:param group: string from flask.request.args['group'] comma-separated string of columns to group chart data by
:param agg: string from flask.request.args['agg'] points to a specific function that can be applied to
:func: pandas.core.groupby.DataFrameGroupBy. Possible values are: count, first, last mean,
median, min, max, std, var, mad, prod, sum
:returns: JSON {
data: {
series1: { x: [x1, x2, ..., xN], y: [y1, y2, ..., yN] },
series2: { x: [x1, x2, ..., xN], y: [y1, y2, ..., yN] },
...,
seriesN: { x: [x1, x2, ..., xN], y: [y1, y2, ..., yN] },
},
min: minY,
max: maxY,
} or {error: 'Exception message', traceback: 'Exception stacktrace'}
"""
try:
query = get_str_arg(request, 'query')
data = DATA[data_id]
if query:
try:
data = data.query(query)
except BaseException as e:
return jsonify(dict(error='Invalid query: {}'.format(str(e))))
if not len(data):
return jsonify(
dict(error='query "{}" found no data, please alter'.format(
query)))
x = get_str_arg(request, 'x')
y = get_json_arg(request, 'y')
group_col = get_json_arg(request, 'group')
agg = get_str_arg(request, 'agg')
allow_duplicates = get_bool_arg(request, 'allowDupes')
window = get_int_arg(request, 'rollingWin')
comp = get_str_arg(request, 'rollingComp')
data = build_chart(data,
x,
y,
group_col,
agg,
allow_duplicates,
rolling_win=window,
rolling_comp=comp)
data['success'] = True
return jsonify(data)
except BaseException as e:
return jsonify(
dict(error=str(e), traceback=str(traceback.format_exc())))
def load_filterable_data(data_id, req, query=None):
filtered = get_bool_arg(req, "filtered")
curr_settings = global_state.get_settings(data_id) or {}
if filtered:
final_query = query or build_query(data_id, curr_settings.get("query"))
return run_query(
handle_predefined(data_id),
final_query,
global_state.get_context_variables(data_id),
ignore_empty=True,
)
return global_state.get_data(data_id)
def test_getters(builtin_pkg):
req = build_req_tuple({
"int": "1",
"empty_int": "",
"str": "hello",
"empty_str": "",
"bool": "true",
"float": "1.1",
})
val = utils.get_str_arg(req, "str")
assert isinstance(val, str) and val == "hello"
val = utils.get_str_arg(req, "str_def", default="def")
assert val == "def"
val = utils.get_str_arg(req, "empty_str")
assert val is None
with mock.patch("{}.str".format(builtin_pkg),
mock.Mock(side_effect=Exception)):
val = utils.get_str_arg(req, "str", default="def")
assert val == "def"
val = utils.get_int_arg(req, "int")
assert isinstance(val, int) and val == 1
val = utils.get_int_arg(req, "int_def", default=2)
assert val == 2
val = utils.get_int_arg(req, "empty_int")
assert val is None
with mock.patch("{}.int".format(builtin_pkg),
mock.Mock(side_effect=Exception)):
val = utils.get_int_arg(req, "int", default=2)
assert val == 2
val = utils.get_bool_arg(req, "bool")
assert isinstance(val, bool) and val
val = utils.get_float_arg(req, "float")
assert isinstance(val, float) and val == 1.1
val = utils.get_float_arg(req, "int_def", default=2.0)
assert val == 2.0
val = utils.get_float_arg(req, "empty_float")
assert val is None
with mock.patch("{}.float".format(builtin_pkg),
mock.Mock(side_effect=Exception)):
val = utils.get_float_arg(req, "float", default=2.0)
assert val == 2
def get_scatter(data_id):
"""
:class:`flask:flask.Flask` route which returns data used in correlation of two columns for scatter chart
:param data_id: integer string identifier for a D-Tale process's data
:type data_id: str
:param query: string from flask.request.args['query'] which is applied to DATA using the query() function
:param cols: comma-separated string from flask.request.args['cols'] containing names of two columns in dataframe
:param dateCol: string from flask.request.args['dateCol'] with name of date-type column in dateframe for timeseries
:param date: string from flask.request.args['date'] date value in dateCol to filter dataframe to
:returns: JSON {
data: [{col1: 0.123, col2: 0.123, index: 1},...,{col1: 0.123, col2: 0.123, index: N}],
stats: {
stats: {
correlated: 50,
only_in_s0: 1,
only_in_s1: 2,
pearson: 0.987,
spearman: 0.879,
}
x: col1,
y: col2
} or {error: 'Exception message', traceback: 'Exception stacktrace'}
"""
cols = get_json_arg(request, 'cols')
query = get_str_arg(request, 'query')
date = get_str_arg(request, 'date')
date_col = get_str_arg(request, 'dateCol')
rolling = get_bool_arg(request, 'rolling')
try:
data = DATA[data_id]
if query:
data = data.query(query)
idx_col = str('index')
y_cols = [cols[1], idx_col]
if rolling:
window = get_int_arg(request, 'window')
idx = min(data[data[date_col] == date].index) + 1
data = data.iloc[max(idx - window, 0):idx]
data = data[list(set(cols)) + [date_col]].dropna(how='any')
y_cols.append(date_col)
else:
data = data[data[date_col] == date] if date else data
data = data[list(set(cols))].dropna(how='any')
data[idx_col] = data.index
s0 = data[cols[0]]
s1 = data[cols[1]]
pearson = s0.corr(s1, method='pearson')
spearman = s0.corr(s1, method='spearman')
stats = dict(
pearson='N/A' if pd.isnull(pearson) else pearson,
spearman='N/A' if pd.isnull(spearman) else spearman,
correlated=len(data),
only_in_s0=len(data[data[cols[0]].isnull()]),
only_in_s1=len(data[data[cols[1]].isnull()])
)
if len(data) > 15000:
return jsonify(
stats=stats,
error='Dataset exceeds 15,000 records, cannot render scatter. Please apply filter...'
)
data = build_chart(data, cols[0], y_cols, allow_duplicates=True)
data['x'] = cols[0]
data['y'] = cols[1]
data['stats'] = stats
return jsonify(data)
except BaseException as e:
return jsonify(dict(error=str(e), traceback=str(traceback.format_exc())))
def __init__(self, req):
self.bins = get_int_arg(req, "bins", 20)
self.target = get_str_arg(req, "target")
self.density = get_bool_arg(req, "density")
