def test_load_flag(unittest, test_data):
initialize_store(test_data)
global_state.set_settings("1", dict(hide_shutdown=True))
unittest.assertEqual(global_state.load_flag("1", "hide_shutdown", False),
True)
global_state.set_settings("1", dict(hide_shutdown=False))
unittest.assertEqual(global_state.load_flag("1", "hide_shutdown", True),
False)
def test_transpose(custom_data, unittest):
from dtale.views import build_dtypes_state
global_state.clear_store()
with app.test_client() as c:
data = {c.port: custom_data}
dtypes = {c.port: build_dtypes_state(custom_data)}
settings = {c.port: {}}
build_data_inst(data)
build_dtypes(dtypes)
build_settings(settings)
reshape_cfg = dict(index=["security_id"], columns=["Col0"])
resp = c.get(
"/dtale/reshape/{}".format(c.port),
query_string=dict(output="new",
type="transpose",
cfg=json.dumps(reshape_cfg)),
)
response_data = json.loads(resp.data)
new_key = int(c.port) + 1
assert "error" in response_data
min_date = custom_data["date"].min().strftime("%Y-%m-%d")
global_state.set_settings(c.port,
dict(query="date == '{}'".format(min_date)))
reshape_cfg = dict(index=["date", "security_id"], columns=["Col0"])
resp = c.get(
"/dtale/reshape/{}".format(c.port),
query_string=dict(output="new",
type="transpose",
cfg=json.dumps(reshape_cfg)),
)
response_data = json.loads(resp.data)
assert response_data["data_id"] == new_key
assert len(global_state.keys()) == 2
unittest.assertEqual(
[d["name"] for d in global_state.get_dtypes(new_key)],
[
"index",
"{} 00:00:00 100000".format(min_date),
"{} 00:00:00 100001".format(min_date),
],
)
assert len(global_state.get_data(new_key)) == 1
assert global_state.get_settings(new_key).get(
"startup_code") is not None
c.get("/dtale/cleanup-datasets", query_string=dict(dataIds=new_key))
reshape_cfg = dict(index=["date", "security_id"])
resp = c.get(
"/dtale/reshape/{}".format(c.port),
query_string=dict(output="override",
type="transpose",
cfg=json.dumps(reshape_cfg)),
)
response_data = json.loads(resp.data)
assert response_data["data_id"] == c.port
def save_filter(self):
curr_settings = global_state.get_settings(self.data_id)
curr_filters = curr_settings.get('columnFilters') or {}
fltr = self.builder.build_filter()
if fltr is None:
curr_filters.pop(self.column, None)
else:
curr_filters[self.column] = fltr
curr_settings['columnFilters'] = curr_filters
global_state.set_settings(self.data_id, curr_settings)
def initialize_store(test_data):
"""Helper function that sets up a default store with some data in it"""
global_state.cleanup()
global_state.use_default_store()
for data_id in ["1", "2"]:
global_state.set_data(data_id, test_data)
global_state.set_dtypes(data_id, build_dtypes_state(test_data))
global_state.set_settings(data_id, dict(locked=[]))
global_state.set_name(data_id, "test_name" + data_id)
global_state.set_context_variables(
data_id, dict(favorite_words=["foo", "bar", "baz"]))
global_state.set_history(data_id, ["foo", "bar", "baz"])
def initialize_store(test_data):
"""Helper function that sets up a default store with some data in it"""
global_state.cleanup()
global_state.use_default_store()
for data_id in ['1', '2']:
global_state.set_data(data_id, test_data)
global_state.set_dtypes(data_id, build_dtypes_state(test_data))
global_state.set_settings(data_id, dict(locked=[]))
global_state.set_metadata(data_id, dict(name='test_name'))
global_state.set_context_variables(
data_id, dict(favorite_words=['foo', 'bar', 'baz']))
global_state.set_history(data_id, ['foo', 'bar', 'baz'])
def save_filter(self):
curr_settings = global_state.get_settings(self.data_id)
filters_key = "{}Filters".format("outlier" if self.cfg["type"] ==
"outliers" else "column")
curr_filters = curr_settings.get(filters_key) or {}
fltr = self.builder.build_filter()
if fltr is None:
curr_filters.pop(self.column, None)
else:
curr_filters[self.column] = fltr
curr_settings[filters_key] = curr_filters
global_state.set_settings(self.data_id, curr_settings)
return curr_filters
def save_filter(self):
curr_settings = global_state.get_settings(self.data_id)
filters_key = '{}Filters'.format('outlier' if self.cfg['type'] ==
'outliers' else 'column')
curr_filters = curr_settings.get(filters_key) or {}
fltr = self.builder.build_filter()
if fltr is None:
curr_filters.pop(self.column, None)
else:
curr_filters[self.column] = fltr
curr_settings[filters_key] = curr_filters
global_state.set_settings(self.data_id, curr_settings)
return curr_filters
def execute(self):
from dtale.views import startup
data = global_state.get_data(self.data_id)
try:
df, code = self.checker.remove(data)
instance = startup(data=df, **self.checker.startup_kwargs)
curr_settings = global_state.get_settings(instance._data_id)
global_state.set_settings(
instance._data_id,
dict_merge(curr_settings, dict(startup_code=code)),
)
return instance._data_id
except NoDuplicatesException:
return self.data_id
def test_get_column_analysis(unittest, test_data):
import dtale.views as views
with app.test_client() as c:
with ExitStack() as stack:
build_data_inst({c.port: test_data})
build_dtypes({c.port: views.build_dtypes_state(test_data)})
build_settings({c.port: {}})
response = c.get(
"/dtale/column-analysis/{}".format(c.port), query_string=dict(col="foo")
)
response_data = json.loads(response.data)
expected = dict(
labels=[
"0.6",
"0.6",
"0.7",
"0.7",
"0.8",
"0.8",
"0.9",
"0.9",
"0.9",
"1.0",
"1.1",
"1.1",
"1.1",
"1.2",
"1.2",
"1.3",
"1.4",
"1.4",
"1.5",
"1.5",
],
data=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0],
desc={
"count": "50",
"std": "0",
"min": "1",
"max": "1",
"50%": "1",
"25%": "1",
"75%": "1",
"mean": "1",
"missing_ct": "0",
"missing_pct": 0.0,
"total_count": "50",
"kurt": 0.0,
"skew": 0.0,
},
chart_type="histogram",
dtype="int64",
query="",
)
unittest.assertEqual(
{k: v for k, v in response_data.items() if k not in ["code", "cols"]},
expected,
"should return 20-bin histogram for foo",
)
unittest.assertEqual(response_data["code"], HISTOGRAM_CODE)
response = c.get(
"/dtale/column-analysis/{}".format(c.port),
query_string=dict(col="foo", bins=5),
)
response_data = json.loads(response.data)
expected = dict(
labels=["0.7", "0.9", "1.1", "1.3", "1.5"],
data=[0, 0, 50, 0, 0],
desc={
"count": "50",
"std": "0",
"min": "1",
"max": "1",
"50%": "1",
"25%": "1",
"75%": "1",
"mean": "1",
"missing_ct": "0",
"missing_pct": 0.0,
"total_count": "50",
"kurt": 0.0,
"skew": 0.0,
},
chart_type="histogram",
dtype="int64",
query="",
)
unittest.assertEqual(
{k: v for k, v in response_data.items() if k not in ["code", "cols"]},
expected,
"should return 5-bin histogram for foo",
)
global_state.set_settings(c.port, dict(query="security_id > 10"))
response = c.get(
"/dtale/column-analysis/{}".format(c.port),
query_string=dict(col="foo", bins=5),
)
response_data = json.loads(response.data)
expected = dict(
labels=["0.7", "0.9", "1.1", "1.3", "1.5"],
data=[0, 0, 39, 0, 0],
desc={
"count": "39",
"std": "0",
"min": "1",
"max": "1",
"50%": "1",
"25%": "1",
"75%": "1",
"mean": "1",
"missing_ct": "0",
"missing_pct": 0.0,
"total_count": "39",
"kurt": 0.0,
"skew": 0.0,
},
chart_type="histogram",
dtype="int64",
query="security_id > 10",
)
unittest.assertEqual(
{k: v for k, v in response_data.items() if k not in ["code", "cols"]},
expected,
"should return a filtered 5-bin histogram for foo",
)
global_state.set_settings(c.port, {})
response = c.get(
"/dtale/column-analysis/{}".format(c.port),
query_string=dict(col="foo", type="value_counts", top=2),
)
response_data = json.loads(response.data)
assert response_data["chart_type"] == "value_counts"
response = c.get(
"/dtale/column-analysis/{}".format(c.port),
query_string=dict(
col="foo", type="value_counts", ordinalCol="bar", ordinalAgg="mean"
),
)
response_data = json.loads(response.data)
assert "ordinal" in response_data
response = c.get(
"/dtale/column-analysis/{}".format(c.port),
query_string=dict(
col="foo",
type="value_counts",
ordinalCol="bar",
ordinalAgg="pctsum",
),
)
response_data = json.loads(response.data)
assert "ordinal" in response_data
response = c.get(
"/dtale/column-analysis/{}".format(c.port),
query_string=dict(
col="bar", type="categories", categoryCol="foo", categoryAgg="mean"
),
)
response_data = json.loads(response.data)
assert "count" in response_data
response = c.get(
"/dtale/column-analysis/{}".format(c.port),
query_string=dict(
col="bar",
type="categories",
categoryCol="foo",
categoryAgg="pctsum",
),
)
response_data = json.loads(response.data)
assert "count" in response_data
with app.test_client() as c:
with ExitStack() as stack:
build_data_inst({c.port: test_data})
stack.enter_context(
mock.patch(
"numpy.histogram",
mock.Mock(side_effect=Exception("histogram failure")),
)
)
response = c.get(
"/dtale/column-analysis/{}".format(c.port), query_string=dict(col="foo")
)
response_data = json.loads(response.data)
unittest.assertEqual(
response_data["error"],
"histogram failure",
"should handle histogram exception",
)
def test_set_filters(unittest):
filters = []
df = pd.DataFrame(
([[1, 2, 3, 4, 5, 6], [7, 8, 9, 10, 11, 12], [13, 14, 15, 16, 17, 18]
]),
columns=["A", "B", "C", "D", "E", "F"],
)
with ExitStack() as stack:
stack.enter_context(
mock.patch("dtale.predefined_filters.PREDEFINED_FILTERS", filters))
predefined_filters.set_filters([
{
"name": "A and B > 2",
"column": "A",
"description": "Filter A with B greater than 2",
"handler": lambda df, val: df[
(df["A"] == val) & (df["B"] > 2)],
"input_type": "input",
},
{
"name":
"A and (B % 2) == 0",
"column":
"A",
"description":
"Filter A with B mod 2 equals zero (is even)",
"handler":
lambda df, val: df[(df["A"] == val) & (df["B"] % 2 == 0)],
"input_type":
"select",
},
{
"name":
"A in values and (B % 2) == 0",
"column":
"A",
"description":
"A is within a group of values and B mod 2 equals zero (is even)",
"handler":
lambda df, val: df[df["A"].isin(val) & (df["B"] % 2 == 0)],
"input_type":
"multiselect",
},
])
assert len(predefined_filters.get_filters()[0].handler(df, 1)) == 0
assert len(predefined_filters.get_filters()[0].handler(df, 7)) == 1
unittest.assertEqual(
predefined_filters.get_filters()[0].asdict(),
{
"name": "A and B > 2",
"column": "A",
"description": "Filter A with B greater than 2",
"inputType": "input",
},
)
assert len(predefined_filters.get_filters()[1].handler(df, 1)) == 1
assert len(predefined_filters.get_filters()[1].handler(df, 7)) == 1
assert len(predefined_filters.get_filters()[2].handler(df, [1])) == 1
assert len(predefined_filters.get_filters()[2].handler(df,
[1, 7])) == 2
global_state.set_data("1", df)
assert len(handle_predefined("1")) == 3
global_state.set_settings("1",
dict(predefinedFilters={"A and B > 2": 7}))
assert len(handle_predefined("1")) == 1
def build_settings(input_dict):
for data_id, data in input_dict.items():
global_state.set_settings(data_id, data)
