def test_time_dataframe_dataset():
import dtale.global_state as global_state
global_state.clear_store()
with build_app(url=URL).test_client() as c:
c.get("/dtale/datasets", query_string=dict(dataset="time_dataframe"))
assert global_state.get_data(1)["A"].isnull().sum() == 0
def test_covid_dataset():
import dtale.global_state as global_state
global_state.clear_store()
def mock_load_csv(**kwargs):
if (
kwargs.get("path")
== "https://raw.githubusercontent.com/nytimes/covid-19-data/master/us-states.csv"
):
return pd.DataFrame(dict(state=["a", "b"]))
elif (
kwargs.get("path")
== "https://raw.githubusercontent.com/jasonong/List-of-US-States/master/states.csv"
):
return pd.DataFrame(dict(State=["a"], Abbreviation=["A"]))
return None
with build_app(url=URL).test_client() as c:
with ExitStack() as stack:
stack.enter_context(
mock.patch("dtale.cli.loaders.csv_loader.loader_func", mock_load_csv)
)
c.get("/dtale/datasets", query_string=dict(dataset="covid"))
assert global_state.get_data(1).state_code.values[0] == "A"
def test_hide_columns():
from dtale.app import show
import dtale.global_state as global_state
global_state.clear_store()
df = pd.DataFrame(dict(a=[1, 2], b=[2, 3]))
with ExitStack() as stack:
stack.enter_context(mock.patch("dtale.app.DtaleFlask", MockDtaleFlask))
stack.enter_context(mock.patch("dtale.app.DtaleFlask.run",
mock.Mock()))
stack.enter_context(
mock.patch("dtale.app.find_free_port",
mock.Mock(return_value=9999)))
stack.enter_context(
mock.patch("socket.gethostname",
mock.Mock(return_value="localhost")))
stack.enter_context(
mock.patch("dtale.app.is_up", mock.Mock(return_value=False)))
stack.enter_context(mock.patch("requests.get", mock.Mock()))
instance = show(data=df,
ignore_duplicate=True,
hide_columns=["b"],
subprocess=False)
assert global_state.get_dtypes(instance._data_id)[0]["visible"] is True
assert not global_state.get_dtypes(
instance._data_id)[1]["visible"] is True
def test_transpose(custom_data, unittest):
from dtale.views import build_dtypes_state
global_state.clear_store()
with app.test_client() as c:
data = {c.port: custom_data}
dtypes = {c.port: build_dtypes_state(custom_data)}
settings = {c.port: {}}
build_data_inst(data)
build_dtypes(dtypes)
build_settings(settings)
reshape_cfg = dict(index=["security_id"], columns=["Col0"])
resp = c.get(
"/dtale/reshape/{}".format(c.port),
query_string=dict(output="new",
type="transpose",
cfg=json.dumps(reshape_cfg)),
)
response_data = json.loads(resp.data)
new_key = int(c.port) + 1
assert "error" in response_data
min_date = custom_data["date"].min().strftime("%Y-%m-%d")
global_state.set_settings(c.port,
dict(query="date == '{}'".format(min_date)))
reshape_cfg = dict(index=["date", "security_id"], columns=["Col0"])
resp = c.get(
"/dtale/reshape/{}".format(c.port),
query_string=dict(output="new",
type="transpose",
cfg=json.dumps(reshape_cfg)),
)
response_data = json.loads(resp.data)
assert response_data["data_id"] == new_key
assert len(global_state.keys()) == 2
unittest.assertEqual(
[d["name"] for d in global_state.get_dtypes(new_key)],
[
"index",
"{} 00:00:00 100000".format(min_date),
"{} 00:00:00 100001".format(min_date),
],
)
assert len(global_state.get_data(new_key)) == 1
assert global_state.get_settings(new_key).get(
"startup_code") is not None
c.get("/dtale/cleanup-datasets", query_string=dict(dataIds=new_key))
reshape_cfg = dict(index=["date", "security_id"])
resp = c.get(
"/dtale/reshape/{}".format(c.port),
query_string=dict(output="override",
type="transpose",
cfg=json.dumps(reshape_cfg)),
)
response_data = json.loads(resp.data)
assert response_data["data_id"] == c.port
def test_stack(unittest):
from dtale.views import build_dtypes_state
import dtale.global_state as global_state
global_state.clear_store()
df1 = pd.DataFrame({
"A": ["A0", "A1"],
"B": ["B0", "B1"],
"C": ["C0", "C1"],
"D": ["D0", "D1"],
})
df2 = pd.DataFrame({
"A": ["A2", "A3"],
"B": ["B3", "B3"],
"C": ["C3", "C3"],
"D": ["D3", "D3"],
})
with app.test_client() as c:
data = {"1": df1, "2": df2}
dtypes = {k: build_dtypes_state(v) for k, v in data.items()}
settings = {k: {} for k in data.keys()}
build_data_inst(data)
build_dtypes(dtypes)
build_settings(settings)
datasets = [dict(dataId="1", columns=[]), dict(dataId="2", columns=[])]
config = dict(ignore_index=False)
resp = c.post(
"/dtale/merge",
data=dict(
action="stack",
config=json.dumps(config),
datasets=json.dumps(datasets),
),
)
assert resp.status_code == 200
final_df = global_state.get_data(resp.json["data_id"])
unittest.assertEqual(list(final_df["A"].values),
["A0", "A1", "A2", "A3"])
unittest.assertEqual(list(final_df["index"].values), [0, 1, 0, 1])
config["ignoreIndex"] = True
resp = c.post(
"/dtale/merge",
data=dict(
action="stack",
config=json.dumps(config),
datasets=json.dumps(datasets),
),
)
assert resp.status_code == 200
final_df = global_state.get_data(resp.json["data_id"])
assert "index" not in final_df.columns
unittest.assertEqual(list(final_df["A"].values),
["A0", "A1", "A2", "A3"])
def test_view(unittest):
from dtale.views import build_dtypes_state
import dtale.global_state as global_state
global_state.clear_store()
df = replacements_data()
with app.test_client() as c:
data = {c.port: df}
dtypes = {c.port: build_dtypes_state(df)}
build_data_inst(data)
build_dtypes(dtypes)
resp = c.get(
"/dtale/build-replacement/{}".format(c.port),
query_string=dict(type="not_implemented", name="test", cfg=json.dumps({})),
)
response_data = resp.json
assert (
response_data["error"]
== "'not_implemented' replacement not implemented yet!"
)
params = dict(
type="value",
col="e",
name="a",
cfg=json.dumps(
{"value": [dict(value="nan", type="raw", replace="for test")]}
),
)
resp = c.get("/dtale/build-replacement/{}".format(c.port), query_string=params)
response_data = resp.json
assert response_data["error"] == "A column named 'a' already exists!"
params = dict(
type="value",
col="e",
name="e2",
cfg=json.dumps(
{"value": [dict(value="nan", type="raw", replace="for test")]}
),
)
c.get("/dtale/build-replacement/{}".format(c.port), query_string=params)
unittest.assertEqual(list(data[c.port]["e2"].values), ["a", "for test", "b"])
dtypes = global_state.get_dtypes(c.port)
assert dtypes[-1]["name"] == "e2"
assert dtypes[-1]["dtype"] == "string" if PY3 else "mixed"
assert not dtypes[-1]["hasMissing"]
del params["name"]
c.get("/dtale/build-replacement/{}".format(c.port), query_string=params)
dtypes = global_state.get_dtypes(c.port)
unittest.assertEqual(list(data[c.port]["e"].values), ["a", "for test", "b"])
e_dtype = next((d for d in dtypes if d["name"] == "e"))
assert not e_dtype["hasMissing"]
def test_columns(unittest):
import dtale.global_state as global_state
global_state.clear_store()
data_id, duplicates_type = "1", "columns"
data = {data_id: duplicates_data()}
build_data_inst(data)
builder = DuplicateCheck(data_id, duplicates_type, {"keep": "first"})
unittest.assertEqual(builder.test(), {"Foo": ["foo"]})
new_data_id = builder.execute()
unittest.assertEqual(
list(global_state.get_data(new_data_id).columns), ["Foo", "fOo", "foO", "bar"]
)
data = {data_id: duplicates_data()}
build_data_inst(data)
builder = DuplicateCheck(data_id, duplicates_type, {"keep": "last"})
unittest.assertEqual(builder.test(), {"foo": ["Foo"]})
new_data_id = builder.execute()
unittest.assertEqual(
list(global_state.get_data(new_data_id).columns), ["foo", "fOo", "foO", "bar"]
)
data = {data_id: duplicates_data()}
build_data_inst(data)
builder = DuplicateCheck(data_id, duplicates_type, {"keep": "none"})
unittest.assertEqual(builder.test(), {"Foo": ["foo"]})
new_data_id = builder.execute()
unittest.assertEqual(
list(global_state.get_data(new_data_id).columns), ["fOo", "foO", "bar"]
)
data = {data_id: duplicates_data().drop(["fOo", "foO", "bar"], axis=1)}
build_data_inst(data)
builder = DuplicateCheck(data_id, duplicates_type, {"keep": "none"})
with pytest.raises(RemoveAllDataException):
builder.execute()
data = {data_id: non_duplicate_data()}
build_data_inst(data)
builder = DuplicateCheck(data_id, duplicates_type, {"keep": "none"})
with pytest.raises(NoDuplicatesException):
builder.checker.remove(data[data_id])
def test_seinfeld_dataset():
import dtale.global_state as global_state
global_state.clear_store()
def mock_load_csv(**kwargs):
return pd.DataFrame(dict(SEID=["a"]))
with build_app(url=URL).test_client() as c:
with ExitStack() as stack:
stack.enter_context(
mock.patch("dtale.cli.loaders.csv_loader.loader_func", mock_load_csv)
)
c.get("/dtale/datasets", query_string=dict(dataset="seinfeld"))
assert global_state.get_data(1).SEID.values[0] == "a"
def test_use_default_store(unittest, tmpdir, test_data):
global_state.clear_store()
"""Make sure flipping back and forth multiple times doesn't corrupt the data"""
initialize_store(test_data)
contents_before = get_store_contents()
type_before = get_store_type()
directory = tmpdir.mkdir("test_use_default_store").dirname
global_state.use_shelve_store(directory)
global_state.use_default_store()
contents_after = get_store_contents()
type_after = get_store_type()
unittest.assertEqual(contents_before[1], contents_after[1])
unittest.assertEqual(type_before, type_after)
def test_rows(unittest):
import dtale.global_state as global_state
global_state.clear_store()
data_id, duplicates_type = "1", "rows"
data = {data_id: duplicates_data()}
build_data_inst(data)
builder = DuplicateCheck(
data_id, duplicates_type, {"keep": "first", "subset": "foo"}
)
unittest.assertEqual(builder.test(), dict(removed=0, total=5, remaining=5))
pre_length = len(data[data_id])
new_data_id = builder.execute()
assert pre_length == len(global_state.get_data(new_data_id))
data = {data_id: duplicates_data()}
build_data_inst(data)
builder = DuplicateCheck(
data_id, duplicates_type, {"keep": "first", "subset": ["foO", "bar"]}
)
unittest.assertEqual(builder.test(), dict(removed=3, total=5, remaining=2))
new_data_id = builder.execute()
assert len(global_state.get_data(new_data_id)) == 2
unittest.assertEqual(global_state.get_data(new_data_id)["Foo"].tolist(), [1, 4])
data = {data_id: duplicates_data()}
build_data_inst(data)
builder = DuplicateCheck(
data_id, duplicates_type, {"keep": "last", "subset": ["foO", "bar"]}
)
unittest.assertEqual(builder.test(), dict(removed=3, total=5, remaining=2))
new_data_id = builder.execute()
assert len(global_state.get_data(new_data_id)) == 2
unittest.assertEqual(global_state.get_data(new_data_id)["Foo"].tolist(), [3, 5])
data = {data_id: duplicates_data()}
build_data_inst(data)
builder = DuplicateCheck(
data_id, duplicates_type, {"keep": "none", "subset": ["foO", "bar"]}
)
unittest.assertEqual(builder.test(), dict(removed=5, total=5, remaining=0))
with pytest.raises(RemoveAllDataException):
builder.execute()
def test_convert():
from dtale.views import startup
from tests.dtale.test_replacements import replacements_data
import dtale.global_state as global_state
global_state.clear_store()
with app.test_client() as c:
global_state.new_data_inst(c.port)
startup(URL, data=replacements_data(), data_id=c.port)
resp = c.get(
"/dtale/to-xarray/{}".format(c.port),
query_string=dict(index=json.dumps(["a"])),
)
assert resp.status_code == 200
assert global_state.get_dataset(c.port) is not None
assert global_state.get_settings(c.port)["locked"] == ["a"]
def test_resample(unittest):
from dtale.views import build_dtypes_state, format_data
start, end = "2000-10-01 23:30:00", "2000-10-03 00:30:00"
rng = pd.date_range(start, end, freq="7min")
ts = pd.Series(np.arange(len(rng)) * 3, index=rng)
ts2 = pd.Series(np.arange(len(rng)) * 0.32, index=rng)
df = pd.DataFrame(data={"col1": ts, "col2": ts2})
df, _ = format_data(df)
global_state.clear_store()
with app.test_client() as c:
data = {c.port: df}
dtypes = {c.port: build_dtypes_state(df)}
settings = {c.port: {}}
build_data_inst(data)
build_dtypes(dtypes)
build_settings(settings)
reshape_cfg = dict(index="index",
columns=["col1"],
freq="17min",
agg="mean")
resp = c.get(
"/dtale/reshape/{}".format(c.port),
query_string=dict(output="new",
type="resample",
cfg=json.dumps(reshape_cfg)),
)
response_data = json.loads(resp.data)
new_key = int(c.port) + 1
assert response_data["data_id"] == new_key
assert len(global_state.keys()) == 2
unittest.assertEqual(
[d["name"] for d in global_state.get_dtypes(new_key)],
["index_17min", "col1"],
)
assert len(global_state.get_data(new_key)) == 90
assert global_state.get_settings(new_key).get(
"startup_code") is not None
c.get("/dtale/cleanup-datasets", query_string=dict(dataIds=new_key))
def test_show_duplicates(unittest):
import dtale.global_state as global_state
global_state.clear_store()
data_id, duplicates_type = "1", "show"
data = {data_id: duplicates_data()}
build_data_inst(data)
builder = DuplicateCheck(data_id, duplicates_type, {"group": ["foo"]})
unittest.assertEqual(builder.test(), {})
with pytest.raises(NoDuplicatesToShowException):
builder.execute()
data = {data_id: duplicates_data()}
build_data_inst(data)
builder = DuplicateCheck(data_id, duplicates_type, {"group": ["foO", "bar"]})
unittest.assertEqual(
builder.test(),
{
"4, 5": dict(count=3, filter=["4", "5"]),
"4, 6": dict(count=2, filter=["4", "6"]),
},
)
new_data_id = builder.execute()
assert new_data_id == 2
unittest.assertEqual(
global_state.get_data(new_data_id)["Foo"].tolist(), [1, 2, 3, 4, 5]
)
data = {data_id: duplicates_data()}
build_data_inst(data)
builder = DuplicateCheck(
data_id, duplicates_type, {"group": ["foO", "bar"], "filter": ["4", "5"]}
)
new_data_id = builder.execute()
unittest.assertEqual(global_state.get_data(new_data_id)["Foo"].tolist(), [1, 2, 3])
def test_view(unittest):
from dtale.views import build_dtypes_state
import dtale.global_state as global_state
global_state.clear_store()
df = duplicates_data()
with app.test_client() as c:
data = {c.port: df}
dtypes = {c.port: build_dtypes_state(df)}
build_data_inst(data)
build_dtypes(dtypes)
resp = c.get(
"/dtale/duplicates/{}".format(c.port),
query_string=dict(
type="not_implemented", action="execute", cfg=json.dumps({})
),
)
response_data = resp.json
assert (
response_data["error"]
== "'not_implemented' duplicate check not implemented yet!"
)
params = dict(
type="columns",
action="test",
cfg=json.dumps({"keep": "first"}),
)
resp = c.get("/dtale/duplicates/{}".format(c.port), query_string=params)
response_data = resp.json
unittest.assertEqual(response_data, {"results": {"Foo": ["foo"]}})
params["action"] = "execute"
resp = c.get("/dtale/duplicates/{}".format(c.port), query_string=params)
response_data = resp.json
assert response_data["data_id"] == c.port
def test_pivot(custom_data, unittest):
from dtale.views import build_dtypes_state
global_state.clear_store()
with app.test_client() as c:
data = {c.port: custom_data}
dtypes = {c.port: build_dtypes_state(custom_data)}
settings = {c.port: {}}
build_data_inst(data)
build_dtypes(dtypes)
build_settings(settings)
reshape_cfg = dict(index=["date"],
columns=["security_id"],
values=["Col0"],
aggfunc="mean")
resp = c.get(
"/dtale/reshape/{}".format(c.port),
query_string=dict(output="new",
type="pivot",
cfg=json.dumps(reshape_cfg)),
)
response_data = json.loads(resp.data)
new_key = int(c.port) + 1
assert response_data["data_id"] == new_key
assert len(global_state.keys()) == 2
unittest.assertEqual(
[d["name"] for d in global_state.get_dtypes(new_key)],
["date", "100000", "100001"],
)
assert len(global_state.get_data(new_key)) == 365
assert global_state.get_settings(new_key).get(
"startup_code") is not None
resp = c.get("/dtale/cleanup-datasets",
query_string=dict(dataIds=new_key))
assert json.loads(resp.data)["success"]
assert len(global_state.keys()) == 1
reshape_cfg["columnNameHeaders"] = True
reshape_cfg["aggfunc"] = "sum"
resp = c.get(
"/dtale/reshape/{}".format(c.port),
query_string=dict(output="new",
type="pivot",
cfg=json.dumps(reshape_cfg)),
)
response_data = json.loads(resp.data)
assert response_data["data_id"] == new_key
assert len(global_state.keys()) == 2
unittest.assertEqual(
[d["name"] for d in global_state.get_dtypes(new_key)],
["date", "security_id-100000", "security_id-100001"],
)
assert len(global_state.get_data(new_key)) == 365
assert global_state.get_settings(new_key).get(
"startup_code") is not None
c.get("/dtale/cleanup-datasets", query_string=dict(dataIds=new_key))
reshape_cfg["columnNameHeaders"] = False
reshape_cfg["values"] = ["Col0", "Col1"]
resp = c.get(
"/dtale/reshape/{}".format(c.port),
query_string=dict(output="new",
type="pivot",
cfg=json.dumps(reshape_cfg)),
)
response_data = json.loads(resp.data)
assert response_data["data_id"] == new_key
assert len(global_state.keys()) == 2
unittest.assertEqual(
[d["name"] for d in global_state.get_dtypes(new_key)],
[
"date", "Col0 100000", "Col0 100001", "Col1 100000",
"Col1 100001"
],
)
assert len(global_state.get_data(new_key)) == 365
assert global_state.get_settings(new_key).get(
"startup_code") is not None
c.get("/dtale/cleanup-datasets", query_string=dict(dataIds=new_key))
def test_aggregate(custom_data, unittest):
from dtale.views import build_dtypes_state
global_state.clear_store()
with app.test_client() as c:
data = {c.port: custom_data}
dtypes = {c.port: build_dtypes_state(custom_data)}
settings = {c.port: {}}
build_data_inst(data)
build_dtypes(dtypes)
build_settings(settings)
reshape_cfg = dict(
index="date",
agg=dict(type="col",
cols={
"Col0": ["sum", "mean"],
"Col1": ["count"]
}),
)
resp = c.get(
"/dtale/reshape/{}".format(c.port),
query_string=dict(output="new",
type="aggregate",
cfg=json.dumps(reshape_cfg)),
)
response_data = json.loads(resp.data)
new_key = int(c.port) + 1
assert response_data["data_id"] == new_key
assert len(global_state.keys()) == 2
unittest.assertEqual(
[d["name"] for d in global_state.get_dtypes(new_key)],
["date", "Col0 sum", "Col0 mean", "Col1 count"],
)
assert len(global_state.get_data(new_key)) == 365
assert global_state.get_settings(new_key).get(
"startup_code") is not None
c.get("/dtale/cleanup-datasets", query_string=dict(dataIds=new_key))
reshape_cfg = dict(index="date",
agg=dict(type="func",
func="mean",
cols=["Col0", "Col1"]))
resp = c.get(
"/dtale/reshape/{}".format(c.port),
query_string=dict(output="new",
type="aggregate",
cfg=json.dumps(reshape_cfg)),
)
response_data = json.loads(resp.data)
assert response_data["data_id"] == new_key
assert len(global_state.keys()) == 2
unittest.assertEqual(
[d["name"] for d in global_state.get_dtypes(new_key)],
["date", "Col0", "Col1"],
)
assert len(global_state.get_data(new_key)) == 365
assert global_state.get_settings(new_key).get(
"startup_code") is not None
c.get("/dtale/cleanup-datasets", query_string=dict(dataIds=new_key))
reshape_cfg = dict(index="date", agg=dict(type="func", func="mean"))
resp = c.get(
"/dtale/reshape/{}".format(c.port),
query_string=dict(output="new",
type="aggregate",
cfg=json.dumps(reshape_cfg)),
)
response_data = json.loads(resp.data)
assert response_data["data_id"] == new_key
assert len(global_state.keys()) == 2
unittest.assertEqual(
[d["name"] for d in global_state.get_dtypes(new_key)],
[
"date", "security_id", "int_val", "Col0", "Col1", "Col2",
"bool_val"
],
)
assert len(global_state.get_data(new_key)) == 365
assert global_state.get_settings(new_key).get(
"startup_code") is not None
c.get("/dtale/cleanup-datasets", query_string=dict(dataIds=new_key))
def test_view(unittest):
from dtale.views import startup
import dtale.global_state as global_state
global_state.clear_store()
with app.test_client() as c:
global_state.new_data_inst(c.port)
startup(URL, data=xarray_data(), data_id=c.port)
assert global_state.get_dataset(c.port) is not None
response = c.get("/dtale/main/{}".format(c.port))
assert 'input id="xarray" value="True"' not in str(response.data)
assert 'input id="xarray_dim" value="{}"' not in str(response.data)
resp = c.get("/dtale/code-export/{}".format(c.port))
assert resp.status_code == 200
response_data = resp.json
assert response_data["success"]
resp = c.get("/dtale/xarray-coordinates/{}".format(c.port))
response_data = resp.json
expected = [
{
"count": 3,
"dtype": "str64" if PY3 else "string16",
"name": "location",
},
{
"count": 731,
"dtype": "datetime64[ns]",
"name": "time"
},
]
unittest.assertEqual(
sorted(response_data["data"], key=lambda c: c["name"]), expected)
resp = c.get("/dtale/xarray-dimension-values/{}/location".format(
c.port))
response_data = resp.json
unittest.assertEqual(
response_data["data"],
[{
"value": "IA"
}, {
"value": "IN"
}, {
"value": "IL"
}],
)
resp = c.get(
"/dtale/update-xarray-selection/{}".format(c.port),
query_string=dict(selection=json.dumps(dict(location="IA"))),
)
assert resp.status_code == 200
assert list(global_state.get_data(c.port).location.unique()) == ["IA"]
assert global_state.get_dataset_dim(c.port)["location"] == "IA"
resp = c.get(
"/dtale/update-xarray-selection/{}".format(c.port),
query_string=dict(selection=json.dumps(dict())),
)
assert resp.status_code == 200
assert list(global_state.get_data(c.port).location.unique()) == [
"IA",
"IN",
"IL",
]
resp = c.get("/dtale/code-export/{}".format(c.port))
assert resp.status_code == 200
response_data = resp.json
assert response_data["success"]
with app.test_client() as c:
zero_dim_xarray = xarray_data().sel(location="IA", time="2000-01-01")
startup(URL, data=zero_dim_xarray, data_id=c.port)
assert global_state.get_dataset(c.port) is not None
response = c.get("/dtale/main/{}".format(c.port))
assert 'input id="xarray" value="True"' not in str(response.data)
assert 'input id="xarray_dim" value="{}"' not in str(response.data)
def test_show_jupyter_server_proxy(unittest):
from dtale.app import show, get_instance, instances
import dtale.views as views
import dtale.global_state as global_state
global_state.clear_store()
test_data = pd.DataFrame([dict(a=1, b=2)])
with ExitStack() as stack:
stack.enter_context(mock.patch("dtale.app.JUPYTER_SERVER_PROXY", True))
mock_run = stack.enter_context(
mock.patch("dtale.app.DtaleFlask.run", mock.Mock()))
stack.enter_context(
mock.patch("dtale.app.is_up", mock.Mock(return_value=False)))
stack.enter_context(mock.patch("dtale.app.ACTIVE_PORT", 40000))
stack.enter_context(mock.patch("dtale.app.ACTIVE_HOST", "localhost"))
mock_requests = stack.enter_context(
mock.patch("requests.get", mock.Mock()))
instance = show(data=test_data,
subprocess=False,
name="foo",
ignore_duplicate=True)
assert "/user/{}/proxy/40000".format(
getpass.getuser()) == instance._url
mock_run.assert_called_once()
pdt.assert_frame_equal(instance.data, test_data)
tmp = test_data.copy()
tmp["biz"] = 2.5
instance.data = tmp
unittest.assertEqual(
global_state.get_dtypes(instance._data_id),
views.build_dtypes_state(tmp),
"should update app data/dtypes",
)
instance2 = get_instance(instance._data_id)
assert instance2._url == instance._url
instances()
assert get_instance(
20) is None # should return None for invalid data ids
instance.kill()
mock_requests.assert_called_once()
assert mock_requests.call_args[0][
0] == "/user/{}/proxy/40000/shutdown".format(getpass.getuser())
with ExitStack() as stack:
stack.enter_context(mock.patch("dtale.app.JUPYTER_SERVER_PROXY", True))
stack.enter_context(mock.patch("dtale.app.ACTIVE_PORT", 40000))
stack.enter_context(mock.patch("dtale.app.ACTIVE_HOST", "localhost"))
mock_run = stack.enter_context(
mock.patch("dtale.app.DtaleFlask.run", mock.Mock()))
stack.enter_context(
mock.patch("dtale.app.is_up", mock.Mock(return_value=False)))
mock_requests = stack.enter_context(
mock.patch("requests.get", mock.Mock()))
instance = show(
data=test_data,
subprocess=False,
ignore_duplicate=True,
app_root="/custom_root/",
)
assert "/custom_root/40000" == instance._url
mock_run.assert_called_once()
instance2 = get_instance(instance._data_id)
# this is a known bug where get_instance will not work if you've specified an `app_root' in show()
assert not instance2._url == instance._url
instances()
instance.kill()
mock_requests.assert_called_once()
assert mock_requests.call_args[0][0] == "/custom_root/40000/shutdown"
def test_show(unittest):
from dtale.app import show, get_instance, instances
import dtale.views as views
import dtale.global_state as global_state
global_state.clear_store()
instances()
test_data = pd.DataFrame([dict(a=1, b=2)])
with ExitStack() as stack:
mock_run = stack.enter_context(
mock.patch("dtale.app.DtaleFlask.run", mock.Mock()))
mock_find_free_port = stack.enter_context(
mock.patch("dtale.app.find_free_port",
mock.Mock(return_value=9999)))
stack.enter_context(
mock.patch("socket.gethostname",
mock.Mock(return_value="localhost")))
stack.enter_context(
mock.patch("dtale.app.is_up", mock.Mock(return_value=False)))
mock_requests = stack.enter_context(
mock.patch("requests.get", mock.Mock()))
instance = show(data=test_data,
subprocess=False,
ignore_duplicate=True)
assert "http://localhost:9999" == instance._url
assert "http://localhost:9999/dtale/main/1" == instance.main_url()
mock_run.assert_called_once()
mock_find_free_port.assert_called_once()
pdt.assert_frame_equal(instance.data, test_data)
tmp = test_data.copy()
tmp["biz"] = 2.5
instance.data = tmp
unittest.assertEqual(
global_state.get_dtypes(instance._data_id),
views.build_dtypes_state(tmp),
"should update app data/dtypes",
)
instance2 = get_instance(instance._data_id)
assert instance2._url == instance._url
# removed data name test. wait for proper data name implementation.
instances()
assert get_instance(
20) is None # should return None for invalid data ids
instance.kill()
mock_requests.assert_called_once()
assert mock_requests.call_args[0][
0] == "http://localhost:9999/shutdown"
instance3 = show(data=test_data,
subprocess=False,
name="It's Here",
ignore_duplicate=True)
assert instance3.main_url() == "http://localhost:9999/dtale/main/2"
pdt.assert_frame_equal(instance3.data, test_data)
with ExitStack() as stack:
mock_run = stack.enter_context(
mock.patch("dtale.app.DtaleFlask.run", mock.Mock()))
mock_find_free_port = stack.enter_context(
mock.patch("dtale.app.find_free_port",
mock.Mock(return_value=9999)))
stack.enter_context(
mock.patch("socket.gethostname",
mock.Mock(return_value="localhost")))
stack.enter_context(
mock.patch("dtale.app.is_up", mock.Mock(return_value=False)))
mock_data_loader = mock.Mock(return_value=test_data)
instance = show(
data_loader=mock_data_loader,
subprocess=False,
port=9999,
force=True,
debug=True,
ignore_duplicate=True,
)
assert "http://localhost:9999" == instance._url
mock_run.assert_called_once()
mock_find_free_port.assert_not_called()
mock_data_loader.assert_called_once()
_, kwargs = mock_run.call_args
assert "9999" in instance._url
# RangeIndex test
test_data = pd.DataFrame([1, 2, 3])
with ExitStack() as stack:
stack.enter_context(mock.patch("dtale.app.DtaleFlask", MockDtaleFlask))
stack.enter_context(
mock.patch("dtale.app.find_free_port",
mock.Mock(return_value=9999)))
stack.enter_context(
mock.patch("socket.gethostname",
mock.Mock(return_value="localhost")))
stack.enter_context(
mock.patch("dtale.app.is_up", mock.Mock(return_value=False)))
stack.enter_context(mock.patch("dtale.app.logger", mock.Mock()))
instance = show(data=test_data,
subprocess=False,
name="foo",
ignore_duplicate=True)
assert np.array_equal(instance.data["0"].values, test_data[0].values)
with ExitStack() as stack:
stack.enter_context(mock.patch("dtale.app.DtaleFlask", MockDtaleFlask))
stack.enter_context(
mock.patch("dtale.app.find_free_port",
mock.Mock(return_value=9999)))
stack.enter_context(
mock.patch("socket.gethostname",
mock.Mock(return_value="localhost")))
stack.enter_context(
mock.patch("dtale.app.is_up", mock.Mock(return_value=False)))
stack.enter_context(mock.patch("dtale.app.logger", mock.Mock()))
stack.enter_context(
mock.patch("dtale.views.in_ipython_frontend",
mock.Mock(return_value=False)))
get_calls = {"ct": 0}
getter = namedtuple("get", "ok")
def mock_requests_get(url, verify=True):
if url.endswith("/health"):
is_ok = get_calls["ct"] > 0
get_calls["ct"] += 1
return getter(is_ok)
return getter(True)
stack.enter_context(mock.patch("requests.get", mock_requests_get))
mock_display = stack.enter_context(
mock.patch("IPython.display.display", mock.Mock()))
mock_iframe = stack.enter_context(
mock.patch("IPython.display.IFrame", mock.Mock()))
instance = show(
data=test_data,
subprocess=True,
name="foo1",
notebook=True,
ignore_duplicate=True,
)
mock_display.assert_called_once()
mock_iframe.assert_called_once()
assert mock_iframe.call_args[0][
0] == "http://localhost:9999/dtale/iframe/{}".format(
instance._data_id)
assert type(instance.__str__()).__name__ == "str"
assert type(instance.__repr__()).__name__ == "str"
class MockDtaleFlaskRunTest(Flask):
def __init__(self,
import_name,
reaper_on=True,
url=None,
app_root=None,
*args,
**kwargs):
kwargs.pop("instance_relative_config", None)
kwargs.pop("static_url_path", None)
super(MockDtaleFlaskRunTest,
self).__init__(import_name, *args, **kwargs)
def run(self, *args, **kwargs):
assert self.jinja_env.auto_reload
assert self.config["TEMPLATES_AUTO_RELOAD"]
with mock.patch("dtale.app.DtaleFlask", MockDtaleFlaskRunTest):
show(
data=test_data,
subprocess=False,
port=9999,
debug=True,
ignore_duplicate=True,
)
with mock.patch("dtale.app._thread.start_new_thread",
mock.Mock()) as mock_thread:
show(data=test_data, subprocess=True, ignore_duplicate=True)
mock_thread.assert_called()
test_data = pd.DataFrame([dict(a=1, b=2)])
with ExitStack() as stack:
mock_build_app = stack.enter_context(
mock.patch("dtale.app.build_app", mock.Mock()))
stack.enter_context(
mock.patch("dtale.app.find_free_port",
mock.Mock(return_value=9999)))
stack.enter_context(
mock.patch("socket.gethostname",
mock.Mock(return_value="localhost")))
stack.enter_context(
mock.patch("dtale.app.is_up", mock.Mock(return_value=False)))
stack.enter_context(mock.patch("requests.get", mock.Mock()))
show(data=test_data,
subprocess=False,
name="foo2",
ignore_duplicate=True)
_, kwargs = mock_build_app.call_args
unittest.assertEqual(
{
"app_root": None,
"host": "localhost",
"reaper_on": True
},
kwargs,
"build_app should be called with defaults",
)
# test adding duplicate column
with ExitStack() as stack:
stack.enter_context(mock.patch("dtale.app.DtaleFlask", MockDtaleFlask))
stack.enter_context(
mock.patch("dtale.app.find_free_port",
mock.Mock(return_value=9999)))
stack.enter_context(
mock.patch("socket.gethostname",
mock.Mock(return_value="localhost")))
stack.enter_context(
mock.patch("dtale.app.is_up", mock.Mock(return_value=False)))
stack.enter_context(mock.patch("requests.get", mock.Mock()))
instance = show(
data=pd.DataFrame([dict(a=1, b=2)]),
subprocess=False,
name="foo3",
ignore_duplicate=True,
)
with pytest.raises(Exception):
instance.data = instance.data.rename(columns={"b": "a"})
curr_instance_ct = global_state.size()
show(data=pd.DataFrame([dict(a=1, b=2)]), subprocess=False, name="foo")
assert curr_instance_ct == global_state.size()
# cleanup
global_state.cleanup()
def test_merge(unittest):
from dtale.views import build_dtypes_state
import dtale.global_state as global_state
global_state.clear_store()
left = pd.DataFrame(
{
"key1": ["K0", "K0", "K1", "K2"],
"key2": ["K0", "K1", "K0", "K1"],
"A": ["A0", "A1", "A2", "A3"],
"B": ["B0", "B1", "B2", "B3"],
}
)
right = pd.DataFrame(
{
"key1": ["K0", "K1", "K1", "K2"],
"key2": ["K0", "K0", "K0", "K0"],
"C": ["C0", "C1", "C2", "C3"],
"D": ["D0", "D1", "D2", "D3"],
}
)
right2 = pd.DataFrame(
{
"key1": ["K0", "K1"],
"key2": ["K0", "K0"],
"E": ["E0", "E1"],
"F": ["F0", "F1"],
}
)
with app.test_client() as c:
data = {"1": left, "2": right, "3": right2}
dtypes = {k: build_dtypes_state(v) for k, v in data.items()}
settings = {k: {} for k in data.keys()}
build_data_inst(data)
build_dtypes(dtypes)
build_settings(settings)
datasets = [
dict(dataId="1", columns=[], index=["key1", "key2"], suffix=""),
dict(dataId="2", columns=[], index=["key1", "key2"], suffix=""),
]
config = dict(how="inner", sort=False, indicator=False)
resp = c.post(
"/dtale/merge",
data=json.dumps(
dict(
action="merge",
config=json.dumps(config),
datasets=json.dumps(datasets),
)
),
content_type="application/json",
)
assert resp.status_code == 200
final_df = global_state.get_data(resp.json["data_id"])
unittest.assertEqual(
list(final_df.columns), ["key1", "key2", "A", "B", "C", "D"]
)
assert len(final_df) == 3
datasets[0]["columns"] = ["A"]
datasets[1]["columns"] = ["C"]
config["how"] = "left"
config["indicator"] = True
resp = c.post(
"/dtale/merge",
data=json.dumps(
dict(
action="merge",
config=json.dumps(config),
datasets=json.dumps(datasets),
)
),
content_type="application/json",
)
assert resp.status_code == 200
final_df = global_state.get_data(resp.json["data_id"])
unittest.assertEqual(
list(final_df.columns), ["key1", "key2", "A", "C", "merge_1"]
)
unittest.assertEqual(
list(final_df["merge_1"].values),
["both", "left_only", "both", "both", "left_only"],
)
datasets.append(dict(dataId="3", index=["key1", "key2"], suffix="3"))
resp = c.post(
"/dtale/merge",
data=json.dumps(
dict(
action="merge",
config=json.dumps(config),
datasets=json.dumps(datasets),
)
),
content_type="application/json",
)
assert resp.status_code == 200
final_df = global_state.get_data(resp.json["data_id"])
unittest.assertEqual(
list(final_df.columns),
["key1", "key2", "A", "C", "merge_1", "E", "F", "merge_2"],
)
unittest.assertEqual(
list(final_df["merge_2"].values),
["both", "left_only", "both", "both", "left_only"],
)
def test_upload(unittest):
import dtale.views as views
import dtale.global_state as global_state
global_state.clear_store()
df, _ = views.format_data(pd.DataFrame([1, 2, 3]))
with build_app(url=URL).test_client() as c:
build_data_inst({c.port: df})
global_state.set_dtypes(c.port, views.build_dtypes_state(df))
resp = c.post("/dtale/upload")
assert not resp.get_json()["success"]
c.post(
"/dtale/upload",
data={
"tests_df.csv": (build_upload_data(), "test_df.csv"),
"separatorType": "csv",
},
)
assert global_state.size() == 2
new_key = next((k for k in global_state.keys() if k != c.port), None)
assert list(global_state.get_data(new_key).columns) == ["a", "b", "c"]
with build_app(url=URL).test_client() as c:
global_state.clear_store()
build_data_inst({c.port: df})
global_state.set_dtypes(c.port, views.build_dtypes_state(df))
resp = c.post("/dtale/upload")
assert not resp.get_json()["success"]
c.post(
"/dtale/upload",
data={
"tests_df.csv": (build_upload_data(), "test_df.csv"),
"separatorType": "custom",
"separator": ",",
},
)
assert global_state.size() == 2
new_key = next((k for k in global_state.keys() if k != c.port), None)
assert list(global_state.get_data(new_key).columns) == ["a", "b", "c"]
with build_app(url=URL).test_client() as c:
global_state.clear_store()
build_data_inst({c.port: df})
global_state.set_dtypes(c.port, views.build_dtypes_state(df))
assert global_state.size() == 1
if PY3:
c.post(
"/dtale/upload",
data={
"test_df.xlsx": (
os.path.join(
os.path.dirname(__file__), "..", "data/test_df.xlsx"
),
"test_df.xlsx",
)
},
)
assert global_state.size() == 2
new_key = next((k for k in global_state.keys() if k != c.port), None)
assert list(global_state.get_data(new_key).columns) == ["a", "b", "c"]
with build_app(url=URL).test_client() as c:
with ExitStack() as stack:
global_state.clear_store()
data = {c.port: df}
build_data_inst(data)
global_state.set_dtypes(c.port, views.build_dtypes_state(df))
stack.enter_context(
mock.patch(
"dtale.views.pd.read_excel",
mock.Mock(
return_value={
"Sheet 1": pd.DataFrame(dict(a=[1], b=[2])),
"Sheet 2": pd.DataFrame(dict(c=[1], d=[2])),
}
),
)
)
resp = c.post(
"/dtale/upload",
data={
"test_df.xlsx": (
os.path.join(
os.path.dirname(__file__), "..", "data/test_df.xlsx"
),
"test_df.xlsx",
)
},
)
assert global_state.size() == 3
sheets = resp.json["sheets"]
assert len(sheets) == 2
unittest.assertEqual(
sorted([s["name"] for s in sheets]),
["Sheet 1", "Sheet 2"],
)
def test_web_upload(unittest):
import dtale.global_state as global_state
global_state.clear_store()
with build_app(url=URL).test_client() as c:
with ExitStack() as stack:
load_csv = stack.enter_context(
mock.patch(
"dtale.cli.loaders.csv_loader.loader_func",
mock.Mock(return_value=pd.DataFrame(dict(a=[1], b=[2]))),
)
)
load_excel = stack.enter_context(
mock.patch(
"dtale.cli.loaders.excel_loader.load_file",
mock.Mock(
return_value={"Sheet 1": pd.DataFrame(dict(a=[1], b=[2]))}
),
)
)
load_json = stack.enter_context(
mock.patch(
"dtale.cli.loaders.json_loader.loader_func",
mock.Mock(return_value=pd.DataFrame(dict(a=[1], b=[2]))),
)
)
params = {"type": "csv", "url": "http://test.com"}
c.get("/dtale/web-upload", query_string=params)
load_csv.assert_called_once()
unittest.assertEqual(
load_csv.call_args.kwargs,
{"path": "http://test.com", "proxy": None},
)
assert global_state.size() == 1
load_csv.reset_mock()
params = {"type": "tsv", "url": "http://test.com"}
c.get("/dtale/web-upload", query_string=params)
load_csv.assert_called_once()
unittest.assertEqual(
load_csv.call_args.kwargs,
{"path": "http://test.com", "proxy": None, "delimiter": "\t"},
)
assert global_state.size() == 2
params = {
"type": "json",
"url": "http://test.com",
"proxy": "http://testproxy.com",
}
c.get("/dtale/web-upload", query_string=params)
load_json.assert_called_once()
unittest.assertEqual(
load_json.call_args.kwargs,
{"path": "http://test.com", "proxy": "http://testproxy.com"},
)
assert global_state.size() == 3
params = {"type": "excel", "url": "http://test.com"}
c.get("/dtale/web-upload", query_string=params)
load_excel.assert_called_once()
unittest.assertEqual(
load_excel.call_args.kwargs,
{"path": "http://test.com", "proxy": None},
)
assert global_state.size() == 4
global_state.clear_store()
load_excel.reset_mock()
load_excel.return_value = {
"Sheet 1": pd.DataFrame(dict(a=[1], b=[2])),
"Sheet 2": pd.DataFrame(dict(c=[1], d=[2])),
}
resp = c.get("/dtale/web-upload", query_string=params)
sheets = resp.json["sheets"]
assert len(sheets) == 2
unittest.assertEqual(
sorted([s["name"] for s in sheets]),
["Sheet 1", "Sheet 2"],
)
