def test_time_dataframe_dataset(): import dtale.global_state as global_state global_state.clear_store() with build_app(url=URL).test_client() as c: c.get("/dtale/datasets", query_string=dict(dataset="time_dataframe")) assert global_state.get_data(1)["A"].isnull().sum() == 0
def test_covid_dataset(): import dtale.global_state as global_state global_state.clear_store() def mock_load_csv(**kwargs): if ( kwargs.get("path") == "https://raw.githubusercontent.com/nytimes/covid-19-data/master/us-states.csv" ): return pd.DataFrame(dict(state=["a", "b"])) elif ( kwargs.get("path") == "https://raw.githubusercontent.com/jasonong/List-of-US-States/master/states.csv" ): return pd.DataFrame(dict(State=["a"], Abbreviation=["A"])) return None with build_app(url=URL).test_client() as c: with ExitStack() as stack: stack.enter_context( mock.patch("dtale.cli.loaders.csv_loader.loader_func", mock_load_csv) ) c.get("/dtale/datasets", query_string=dict(dataset="covid")) assert global_state.get_data(1).state_code.values[0] == "A"
def test_hide_columns(): from dtale.app import show import dtale.global_state as global_state global_state.clear_store() df = pd.DataFrame(dict(a=[1, 2], b=[2, 3])) with ExitStack() as stack: stack.enter_context(mock.patch("dtale.app.DtaleFlask", MockDtaleFlask)) stack.enter_context(mock.patch("dtale.app.DtaleFlask.run", mock.Mock())) stack.enter_context( mock.patch("dtale.app.find_free_port", mock.Mock(return_value=9999))) stack.enter_context( mock.patch("socket.gethostname", mock.Mock(return_value="localhost"))) stack.enter_context( mock.patch("dtale.app.is_up", mock.Mock(return_value=False))) stack.enter_context(mock.patch("requests.get", mock.Mock())) instance = show(data=df, ignore_duplicate=True, hide_columns=["b"], subprocess=False) assert global_state.get_dtypes(instance._data_id)[0]["visible"] is True assert not global_state.get_dtypes( instance._data_id)[1]["visible"] is True
def test_transpose(custom_data, unittest): from dtale.views import build_dtypes_state global_state.clear_store() with app.test_client() as c: data = {c.port: custom_data} dtypes = {c.port: build_dtypes_state(custom_data)} settings = {c.port: {}} build_data_inst(data) build_dtypes(dtypes) build_settings(settings) reshape_cfg = dict(index=["security_id"], columns=["Col0"]) resp = c.get( "/dtale/reshape/{}".format(c.port), query_string=dict(output="new", type="transpose", cfg=json.dumps(reshape_cfg)), ) response_data = json.loads(resp.data) new_key = int(c.port) + 1 assert "error" in response_data min_date = custom_data["date"].min().strftime("%Y-%m-%d") global_state.set_settings(c.port, dict(query="date == '{}'".format(min_date))) reshape_cfg = dict(index=["date", "security_id"], columns=["Col0"]) resp = c.get( "/dtale/reshape/{}".format(c.port), query_string=dict(output="new", type="transpose", cfg=json.dumps(reshape_cfg)), ) response_data = json.loads(resp.data) assert response_data["data_id"] == new_key assert len(global_state.keys()) == 2 unittest.assertEqual( [d["name"] for d in global_state.get_dtypes(new_key)], [ "index", "{} 00:00:00 100000".format(min_date), "{} 00:00:00 100001".format(min_date), ], ) assert len(global_state.get_data(new_key)) == 1 assert global_state.get_settings(new_key).get( "startup_code") is not None c.get("/dtale/cleanup-datasets", query_string=dict(dataIds=new_key)) reshape_cfg = dict(index=["date", "security_id"]) resp = c.get( "/dtale/reshape/{}".format(c.port), query_string=dict(output="override", type="transpose", cfg=json.dumps(reshape_cfg)), ) response_data = json.loads(resp.data) assert response_data["data_id"] == c.port
def test_stack(unittest): from dtale.views import build_dtypes_state import dtale.global_state as global_state global_state.clear_store() df1 = pd.DataFrame({ "A": ["A0", "A1"], "B": ["B0", "B1"], "C": ["C0", "C1"], "D": ["D0", "D1"], }) df2 = pd.DataFrame({ "A": ["A2", "A3"], "B": ["B3", "B3"], "C": ["C3", "C3"], "D": ["D3", "D3"], }) with app.test_client() as c: data = {"1": df1, "2": df2} dtypes = {k: build_dtypes_state(v) for k, v in data.items()} settings = {k: {} for k in data.keys()} build_data_inst(data) build_dtypes(dtypes) build_settings(settings) datasets = [dict(dataId="1", columns=[]), dict(dataId="2", columns=[])] config = dict(ignore_index=False) resp = c.post( "/dtale/merge", data=dict( action="stack", config=json.dumps(config), datasets=json.dumps(datasets), ), ) assert resp.status_code == 200 final_df = global_state.get_data(resp.json["data_id"]) unittest.assertEqual(list(final_df["A"].values), ["A0", "A1", "A2", "A3"]) unittest.assertEqual(list(final_df["index"].values), [0, 1, 0, 1]) config["ignoreIndex"] = True resp = c.post( "/dtale/merge", data=dict( action="stack", config=json.dumps(config), datasets=json.dumps(datasets), ), ) assert resp.status_code == 200 final_df = global_state.get_data(resp.json["data_id"]) assert "index" not in final_df.columns unittest.assertEqual(list(final_df["A"].values), ["A0", "A1", "A2", "A3"])
def test_view(unittest): from dtale.views import build_dtypes_state import dtale.global_state as global_state global_state.clear_store() df = replacements_data() with app.test_client() as c: data = {c.port: df} dtypes = {c.port: build_dtypes_state(df)} build_data_inst(data) build_dtypes(dtypes) resp = c.get( "/dtale/build-replacement/{}".format(c.port), query_string=dict(type="not_implemented", name="test", cfg=json.dumps({})), ) response_data = resp.json assert ( response_data["error"] == "'not_implemented' replacement not implemented yet!" ) params = dict( type="value", col="e", name="a", cfg=json.dumps( {"value": [dict(value="nan", type="raw", replace="for test")]} ), ) resp = c.get("/dtale/build-replacement/{}".format(c.port), query_string=params) response_data = resp.json assert response_data["error"] == "A column named 'a' already exists!" params = dict( type="value", col="e", name="e2", cfg=json.dumps( {"value": [dict(value="nan", type="raw", replace="for test")]} ), ) c.get("/dtale/build-replacement/{}".format(c.port), query_string=params) unittest.assertEqual(list(data[c.port]["e2"].values), ["a", "for test", "b"]) dtypes = global_state.get_dtypes(c.port) assert dtypes[-1]["name"] == "e2" assert dtypes[-1]["dtype"] == "string" if PY3 else "mixed" assert not dtypes[-1]["hasMissing"] del params["name"] c.get("/dtale/build-replacement/{}".format(c.port), query_string=params) dtypes = global_state.get_dtypes(c.port) unittest.assertEqual(list(data[c.port]["e"].values), ["a", "for test", "b"]) e_dtype = next((d for d in dtypes if d["name"] == "e")) assert not e_dtype["hasMissing"]
def test_columns(unittest): import dtale.global_state as global_state global_state.clear_store() data_id, duplicates_type = "1", "columns" data = {data_id: duplicates_data()} build_data_inst(data) builder = DuplicateCheck(data_id, duplicates_type, {"keep": "first"}) unittest.assertEqual(builder.test(), {"Foo": ["foo"]}) new_data_id = builder.execute() unittest.assertEqual( list(global_state.get_data(new_data_id).columns), ["Foo", "fOo", "foO", "bar"] ) data = {data_id: duplicates_data()} build_data_inst(data) builder = DuplicateCheck(data_id, duplicates_type, {"keep": "last"}) unittest.assertEqual(builder.test(), {"foo": ["Foo"]}) new_data_id = builder.execute() unittest.assertEqual( list(global_state.get_data(new_data_id).columns), ["foo", "fOo", "foO", "bar"] ) data = {data_id: duplicates_data()} build_data_inst(data) builder = DuplicateCheck(data_id, duplicates_type, {"keep": "none"}) unittest.assertEqual(builder.test(), {"Foo": ["foo"]}) new_data_id = builder.execute() unittest.assertEqual( list(global_state.get_data(new_data_id).columns), ["fOo", "foO", "bar"] ) data = {data_id: duplicates_data().drop(["fOo", "foO", "bar"], axis=1)} build_data_inst(data) builder = DuplicateCheck(data_id, duplicates_type, {"keep": "none"}) with pytest.raises(RemoveAllDataException): builder.execute() data = {data_id: non_duplicate_data()} build_data_inst(data) builder = DuplicateCheck(data_id, duplicates_type, {"keep": "none"}) with pytest.raises(NoDuplicatesException): builder.checker.remove(data[data_id])
def test_seinfeld_dataset(): import dtale.global_state as global_state global_state.clear_store() def mock_load_csv(**kwargs): return pd.DataFrame(dict(SEID=["a"])) with build_app(url=URL).test_client() as c: with ExitStack() as stack: stack.enter_context( mock.patch("dtale.cli.loaders.csv_loader.loader_func", mock_load_csv) ) c.get("/dtale/datasets", query_string=dict(dataset="seinfeld")) assert global_state.get_data(1).SEID.values[0] == "a"
def test_use_default_store(unittest, tmpdir, test_data): global_state.clear_store() """Make sure flipping back and forth multiple times doesn't corrupt the data""" initialize_store(test_data) contents_before = get_store_contents() type_before = get_store_type() directory = tmpdir.mkdir("test_use_default_store").dirname global_state.use_shelve_store(directory) global_state.use_default_store() contents_after = get_store_contents() type_after = get_store_type() unittest.assertEqual(contents_before[1], contents_after[1]) unittest.assertEqual(type_before, type_after)
def test_rows(unittest): import dtale.global_state as global_state global_state.clear_store() data_id, duplicates_type = "1", "rows" data = {data_id: duplicates_data()} build_data_inst(data) builder = DuplicateCheck( data_id, duplicates_type, {"keep": "first", "subset": "foo"} ) unittest.assertEqual(builder.test(), dict(removed=0, total=5, remaining=5)) pre_length = len(data[data_id]) new_data_id = builder.execute() assert pre_length == len(global_state.get_data(new_data_id)) data = {data_id: duplicates_data()} build_data_inst(data) builder = DuplicateCheck( data_id, duplicates_type, {"keep": "first", "subset": ["foO", "bar"]} ) unittest.assertEqual(builder.test(), dict(removed=3, total=5, remaining=2)) new_data_id = builder.execute() assert len(global_state.get_data(new_data_id)) == 2 unittest.assertEqual(global_state.get_data(new_data_id)["Foo"].tolist(), [1, 4]) data = {data_id: duplicates_data()} build_data_inst(data) builder = DuplicateCheck( data_id, duplicates_type, {"keep": "last", "subset": ["foO", "bar"]} ) unittest.assertEqual(builder.test(), dict(removed=3, total=5, remaining=2)) new_data_id = builder.execute() assert len(global_state.get_data(new_data_id)) == 2 unittest.assertEqual(global_state.get_data(new_data_id)["Foo"].tolist(), [3, 5]) data = {data_id: duplicates_data()} build_data_inst(data) builder = DuplicateCheck( data_id, duplicates_type, {"keep": "none", "subset": ["foO", "bar"]} ) unittest.assertEqual(builder.test(), dict(removed=5, total=5, remaining=0)) with pytest.raises(RemoveAllDataException): builder.execute()
def test_convert(): from dtale.views import startup from tests.dtale.test_replacements import replacements_data import dtale.global_state as global_state global_state.clear_store() with app.test_client() as c: global_state.new_data_inst(c.port) startup(URL, data=replacements_data(), data_id=c.port) resp = c.get( "/dtale/to-xarray/{}".format(c.port), query_string=dict(index=json.dumps(["a"])), ) assert resp.status_code == 200 assert global_state.get_dataset(c.port) is not None assert global_state.get_settings(c.port)["locked"] == ["a"]
def test_resample(unittest): from dtale.views import build_dtypes_state, format_data start, end = "2000-10-01 23:30:00", "2000-10-03 00:30:00" rng = pd.date_range(start, end, freq="7min") ts = pd.Series(np.arange(len(rng)) * 3, index=rng) ts2 = pd.Series(np.arange(len(rng)) * 0.32, index=rng) df = pd.DataFrame(data={"col1": ts, "col2": ts2}) df, _ = format_data(df) global_state.clear_store() with app.test_client() as c: data = {c.port: df} dtypes = {c.port: build_dtypes_state(df)} settings = {c.port: {}} build_data_inst(data) build_dtypes(dtypes) build_settings(settings) reshape_cfg = dict(index="index", columns=["col1"], freq="17min", agg="mean") resp = c.get( "/dtale/reshape/{}".format(c.port), query_string=dict(output="new", type="resample", cfg=json.dumps(reshape_cfg)), ) response_data = json.loads(resp.data) new_key = int(c.port) + 1 assert response_data["data_id"] == new_key assert len(global_state.keys()) == 2 unittest.assertEqual( [d["name"] for d in global_state.get_dtypes(new_key)], ["index_17min", "col1"], ) assert len(global_state.get_data(new_key)) == 90 assert global_state.get_settings(new_key).get( "startup_code") is not None c.get("/dtale/cleanup-datasets", query_string=dict(dataIds=new_key))
def test_show_duplicates(unittest): import dtale.global_state as global_state global_state.clear_store() data_id, duplicates_type = "1", "show" data = {data_id: duplicates_data()} build_data_inst(data) builder = DuplicateCheck(data_id, duplicates_type, {"group": ["foo"]}) unittest.assertEqual(builder.test(), {}) with pytest.raises(NoDuplicatesToShowException): builder.execute() data = {data_id: duplicates_data()} build_data_inst(data) builder = DuplicateCheck(data_id, duplicates_type, {"group": ["foO", "bar"]}) unittest.assertEqual( builder.test(), { "4, 5": dict(count=3, filter=["4", "5"]), "4, 6": dict(count=2, filter=["4", "6"]), }, ) new_data_id = builder.execute() assert new_data_id == 2 unittest.assertEqual( global_state.get_data(new_data_id)["Foo"].tolist(), [1, 2, 3, 4, 5] ) data = {data_id: duplicates_data()} build_data_inst(data) builder = DuplicateCheck( data_id, duplicates_type, {"group": ["foO", "bar"], "filter": ["4", "5"]} ) new_data_id = builder.execute() unittest.assertEqual(global_state.get_data(new_data_id)["Foo"].tolist(), [1, 2, 3])
def test_view(unittest): from dtale.views import build_dtypes_state import dtale.global_state as global_state global_state.clear_store() df = duplicates_data() with app.test_client() as c: data = {c.port: df} dtypes = {c.port: build_dtypes_state(df)} build_data_inst(data) build_dtypes(dtypes) resp = c.get( "/dtale/duplicates/{}".format(c.port), query_string=dict( type="not_implemented", action="execute", cfg=json.dumps({}) ), ) response_data = resp.json assert ( response_data["error"] == "'not_implemented' duplicate check not implemented yet!" ) params = dict( type="columns", action="test", cfg=json.dumps({"keep": "first"}), ) resp = c.get("/dtale/duplicates/{}".format(c.port), query_string=params) response_data = resp.json unittest.assertEqual(response_data, {"results": {"Foo": ["foo"]}}) params["action"] = "execute" resp = c.get("/dtale/duplicates/{}".format(c.port), query_string=params) response_data = resp.json assert response_data["data_id"] == c.port
def test_pivot(custom_data, unittest): from dtale.views import build_dtypes_state global_state.clear_store() with app.test_client() as c: data = {c.port: custom_data} dtypes = {c.port: build_dtypes_state(custom_data)} settings = {c.port: {}} build_data_inst(data) build_dtypes(dtypes) build_settings(settings) reshape_cfg = dict(index=["date"], columns=["security_id"], values=["Col0"], aggfunc="mean") resp = c.get( "/dtale/reshape/{}".format(c.port), query_string=dict(output="new", type="pivot", cfg=json.dumps(reshape_cfg)), ) response_data = json.loads(resp.data) new_key = int(c.port) + 1 assert response_data["data_id"] == new_key assert len(global_state.keys()) == 2 unittest.assertEqual( [d["name"] for d in global_state.get_dtypes(new_key)], ["date", "100000", "100001"], ) assert len(global_state.get_data(new_key)) == 365 assert global_state.get_settings(new_key).get( "startup_code") is not None resp = c.get("/dtale/cleanup-datasets", query_string=dict(dataIds=new_key)) assert json.loads(resp.data)["success"] assert len(global_state.keys()) == 1 reshape_cfg["columnNameHeaders"] = True reshape_cfg["aggfunc"] = "sum" resp = c.get( "/dtale/reshape/{}".format(c.port), query_string=dict(output="new", type="pivot", cfg=json.dumps(reshape_cfg)), ) response_data = json.loads(resp.data) assert response_data["data_id"] == new_key assert len(global_state.keys()) == 2 unittest.assertEqual( [d["name"] for d in global_state.get_dtypes(new_key)], ["date", "security_id-100000", "security_id-100001"], ) assert len(global_state.get_data(new_key)) == 365 assert global_state.get_settings(new_key).get( "startup_code") is not None c.get("/dtale/cleanup-datasets", query_string=dict(dataIds=new_key)) reshape_cfg["columnNameHeaders"] = False reshape_cfg["values"] = ["Col0", "Col1"] resp = c.get( "/dtale/reshape/{}".format(c.port), query_string=dict(output="new", type="pivot", cfg=json.dumps(reshape_cfg)), ) response_data = json.loads(resp.data) assert response_data["data_id"] == new_key assert len(global_state.keys()) == 2 unittest.assertEqual( [d["name"] for d in global_state.get_dtypes(new_key)], [ "date", "Col0 100000", "Col0 100001", "Col1 100000", "Col1 100001" ], ) assert len(global_state.get_data(new_key)) == 365 assert global_state.get_settings(new_key).get( "startup_code") is not None c.get("/dtale/cleanup-datasets", query_string=dict(dataIds=new_key))
def test_aggregate(custom_data, unittest): from dtale.views import build_dtypes_state global_state.clear_store() with app.test_client() as c: data = {c.port: custom_data} dtypes = {c.port: build_dtypes_state(custom_data)} settings = {c.port: {}} build_data_inst(data) build_dtypes(dtypes) build_settings(settings) reshape_cfg = dict( index="date", agg=dict(type="col", cols={ "Col0": ["sum", "mean"], "Col1": ["count"] }), ) resp = c.get( "/dtale/reshape/{}".format(c.port), query_string=dict(output="new", type="aggregate", cfg=json.dumps(reshape_cfg)), ) response_data = json.loads(resp.data) new_key = int(c.port) + 1 assert response_data["data_id"] == new_key assert len(global_state.keys()) == 2 unittest.assertEqual( [d["name"] for d in global_state.get_dtypes(new_key)], ["date", "Col0 sum", "Col0 mean", "Col1 count"], ) assert len(global_state.get_data(new_key)) == 365 assert global_state.get_settings(new_key).get( "startup_code") is not None c.get("/dtale/cleanup-datasets", query_string=dict(dataIds=new_key)) reshape_cfg = dict(index="date", agg=dict(type="func", func="mean", cols=["Col0", "Col1"])) resp = c.get( "/dtale/reshape/{}".format(c.port), query_string=dict(output="new", type="aggregate", cfg=json.dumps(reshape_cfg)), ) response_data = json.loads(resp.data) assert response_data["data_id"] == new_key assert len(global_state.keys()) == 2 unittest.assertEqual( [d["name"] for d in global_state.get_dtypes(new_key)], ["date", "Col0", "Col1"], ) assert len(global_state.get_data(new_key)) == 365 assert global_state.get_settings(new_key).get( "startup_code") is not None c.get("/dtale/cleanup-datasets", query_string=dict(dataIds=new_key)) reshape_cfg = dict(index="date", agg=dict(type="func", func="mean")) resp = c.get( "/dtale/reshape/{}".format(c.port), query_string=dict(output="new", type="aggregate", cfg=json.dumps(reshape_cfg)), ) response_data = json.loads(resp.data) assert response_data["data_id"] == new_key assert len(global_state.keys()) == 2 unittest.assertEqual( [d["name"] for d in global_state.get_dtypes(new_key)], [ "date", "security_id", "int_val", "Col0", "Col1", "Col2", "bool_val" ], ) assert len(global_state.get_data(new_key)) == 365 assert global_state.get_settings(new_key).get( "startup_code") is not None c.get("/dtale/cleanup-datasets", query_string=dict(dataIds=new_key))
def test_view(unittest): from dtale.views import startup import dtale.global_state as global_state global_state.clear_store() with app.test_client() as c: global_state.new_data_inst(c.port) startup(URL, data=xarray_data(), data_id=c.port) assert global_state.get_dataset(c.port) is not None response = c.get("/dtale/main/{}".format(c.port)) assert 'input id="xarray" value="True"' not in str(response.data) assert 'input id="xarray_dim" value="{}"' not in str(response.data) resp = c.get("/dtale/code-export/{}".format(c.port)) assert resp.status_code == 200 response_data = resp.json assert response_data["success"] resp = c.get("/dtale/xarray-coordinates/{}".format(c.port)) response_data = resp.json expected = [ { "count": 3, "dtype": "str64" if PY3 else "string16", "name": "location", }, { "count": 731, "dtype": "datetime64[ns]", "name": "time" }, ] unittest.assertEqual( sorted(response_data["data"], key=lambda c: c["name"]), expected) resp = c.get("/dtale/xarray-dimension-values/{}/location".format( c.port)) response_data = resp.json unittest.assertEqual( response_data["data"], [{ "value": "IA" }, { "value": "IN" }, { "value": "IL" }], ) resp = c.get( "/dtale/update-xarray-selection/{}".format(c.port), query_string=dict(selection=json.dumps(dict(location="IA"))), ) assert resp.status_code == 200 assert list(global_state.get_data(c.port).location.unique()) == ["IA"] assert global_state.get_dataset_dim(c.port)["location"] == "IA" resp = c.get( "/dtale/update-xarray-selection/{}".format(c.port), query_string=dict(selection=json.dumps(dict())), ) assert resp.status_code == 200 assert list(global_state.get_data(c.port).location.unique()) == [ "IA", "IN", "IL", ] resp = c.get("/dtale/code-export/{}".format(c.port)) assert resp.status_code == 200 response_data = resp.json assert response_data["success"] with app.test_client() as c: zero_dim_xarray = xarray_data().sel(location="IA", time="2000-01-01") startup(URL, data=zero_dim_xarray, data_id=c.port) assert global_state.get_dataset(c.port) is not None response = c.get("/dtale/main/{}".format(c.port)) assert 'input id="xarray" value="True"' not in str(response.data) assert 'input id="xarray_dim" value="{}"' not in str(response.data)
def test_show_jupyter_server_proxy(unittest): from dtale.app import show, get_instance, instances import dtale.views as views import dtale.global_state as global_state global_state.clear_store() test_data = pd.DataFrame([dict(a=1, b=2)]) with ExitStack() as stack: stack.enter_context(mock.patch("dtale.app.JUPYTER_SERVER_PROXY", True)) mock_run = stack.enter_context( mock.patch("dtale.app.DtaleFlask.run", mock.Mock())) stack.enter_context( mock.patch("dtale.app.is_up", mock.Mock(return_value=False))) stack.enter_context(mock.patch("dtale.app.ACTIVE_PORT", 40000)) stack.enter_context(mock.patch("dtale.app.ACTIVE_HOST", "localhost")) mock_requests = stack.enter_context( mock.patch("requests.get", mock.Mock())) instance = show(data=test_data, subprocess=False, name="foo", ignore_duplicate=True) assert "/user/{}/proxy/40000".format( getpass.getuser()) == instance._url mock_run.assert_called_once() pdt.assert_frame_equal(instance.data, test_data) tmp = test_data.copy() tmp["biz"] = 2.5 instance.data = tmp unittest.assertEqual( global_state.get_dtypes(instance._data_id), views.build_dtypes_state(tmp), "should update app data/dtypes", ) instance2 = get_instance(instance._data_id) assert instance2._url == instance._url instances() assert get_instance( 20) is None # should return None for invalid data ids instance.kill() mock_requests.assert_called_once() assert mock_requests.call_args[0][ 0] == "/user/{}/proxy/40000/shutdown".format(getpass.getuser()) with ExitStack() as stack: stack.enter_context(mock.patch("dtale.app.JUPYTER_SERVER_PROXY", True)) stack.enter_context(mock.patch("dtale.app.ACTIVE_PORT", 40000)) stack.enter_context(mock.patch("dtale.app.ACTIVE_HOST", "localhost")) mock_run = stack.enter_context( mock.patch("dtale.app.DtaleFlask.run", mock.Mock())) stack.enter_context( mock.patch("dtale.app.is_up", mock.Mock(return_value=False))) mock_requests = stack.enter_context( mock.patch("requests.get", mock.Mock())) instance = show( data=test_data, subprocess=False, ignore_duplicate=True, app_root="/custom_root/", ) assert "/custom_root/40000" == instance._url mock_run.assert_called_once() instance2 = get_instance(instance._data_id) # this is a known bug where get_instance will not work if you've specified an `app_root' in show() assert not instance2._url == instance._url instances() instance.kill() mock_requests.assert_called_once() assert mock_requests.call_args[0][0] == "/custom_root/40000/shutdown"
def test_show(unittest): from dtale.app import show, get_instance, instances import dtale.views as views import dtale.global_state as global_state global_state.clear_store() instances() test_data = pd.DataFrame([dict(a=1, b=2)]) with ExitStack() as stack: mock_run = stack.enter_context( mock.patch("dtale.app.DtaleFlask.run", mock.Mock())) mock_find_free_port = stack.enter_context( mock.patch("dtale.app.find_free_port", mock.Mock(return_value=9999))) stack.enter_context( mock.patch("socket.gethostname", mock.Mock(return_value="localhost"))) stack.enter_context( mock.patch("dtale.app.is_up", mock.Mock(return_value=False))) mock_requests = stack.enter_context( mock.patch("requests.get", mock.Mock())) instance = show(data=test_data, subprocess=False, ignore_duplicate=True) assert "http://localhost:9999" == instance._url assert "http://localhost:9999/dtale/main/1" == instance.main_url() mock_run.assert_called_once() mock_find_free_port.assert_called_once() pdt.assert_frame_equal(instance.data, test_data) tmp = test_data.copy() tmp["biz"] = 2.5 instance.data = tmp unittest.assertEqual( global_state.get_dtypes(instance._data_id), views.build_dtypes_state(tmp), "should update app data/dtypes", ) instance2 = get_instance(instance._data_id) assert instance2._url == instance._url # removed data name test. wait for proper data name implementation. instances() assert get_instance( 20) is None # should return None for invalid data ids instance.kill() mock_requests.assert_called_once() assert mock_requests.call_args[0][ 0] == "http://localhost:9999/shutdown" instance3 = show(data=test_data, subprocess=False, name="It's Here", ignore_duplicate=True) assert instance3.main_url() == "http://localhost:9999/dtale/main/2" pdt.assert_frame_equal(instance3.data, test_data) with ExitStack() as stack: mock_run = stack.enter_context( mock.patch("dtale.app.DtaleFlask.run", mock.Mock())) mock_find_free_port = stack.enter_context( mock.patch("dtale.app.find_free_port", mock.Mock(return_value=9999))) stack.enter_context( mock.patch("socket.gethostname", mock.Mock(return_value="localhost"))) stack.enter_context( mock.patch("dtale.app.is_up", mock.Mock(return_value=False))) mock_data_loader = mock.Mock(return_value=test_data) instance = show( data_loader=mock_data_loader, subprocess=False, port=9999, force=True, debug=True, ignore_duplicate=True, ) assert "http://localhost:9999" == instance._url mock_run.assert_called_once() mock_find_free_port.assert_not_called() mock_data_loader.assert_called_once() _, kwargs = mock_run.call_args assert "9999" in instance._url # RangeIndex test test_data = pd.DataFrame([1, 2, 3]) with ExitStack() as stack: stack.enter_context(mock.patch("dtale.app.DtaleFlask", MockDtaleFlask)) stack.enter_context( mock.patch("dtale.app.find_free_port", mock.Mock(return_value=9999))) stack.enter_context( mock.patch("socket.gethostname", mock.Mock(return_value="localhost"))) stack.enter_context( mock.patch("dtale.app.is_up", mock.Mock(return_value=False))) stack.enter_context(mock.patch("dtale.app.logger", mock.Mock())) instance = show(data=test_data, subprocess=False, name="foo", ignore_duplicate=True) assert np.array_equal(instance.data["0"].values, test_data[0].values) with ExitStack() as stack: stack.enter_context(mock.patch("dtale.app.DtaleFlask", MockDtaleFlask)) stack.enter_context( mock.patch("dtale.app.find_free_port", mock.Mock(return_value=9999))) stack.enter_context( mock.patch("socket.gethostname", mock.Mock(return_value="localhost"))) stack.enter_context( mock.patch("dtale.app.is_up", mock.Mock(return_value=False))) stack.enter_context(mock.patch("dtale.app.logger", mock.Mock())) stack.enter_context( mock.patch("dtale.views.in_ipython_frontend", mock.Mock(return_value=False))) get_calls = {"ct": 0} getter = namedtuple("get", "ok") def mock_requests_get(url, verify=True): if url.endswith("/health"): is_ok = get_calls["ct"] > 0 get_calls["ct"] += 1 return getter(is_ok) return getter(True) stack.enter_context(mock.patch("requests.get", mock_requests_get)) mock_display = stack.enter_context( mock.patch("IPython.display.display", mock.Mock())) mock_iframe = stack.enter_context( mock.patch("IPython.display.IFrame", mock.Mock())) instance = show( data=test_data, subprocess=True, name="foo1", notebook=True, ignore_duplicate=True, ) mock_display.assert_called_once() mock_iframe.assert_called_once() assert mock_iframe.call_args[0][ 0] == "http://localhost:9999/dtale/iframe/{}".format( instance._data_id) assert type(instance.__str__()).__name__ == "str" assert type(instance.__repr__()).__name__ == "str" class MockDtaleFlaskRunTest(Flask): def __init__(self, import_name, reaper_on=True, url=None, app_root=None, *args, **kwargs): kwargs.pop("instance_relative_config", None) kwargs.pop("static_url_path", None) super(MockDtaleFlaskRunTest, self).__init__(import_name, *args, **kwargs) def run(self, *args, **kwargs): assert self.jinja_env.auto_reload assert self.config["TEMPLATES_AUTO_RELOAD"] with mock.patch("dtale.app.DtaleFlask", MockDtaleFlaskRunTest): show( data=test_data, subprocess=False, port=9999, debug=True, ignore_duplicate=True, ) with mock.patch("dtale.app._thread.start_new_thread", mock.Mock()) as mock_thread: show(data=test_data, subprocess=True, ignore_duplicate=True) mock_thread.assert_called() test_data = pd.DataFrame([dict(a=1, b=2)]) with ExitStack() as stack: mock_build_app = stack.enter_context( mock.patch("dtale.app.build_app", mock.Mock())) stack.enter_context( mock.patch("dtale.app.find_free_port", mock.Mock(return_value=9999))) stack.enter_context( mock.patch("socket.gethostname", mock.Mock(return_value="localhost"))) stack.enter_context( mock.patch("dtale.app.is_up", mock.Mock(return_value=False))) stack.enter_context(mock.patch("requests.get", mock.Mock())) show(data=test_data, subprocess=False, name="foo2", ignore_duplicate=True) _, kwargs = mock_build_app.call_args unittest.assertEqual( { "app_root": None, "host": "localhost", "reaper_on": True }, kwargs, "build_app should be called with defaults", ) # test adding duplicate column with ExitStack() as stack: stack.enter_context(mock.patch("dtale.app.DtaleFlask", MockDtaleFlask)) stack.enter_context( mock.patch("dtale.app.find_free_port", mock.Mock(return_value=9999))) stack.enter_context( mock.patch("socket.gethostname", mock.Mock(return_value="localhost"))) stack.enter_context( mock.patch("dtale.app.is_up", mock.Mock(return_value=False))) stack.enter_context(mock.patch("requests.get", mock.Mock())) instance = show( data=pd.DataFrame([dict(a=1, b=2)]), subprocess=False, name="foo3", ignore_duplicate=True, ) with pytest.raises(Exception): instance.data = instance.data.rename(columns={"b": "a"}) curr_instance_ct = global_state.size() show(data=pd.DataFrame([dict(a=1, b=2)]), subprocess=False, name="foo") assert curr_instance_ct == global_state.size() # cleanup global_state.cleanup()
def test_merge(unittest): from dtale.views import build_dtypes_state import dtale.global_state as global_state global_state.clear_store() left = pd.DataFrame( { "key1": ["K0", "K0", "K1", "K2"], "key2": ["K0", "K1", "K0", "K1"], "A": ["A0", "A1", "A2", "A3"], "B": ["B0", "B1", "B2", "B3"], } ) right = pd.DataFrame( { "key1": ["K0", "K1", "K1", "K2"], "key2": ["K0", "K0", "K0", "K0"], "C": ["C0", "C1", "C2", "C3"], "D": ["D0", "D1", "D2", "D3"], } ) right2 = pd.DataFrame( { "key1": ["K0", "K1"], "key2": ["K0", "K0"], "E": ["E0", "E1"], "F": ["F0", "F1"], } ) with app.test_client() as c: data = {"1": left, "2": right, "3": right2} dtypes = {k: build_dtypes_state(v) for k, v in data.items()} settings = {k: {} for k in data.keys()} build_data_inst(data) build_dtypes(dtypes) build_settings(settings) datasets = [ dict(dataId="1", columns=[], index=["key1", "key2"], suffix=""), dict(dataId="2", columns=[], index=["key1", "key2"], suffix=""), ] config = dict(how="inner", sort=False, indicator=False) resp = c.post( "/dtale/merge", data=json.dumps( dict( action="merge", config=json.dumps(config), datasets=json.dumps(datasets), ) ), content_type="application/json", ) assert resp.status_code == 200 final_df = global_state.get_data(resp.json["data_id"]) unittest.assertEqual( list(final_df.columns), ["key1", "key2", "A", "B", "C", "D"] ) assert len(final_df) == 3 datasets[0]["columns"] = ["A"] datasets[1]["columns"] = ["C"] config["how"] = "left" config["indicator"] = True resp = c.post( "/dtale/merge", data=json.dumps( dict( action="merge", config=json.dumps(config), datasets=json.dumps(datasets), ) ), content_type="application/json", ) assert resp.status_code == 200 final_df = global_state.get_data(resp.json["data_id"]) unittest.assertEqual( list(final_df.columns), ["key1", "key2", "A", "C", "merge_1"] ) unittest.assertEqual( list(final_df["merge_1"].values), ["both", "left_only", "both", "both", "left_only"], ) datasets.append(dict(dataId="3", index=["key1", "key2"], suffix="3")) resp = c.post( "/dtale/merge", data=json.dumps( dict( action="merge", config=json.dumps(config), datasets=json.dumps(datasets), ) ), content_type="application/json", ) assert resp.status_code == 200 final_df = global_state.get_data(resp.json["data_id"]) unittest.assertEqual( list(final_df.columns), ["key1", "key2", "A", "C", "merge_1", "E", "F", "merge_2"], ) unittest.assertEqual( list(final_df["merge_2"].values), ["both", "left_only", "both", "both", "left_only"], )
def test_upload(unittest): import dtale.views as views import dtale.global_state as global_state global_state.clear_store() df, _ = views.format_data(pd.DataFrame([1, 2, 3])) with build_app(url=URL).test_client() as c: build_data_inst({c.port: df}) global_state.set_dtypes(c.port, views.build_dtypes_state(df)) resp = c.post("/dtale/upload") assert not resp.get_json()["success"] c.post( "/dtale/upload", data={ "tests_df.csv": (build_upload_data(), "test_df.csv"), "separatorType": "csv", }, ) assert global_state.size() == 2 new_key = next((k for k in global_state.keys() if k != c.port), None) assert list(global_state.get_data(new_key).columns) == ["a", "b", "c"] with build_app(url=URL).test_client() as c: global_state.clear_store() build_data_inst({c.port: df}) global_state.set_dtypes(c.port, views.build_dtypes_state(df)) resp = c.post("/dtale/upload") assert not resp.get_json()["success"] c.post( "/dtale/upload", data={ "tests_df.csv": (build_upload_data(), "test_df.csv"), "separatorType": "custom", "separator": ",", }, ) assert global_state.size() == 2 new_key = next((k for k in global_state.keys() if k != c.port), None) assert list(global_state.get_data(new_key).columns) == ["a", "b", "c"] with build_app(url=URL).test_client() as c: global_state.clear_store() build_data_inst({c.port: df}) global_state.set_dtypes(c.port, views.build_dtypes_state(df)) assert global_state.size() == 1 if PY3: c.post( "/dtale/upload", data={ "test_df.xlsx": ( os.path.join( os.path.dirname(__file__), "..", "data/test_df.xlsx" ), "test_df.xlsx", ) }, ) assert global_state.size() == 2 new_key = next((k for k in global_state.keys() if k != c.port), None) assert list(global_state.get_data(new_key).columns) == ["a", "b", "c"] with build_app(url=URL).test_client() as c: with ExitStack() as stack: global_state.clear_store() data = {c.port: df} build_data_inst(data) global_state.set_dtypes(c.port, views.build_dtypes_state(df)) stack.enter_context( mock.patch( "dtale.views.pd.read_excel", mock.Mock( return_value={ "Sheet 1": pd.DataFrame(dict(a=[1], b=[2])), "Sheet 2": pd.DataFrame(dict(c=[1], d=[2])), } ), ) ) resp = c.post( "/dtale/upload", data={ "test_df.xlsx": ( os.path.join( os.path.dirname(__file__), "..", "data/test_df.xlsx" ), "test_df.xlsx", ) }, ) assert global_state.size() == 3 sheets = resp.json["sheets"] assert len(sheets) == 2 unittest.assertEqual( sorted([s["name"] for s in sheets]), ["Sheet 1", "Sheet 2"], )
def test_web_upload(unittest): import dtale.global_state as global_state global_state.clear_store() with build_app(url=URL).test_client() as c: with ExitStack() as stack: load_csv = stack.enter_context( mock.patch( "dtale.cli.loaders.csv_loader.loader_func", mock.Mock(return_value=pd.DataFrame(dict(a=[1], b=[2]))), ) ) load_excel = stack.enter_context( mock.patch( "dtale.cli.loaders.excel_loader.load_file", mock.Mock( return_value={"Sheet 1": pd.DataFrame(dict(a=[1], b=[2]))} ), ) ) load_json = stack.enter_context( mock.patch( "dtale.cli.loaders.json_loader.loader_func", mock.Mock(return_value=pd.DataFrame(dict(a=[1], b=[2]))), ) ) params = {"type": "csv", "url": "http://test.com"} c.get("/dtale/web-upload", query_string=params) load_csv.assert_called_once() unittest.assertEqual( load_csv.call_args.kwargs, {"path": "http://test.com", "proxy": None}, ) assert global_state.size() == 1 load_csv.reset_mock() params = {"type": "tsv", "url": "http://test.com"} c.get("/dtale/web-upload", query_string=params) load_csv.assert_called_once() unittest.assertEqual( load_csv.call_args.kwargs, {"path": "http://test.com", "proxy": None, "delimiter": "\t"}, ) assert global_state.size() == 2 params = { "type": "json", "url": "http://test.com", "proxy": "http://testproxy.com", } c.get("/dtale/web-upload", query_string=params) load_json.assert_called_once() unittest.assertEqual( load_json.call_args.kwargs, {"path": "http://test.com", "proxy": "http://testproxy.com"}, ) assert global_state.size() == 3 params = {"type": "excel", "url": "http://test.com"} c.get("/dtale/web-upload", query_string=params) load_excel.assert_called_once() unittest.assertEqual( load_excel.call_args.kwargs, {"path": "http://test.com", "proxy": None}, ) assert global_state.size() == 4 global_state.clear_store() load_excel.reset_mock() load_excel.return_value = { "Sheet 1": pd.DataFrame(dict(a=[1], b=[2])), "Sheet 2": pd.DataFrame(dict(c=[1], d=[2])), } resp = c.get("/dtale/web-upload", query_string=params) sheets = resp.json["sheets"] assert len(sheets) == 2 unittest.assertEqual( sorted([s["name"] for s in sheets]), ["Sheet 1", "Sheet 2"], )