def test_columns_info(self): import liquer.ext.lq_pandas # register pandas commands and state type from liquer.state_types import encode_state_data, decode_state_data filename = encode_token(os.path.dirname( inspect.getfile(self.__class__))+"/test.csv") assert evaluate(f"df_from-{filename}/df_columns").get() == ["a", "b"] assert evaluate( f"df_from-{filename}/columns_info").get()["columns"] == ["a", "b"] assert evaluate( f"df_from-{filename}/columns_info").get()["has_tags"] == False assert evaluate( f"df_from-{filename}/columns_info").get()["types"]["a"].startswith("int") filename = encode_token(os.path.dirname( inspect.getfile(self.__class__))+"/test_hxl.csv") assert evaluate(f"df_from-{filename}/df_columns").get() == ["a", "b"] assert evaluate( f"df_from-{filename}/columns_info").get()["columns"] == ["a", "b"] assert evaluate( f"df_from-{filename}/columns_info").get()["has_tags"] == True assert evaluate( f"df_from-{filename}/columns_info").get()["tags"]["a"] == "#indicator +num +aaa" assert evaluate( f"df_from-{filename}/columns_info").get()["tags"]["b"] == "#indicator +num +bbb" info = evaluate(f"df_from-{filename}/columns_info").get() b, mime, tid = encode_state_data(info) assert info == decode_state_data(b, tid)
def test_from_with_cache(self, httpserver): import liquer.ext.lq_hxl # register HXL commands and state type test_hxl = open( os.path.dirname(inspect.getfile(self.__class__)) + "/test_hxl.csv").read() httpserver.expect_request("/test_hxl.csv").respond_with_data(test_hxl) url = encode_token(httpserver.url_for("/test_hxl.csv")) query = f"hxl_from-{url}" with tempfile.TemporaryDirectory() as cachepath: set_cache(FileCache(cachepath)) state = evaluate(query) data = state.get() assert data.columns[0].header == "a" assert data.columns[0].display_tag == "#indicator+num+aaa" assert data.columns[1].header == "b" assert data.columns[1].display_tag == "#indicator+num+bbb" state = evaluate(query) data = state.get() assert data.columns[0].header == "a" assert data.columns[0].display_tag == "#indicator+num+aaa" assert data.columns[1].header == "b" assert data.columns[1].display_tag == "#indicator+num+bbb" set_cache(None)
def test_head(self): import liquer.ext.lq_pandas # register pandas commands and state type filename = encode_token(os.path.dirname( inspect.getfile(self.__class__))+"/test.csv") df = evaluate(f"df_from-{filename}/head_df-1").get() assert list(df.a) == [1] assert list(df.b) == [2]
def test_tsplit(self): import importlib import liquer.ext.lq_pandas # register pandas commands and state type import liquer.ext.basic from liquer.commands import reset_command_registry reset_command_registry() # prevent double-registration # Hack to enforce registering of the commands importlib.reload(liquer.ext.lq_pandas) importlib.reload(liquer.ext.basic) set_var("server", "http://localhost") set_var("api_path", "/q/") filename = encode_token( os.path.dirname(inspect.getfile(self.__class__)) + "/test_hxl.csv") df = evaluate(f"df_from-{filename}/tsplit_df-a").get() assert "a" in df.columns assert "b" not in df.columns assert list(df.a) == ["#indicator +num +aaa", "1", "3"] assert list(df["query"])[1:] == [ f"df_from-{filename}/teq-a-1", f"df_from-{filename}/teq-a-3", ] assert list(df["link"])[1:] == [ f"http://localhost/q/df_from-{filename}/teq-a-1", f"http://localhost/q/df_from-{filename}/teq-a-3", ]
def test_work_with_parquet(self): filename = encode_token( os.path.dirname(inspect.getfile(self.__class__)) + "/test.csv") with TemporaryDirectory() as tmpdir: path = Path(tmpdir) / "test.parquet" evaluate_and_save(f"df_from-{filename}/test.parquet", target_directory=tmpdir) @first_command(volatile=True, cache=False) def execution_context(): ctx = daf.ExecutionContext() ctx.register_parquet("a", str(path)) return ctx @command def process(ctx): return ctx.sql("SELECT a, b, a+b AS c FROM a") evaluate_and_save(f"execution_context/process/result.parquet", target_directory=tmpdir) df = evaluate( f"execution_context/process/datafusion_to_pandas").get() assert "a" in df.columns assert "b" in df.columns assert "c" in df.columns assert list(df.a) == [1, 3] assert list(df.b) == [2, 4] assert list(df.c) == [3, 7]
def test_qsplit2(self): import liquer.ext.lq_pandas # register pandas commands and state type filename = encode_token(os.path.dirname( inspect.getfile(self.__class__))+"/test.csv") df = evaluate(f"df_from-{filename}/qsplit_df-a-b").get() assert list(df.a) == [1, 3] assert list(df.b) == [2, 4] assert list(df["query"]) == [ f"df_from-{filename}/eq-a-1-b-2", f"df_from-{filename}/eq-a-3-b-4"]
def test_df2hxl(self): import liquer.ext.lq_hxl # register HXL commands and state type import liquer.ext.lq_pandas # register pandas commands and state type path = encode_token(os.path.dirname( inspect.getfile(self.__class__))+"/test_hxl.csv") data = evaluate(f"df_from-{path}/df2hxl").get() assert data.columns[0].header == "a" assert data.columns[0].display_tag == "#indicator+num+aaa" assert data.columns[1].header == "b" assert data.columns[1].display_tag == "#indicator+num+bbb"
def test_qsplit1(self): import liquer.ext.lq_pandas # register pandas commands and state type filename = encode_token(os.path.dirname( inspect.getfile(self.__class__))+"/test.csv") df = evaluate(f"df_from-{filename}/qsplit_df-a").get() assert "a" in df.columns assert "b" not in df.columns assert list(df.a) == [1, 3] assert list(df["query"]) == [ f"df_from-{filename}/eq-a-1", f"df_from-{filename}/eq-a-3"]
def test_append(self): import liquer.ext.lq_pandas # register pandas commands and state type filename = encode_token(os.path.dirname( inspect.getfile(self.__class__))+"/test.csv") state = evaluate(f"df_from-{filename}/append_df-{filename}") df = state.get() assert "a" in df.columns assert "b" in df.columns assert list(df.a) == [1, 3, 1, 3] assert list(df.b) == [2, 4, 2, 4]
def test_hxl2df(self, httpserver): import liquer.ext.lq_hxl # register HXL commands and state type test_hxl = open(os.path.dirname( inspect.getfile(self.__class__))+"/test_hxl.csv").read() httpserver.expect_request("/test_hxl.csv").respond_with_data(test_hxl) url = encode_token(httpserver.url_for("/test_hxl.csv")) df = evaluate(f"hxl_from-{url}/hxl2df").get() assert list(df.columns) == ["a", "b"] assert list(df.a[1:]) == ["1", "3"] assert list(df.b[1:]) == ["2", "4"]
def test_set_tags(self): import liquer.ext.lq_hxl # register HXL commands and state type import liquer.ext.lq_pandas # register pandas commands and state type filename = encode_token(os.path.dirname( inspect.getfile(self.__class__))+"/test.csv") data = evaluate(f"df_from-{filename}/set_tags-b-indicator+b-a-indicator+a/df2hxl").get() assert data.columns[0].header == "a" assert data.columns[0].display_tag == "#indicator+a" assert data.columns[1].header == "b" assert data.columns[1].display_tag == "#indicator+b" df = evaluate(f"df_from-{filename}/set_tags-b-indicator+b").get() assert list(df.a) == ["",1,3] assert list(df.b) == ["#indicator+b",2,4]
def test_workbook(self): store = MemoryStore() set_store(store) filename = encode_token( os.path.dirname(inspect.getfile(self.__class__)) + "/test.csv") evaluate_and_save(f"df_from-{filename}/test.xlsx", target_resource_directory="testdir") df = evaluate("testdir/test.xlsx/-/workbook/workbook_sheet_df").get() assert "a" in df.columns assert "b" in df.columns assert list(df.a) == [1, 3] assert list(df.b) == [2, 4]
def test_eq(self): import liquer.ext.lq_pandas # register pandas commands and state type filename = encode_token(os.path.dirname( inspect.getfile(self.__class__))+"/test.csv") state = evaluate(f"df_from-{filename}/eq-a-1") df = state.get() assert "a" in df.columns assert "b" in df.columns assert list(df.a) == [1] assert list(df.b) == [2] df = evaluate(f"df_from-{filename}/eq-a-3-b-4").get() assert list(df.a) == [3] assert list(df.b) == [4] df = evaluate(f"df_from-{filename}/eq-a-1-b-4").get() assert list(df.a) == [] assert list(df.b) == []
def test_teq(self): import liquer.ext.lq_pandas # register pandas commands and state type filename = encode_token(os.path.dirname( inspect.getfile(self.__class__))+"/test_hxl.csv") state = evaluate(f"df_from-{filename}/teq-a-1") df = state.get() assert "a" in df.columns assert "b" in df.columns assert list(df.a) == ["#indicator +num +aaa", "1"] assert list(df.b) == ["#indicator +num +bbb", "2"] df = evaluate(f"df_from-{filename}/teq-a-3-b-4").get() assert list(df.a) == ["#indicator +num +aaa", "3"] assert list(df.b) == ["#indicator +num +bbb", "4"] df = evaluate(f"df_from-{filename}/teq-a-1-b-4").get() assert list(df.a) == ["#indicator +num +aaa"] assert list(df.b) == ["#indicator +num +bbb"]
def test_append_with_cache(self): import liquer.ext.lq_pandas # register pandas commands and state type with tempfile.TemporaryDirectory() as cachepath: set_cache(FileCache(cachepath)) filename = encode_token(os.path.dirname( inspect.getfile(self.__class__))+"/test.csv") df = evaluate(f"df_from-{filename}/append_df-{filename}").get() assert "a" in df.columns assert "b" in df.columns assert list(df.a) == [1, 3, 1, 3] assert list(df.b) == [2, 4, 2, 4] df = evaluate(f"df_from-{filename}/append_df-{filename}").get() assert "a" in df.columns assert "b" in df.columns assert list(df.a) == [1, 3, 1, 3] assert list(df.b) == [2, 4, 2, 4] set_cache(None)
def gui(state): import liquer.parser as p d = state.as_dict() c = d["extended_commands"][-1] m = c["command_metadata"] ns = c['ns'] name = m['name'] title = d["attributes"].get("title", m["label"]) help_link = (d["vars"].get("server", "http://localhost") + d["vars"].get("api_path", "/q/") + f"ns-meta/help-{name}-{ns}/help.html") query_start = p.encode(d["commands"][:-1] + [[d["commands"][-1][0]]]) html = f""" <!DOCTYPE html> <html> <head> <link href="https://fonts.googleapis.com/css?family=Roboto:100,300,400,500,700,900" rel="stylesheet"> <link href="https://cdn.jsdelivr.net/npm/@mdi/[email protected]/css/materialdesignicons.min.css" rel="stylesheet"> <link href="https://cdn.jsdelivr.net/npm/[email protected]/dist/vuetify.min.css" rel="stylesheet"> <meta name="viewport" content="width=device-width, initial-scale=1, maximum-scale=1, user-scalable=no, minimal-ui"> <title>{title}</title> </head> <body> <div id="app" color="indigo"> <v-app> <v-content> <v-container class="fill-height" fluid> <v-card width="100%"> <v-toolbar color="secondary" dark> <v-toolbar-title>{title}</v-toolbar-title> <v-spacer></v-spacer> <v-btn href="{help_link}">Help</v-btn> </v-toolbar> <v-content> <v-container> """ for value, a in c["arguments"]: html += f"""<v-row><v-text-field v-model="{a['name']}" label="{a['label']}"></v-text-field></v-row> """ html += """<v-row><b>Query:</b>{{_query}}</v-row> """ html += """<v-row><b>Link:</b><a :href="_link">{{_link}}</a></v-row> """ html += """<v-row><v-spacer></v-spacer><v-btn :href="_link">Execute</v-btn></v-row> """ html += """ </v-container> <v-content> </v-card> </v-container> </v-content> </v-app> </div> <script src="https://cdn.jsdelivr.net/npm/[email protected]/dist/vue.js"></script> <script src="https://cdn.jsdelivr.net/npm/[email protected]"></script> <script src="https://cdn.jsdelivr.net/npm/[email protected]/dist/vuetify.min.js"></script> <script> new Vue({ el: '#app', data:{ """ qq = "" for value, a in c["arguments"]: if isinstance(value, list): value = "-".join(p.encode_token(value)) html += f"""{a['name']}:"{value}", """ qq += f"+'-'+this.{a['name']}" html += """ }, computed:{ _query: function(){ return '%s'%s; }, _link: function(){ return '%s'+this._query; } }, vuetify: new Vuetify(), }) </script> </body> </html> """ % (query_start, qq, d["vars"].get("server", "http://localhost") + d["vars"].get("api_path", "/q/")) return state.with_data(html).with_filename("gui.html")
def test_save_feather(self): import liquer.ext.lq_pandas # register pandas commands and state type filename = encode_token( os.path.dirname(inspect.getfile(self.__class__)) + "/test.csv") evaluate_and_save(f"df_from-{filename}/head_df-1/test.feather")
# Make it run from the examples directory import sys sys.path.append("..") from liquer import * from liquer.parser import encode, encode_token import liquer.ext.lq_pandas url = "https://raw.githubusercontent.com/orest-d/liquer/master/tests/test.csv" query = encode([["df_from", url]]) print(f"Query: {query}") print(evaluate(query).get()) query = "df_from-" + encode_token(url) print(f"Query: {query}") print(evaluate(query).get()) query = f"{query}/eq-a-3" print(f"Query: {query}") print(evaluate(query).get())
def test_save_parquet(self): filename = encode_token( os.path.dirname(inspect.getfile(self.__class__)) + "/test.csv" ) evaluate_and_save(f"df_from-{filename}/polars_df/test.parquet", target_directory=".")
app = Flask(__name__) # Registering the liquer blueprint under a given url prefix and letting LiQuer know where it is... url_prefix = '/liquer' app.register_blueprint(bp.app, url_prefix=url_prefix) set_var("api_path", url_prefix + "/q/") set_var("server", "http://localhost:5000") # Setting the cache set_cache(FileCache("../cache")) #set_cache(MemoryCache()) wfp_url = "https://data.humdata.org/dataset/4fdcd4dc-5c2f-43af-a1e4-93c9b6539a27/resource/12d7c8e3-eff9-4db0-93b7-726825c4fe9a/download/wfpvam_foodprices.csv" #wfp_url = "https://raw.githubusercontent.com/orest-d/liquer/master/tests/test.csv" wfp_query = "df_from-" + encode_token(wfp_url) @command def datemy(df, y="mp_year", m="mp_month", target="date"): df.loc[:, target] = [ "%04d-%02d-01" % (int(year), int(month)) for year, month in zip(df[y], df[m]) ] return df @command def count(df, *groupby_columns): df.loc[:, "count"] = 1 return df.groupby(