def test_columns_info(self):
import liquer.ext.lq_pandas # register pandas commands and state type
from liquer.state_types import encode_state_data, decode_state_data
filename = encode_token(os.path.dirname(
inspect.getfile(self.__class__))+"/test.csv")
assert evaluate(f"df_from-{filename}/df_columns").get() == ["a", "b"]
assert evaluate(
f"df_from-{filename}/columns_info").get()["columns"] == ["a", "b"]
assert evaluate(
f"df_from-{filename}/columns_info").get()["has_tags"] == False
assert evaluate(
f"df_from-{filename}/columns_info").get()["types"]["a"].startswith("int")
filename = encode_token(os.path.dirname(
inspect.getfile(self.__class__))+"/test_hxl.csv")
assert evaluate(f"df_from-{filename}/df_columns").get() == ["a", "b"]
assert evaluate(
f"df_from-{filename}/columns_info").get()["columns"] == ["a", "b"]
assert evaluate(
f"df_from-{filename}/columns_info").get()["has_tags"] == True
assert evaluate(
f"df_from-{filename}/columns_info").get()["tags"]["a"] == "#indicator +num +aaa"
assert evaluate(
f"df_from-{filename}/columns_info").get()["tags"]["b"] == "#indicator +num +bbb"
info = evaluate(f"df_from-{filename}/columns_info").get()
b, mime, tid = encode_state_data(info)
assert info == decode_state_data(b, tid)
def test_from_with_cache(self, httpserver):
import liquer.ext.lq_hxl # register HXL commands and state type
test_hxl = open(
os.path.dirname(inspect.getfile(self.__class__)) +
"/test_hxl.csv").read()
httpserver.expect_request("/test_hxl.csv").respond_with_data(test_hxl)
url = encode_token(httpserver.url_for("/test_hxl.csv"))
query = f"hxl_from-{url}"
with tempfile.TemporaryDirectory() as cachepath:
set_cache(FileCache(cachepath))
state = evaluate(query)
data = state.get()
assert data.columns[0].header == "a"
assert data.columns[0].display_tag == "#indicator+num+aaa"
assert data.columns[1].header == "b"
assert data.columns[1].display_tag == "#indicator+num+bbb"
state = evaluate(query)
data = state.get()
assert data.columns[0].header == "a"
assert data.columns[0].display_tag == "#indicator+num+aaa"
assert data.columns[1].header == "b"
assert data.columns[1].display_tag == "#indicator+num+bbb"
set_cache(None)
def test_head(self):
import liquer.ext.lq_pandas # register pandas commands and state type
filename = encode_token(os.path.dirname(
inspect.getfile(self.__class__))+"/test.csv")
df = evaluate(f"df_from-{filename}/head_df-1").get()
assert list(df.a) == [1]
assert list(df.b) == [2]
def test_tsplit(self):
import importlib
import liquer.ext.lq_pandas # register pandas commands and state type
import liquer.ext.basic
from liquer.commands import reset_command_registry
reset_command_registry() # prevent double-registration
# Hack to enforce registering of the commands
importlib.reload(liquer.ext.lq_pandas)
importlib.reload(liquer.ext.basic)
set_var("server", "http://localhost")
set_var("api_path", "/q/")
filename = encode_token(
os.path.dirname(inspect.getfile(self.__class__)) + "/test_hxl.csv")
df = evaluate(f"df_from-{filename}/tsplit_df-a").get()
assert "a" in df.columns
assert "b" not in df.columns
assert list(df.a) == ["#indicator +num +aaa", "1", "3"]
assert list(df["query"])[1:] == [
f"df_from-{filename}/teq-a-1",
f"df_from-{filename}/teq-a-3",
]
assert list(df["link"])[1:] == [
f"http://localhost/q/df_from-{filename}/teq-a-1",
f"http://localhost/q/df_from-{filename}/teq-a-3",
]
def test_work_with_parquet(self):
filename = encode_token(
os.path.dirname(inspect.getfile(self.__class__)) + "/test.csv")
with TemporaryDirectory() as tmpdir:
path = Path(tmpdir) / "test.parquet"
evaluate_and_save(f"df_from-{filename}/test.parquet",
target_directory=tmpdir)
@first_command(volatile=True, cache=False)
def execution_context():
ctx = daf.ExecutionContext()
ctx.register_parquet("a", str(path))
return ctx
@command
def process(ctx):
return ctx.sql("SELECT a, b, a+b AS c FROM a")
evaluate_and_save(f"execution_context/process/result.parquet",
target_directory=tmpdir)
df = evaluate(
f"execution_context/process/datafusion_to_pandas").get()
assert "a" in df.columns
assert "b" in df.columns
assert "c" in df.columns
assert list(df.a) == [1, 3]
assert list(df.b) == [2, 4]
assert list(df.c) == [3, 7]
def test_qsplit2(self):
import liquer.ext.lq_pandas # register pandas commands and state type
filename = encode_token(os.path.dirname(
inspect.getfile(self.__class__))+"/test.csv")
df = evaluate(f"df_from-{filename}/qsplit_df-a-b").get()
assert list(df.a) == [1, 3]
assert list(df.b) == [2, 4]
assert list(df["query"]) == [
f"df_from-{filename}/eq-a-1-b-2", f"df_from-{filename}/eq-a-3-b-4"]
def test_df2hxl(self):
import liquer.ext.lq_hxl # register HXL commands and state type
import liquer.ext.lq_pandas # register pandas commands and state type
path = encode_token(os.path.dirname(
inspect.getfile(self.__class__))+"/test_hxl.csv")
data = evaluate(f"df_from-{path}/df2hxl").get()
assert data.columns[0].header == "a"
assert data.columns[0].display_tag == "#indicator+num+aaa"
assert data.columns[1].header == "b"
assert data.columns[1].display_tag == "#indicator+num+bbb"
def test_qsplit1(self):
import liquer.ext.lq_pandas # register pandas commands and state type
filename = encode_token(os.path.dirname(
inspect.getfile(self.__class__))+"/test.csv")
df = evaluate(f"df_from-{filename}/qsplit_df-a").get()
assert "a" in df.columns
assert "b" not in df.columns
assert list(df.a) == [1, 3]
assert list(df["query"]) == [
f"df_from-{filename}/eq-a-1", f"df_from-{filename}/eq-a-3"]
def test_append(self):
import liquer.ext.lq_pandas # register pandas commands and state type
filename = encode_token(os.path.dirname(
inspect.getfile(self.__class__))+"/test.csv")
state = evaluate(f"df_from-{filename}/append_df-{filename}")
df = state.get()
assert "a" in df.columns
assert "b" in df.columns
assert list(df.a) == [1, 3, 1, 3]
assert list(df.b) == [2, 4, 2, 4]
def test_hxl2df(self, httpserver):
import liquer.ext.lq_hxl # register HXL commands and state type
test_hxl = open(os.path.dirname(
inspect.getfile(self.__class__))+"/test_hxl.csv").read()
httpserver.expect_request("/test_hxl.csv").respond_with_data(test_hxl)
url = encode_token(httpserver.url_for("/test_hxl.csv"))
df = evaluate(f"hxl_from-{url}/hxl2df").get()
assert list(df.columns) == ["a", "b"]
assert list(df.a[1:]) == ["1", "3"]
assert list(df.b[1:]) == ["2", "4"]
def test_set_tags(self):
import liquer.ext.lq_hxl # register HXL commands and state type
import liquer.ext.lq_pandas # register pandas commands and state type
filename = encode_token(os.path.dirname(
inspect.getfile(self.__class__))+"/test.csv")
data = evaluate(f"df_from-{filename}/set_tags-b-indicator+b-a-indicator+a/df2hxl").get()
assert data.columns[0].header == "a"
assert data.columns[0].display_tag == "#indicator+a"
assert data.columns[1].header == "b"
assert data.columns[1].display_tag == "#indicator+b"
df = evaluate(f"df_from-{filename}/set_tags-b-indicator+b").get()
assert list(df.a) == ["",1,3]
assert list(df.b) == ["#indicator+b",2,4]
def test_workbook(self):
store = MemoryStore()
set_store(store)
filename = encode_token(
os.path.dirname(inspect.getfile(self.__class__)) + "/test.csv")
evaluate_and_save(f"df_from-{filename}/test.xlsx",
target_resource_directory="testdir")
df = evaluate("testdir/test.xlsx/-/workbook/workbook_sheet_df").get()
assert "a" in df.columns
assert "b" in df.columns
assert list(df.a) == [1, 3]
assert list(df.b) == [2, 4]
def test_eq(self):
import liquer.ext.lq_pandas # register pandas commands and state type
filename = encode_token(os.path.dirname(
inspect.getfile(self.__class__))+"/test.csv")
state = evaluate(f"df_from-{filename}/eq-a-1")
df = state.get()
assert "a" in df.columns
assert "b" in df.columns
assert list(df.a) == [1]
assert list(df.b) == [2]
df = evaluate(f"df_from-{filename}/eq-a-3-b-4").get()
assert list(df.a) == [3]
assert list(df.b) == [4]
df = evaluate(f"df_from-{filename}/eq-a-1-b-4").get()
assert list(df.a) == []
assert list(df.b) == []
def test_teq(self):
import liquer.ext.lq_pandas # register pandas commands and state type
filename = encode_token(os.path.dirname(
inspect.getfile(self.__class__))+"/test_hxl.csv")
state = evaluate(f"df_from-{filename}/teq-a-1")
df = state.get()
assert "a" in df.columns
assert "b" in df.columns
assert list(df.a) == ["#indicator +num +aaa", "1"]
assert list(df.b) == ["#indicator +num +bbb", "2"]
df = evaluate(f"df_from-{filename}/teq-a-3-b-4").get()
assert list(df.a) == ["#indicator +num +aaa", "3"]
assert list(df.b) == ["#indicator +num +bbb", "4"]
df = evaluate(f"df_from-{filename}/teq-a-1-b-4").get()
assert list(df.a) == ["#indicator +num +aaa"]
assert list(df.b) == ["#indicator +num +bbb"]
def test_append_with_cache(self):
import liquer.ext.lq_pandas # register pandas commands and state type
with tempfile.TemporaryDirectory() as cachepath:
set_cache(FileCache(cachepath))
filename = encode_token(os.path.dirname(
inspect.getfile(self.__class__))+"/test.csv")
df = evaluate(f"df_from-{filename}/append_df-{filename}").get()
assert "a" in df.columns
assert "b" in df.columns
assert list(df.a) == [1, 3, 1, 3]
assert list(df.b) == [2, 4, 2, 4]
df = evaluate(f"df_from-{filename}/append_df-{filename}").get()
assert "a" in df.columns
assert "b" in df.columns
assert list(df.a) == [1, 3, 1, 3]
assert list(df.b) == [2, 4, 2, 4]
set_cache(None)
def gui(state):
import liquer.parser as p
d = state.as_dict()
c = d["extended_commands"][-1]
m = c["command_metadata"]
ns = c['ns']
name = m['name']
title = d["attributes"].get("title", m["label"])
help_link = (d["vars"].get("server", "http://localhost") +
d["vars"].get("api_path", "/q/") +
f"ns-meta/help-{name}-{ns}/help.html")
query_start = p.encode(d["commands"][:-1] + [[d["commands"][-1][0]]])
html = f"""
<!DOCTYPE html>
<html>
<head>
<link href="https://fonts.googleapis.com/css?family=Roboto:100,300,400,500,700,900" rel="stylesheet">
<link href="https://cdn.jsdelivr.net/npm/@mdi/[email protected]/css/materialdesignicons.min.css" rel="stylesheet">
<link href="https://cdn.jsdelivr.net/npm/[email protected]/dist/vuetify.min.css" rel="stylesheet">
<meta name="viewport" content="width=device-width, initial-scale=1, maximum-scale=1, user-scalable=no, minimal-ui">
<title>{title}</title>
</head>
<body>
<div id="app" color="indigo">
<v-app>
<v-content>
<v-container class="fill-height" fluid>
<v-card width="100%">
<v-toolbar color="secondary" dark>
<v-toolbar-title>{title}</v-toolbar-title>
<v-spacer></v-spacer>
<v-btn href="{help_link}">Help</v-btn>
</v-toolbar>
<v-content>
<v-container>
"""
for value, a in c["arguments"]:
html += f"""<v-row><v-text-field v-model="{a['name']}" label="{a['label']}"></v-text-field></v-row>
"""
html += """<v-row><b>Query:</b>{{_query}}</v-row>
"""
html += """<v-row><b>Link:</b><a :href="_link">{{_link}}</a></v-row>
"""
html += """<v-row><v-spacer></v-spacer><v-btn :href="_link">Execute</v-btn></v-row>
"""
html += """ </v-container>
<v-content>
</v-card>
</v-container>
</v-content>
</v-app>
</div>
<script src="https://cdn.jsdelivr.net/npm/[email protected]/dist/vue.js"></script>
<script src="https://cdn.jsdelivr.net/npm/[email protected]"></script>
<script src="https://cdn.jsdelivr.net/npm/[email protected]/dist/vuetify.min.js"></script>
<script>
new Vue({
el: '#app',
data:{
"""
qq = ""
for value, a in c["arguments"]:
if isinstance(value, list):
value = "-".join(p.encode_token(value))
html += f"""{a['name']}:"{value}",
"""
qq += f"+'-'+this.{a['name']}"
html += """
},
computed:{
_query: function(){
return '%s'%s;
},
_link: function(){
return '%s'+this._query;
}
},
vuetify: new Vuetify(),
})
</script>
</body>
</html>
""" % (query_start, qq, d["vars"].get("server", "http://localhost") +
d["vars"].get("api_path", "/q/"))
return state.with_data(html).with_filename("gui.html")
def test_save_feather(self):
import liquer.ext.lq_pandas # register pandas commands and state type
filename = encode_token(
os.path.dirname(inspect.getfile(self.__class__)) + "/test.csv")
evaluate_and_save(f"df_from-{filename}/head_df-1/test.feather")
# Make it run from the examples directory
import sys
sys.path.append("..")
from liquer import *
from liquer.parser import encode, encode_token
import liquer.ext.lq_pandas
url = "https://raw.githubusercontent.com/orest-d/liquer/master/tests/test.csv"
query = encode([["df_from", url]])
print(f"Query: {query}")
print(evaluate(query).get())
query = "df_from-" + encode_token(url)
print(f"Query: {query}")
print(evaluate(query).get())
query = f"{query}/eq-a-3"
print(f"Query: {query}")
print(evaluate(query).get())
def test_save_parquet(self):
filename = encode_token(
os.path.dirname(inspect.getfile(self.__class__)) + "/test.csv"
)
evaluate_and_save(f"df_from-{filename}/polars_df/test.parquet", target_directory=".")
app = Flask(__name__)
# Registering the liquer blueprint under a given url prefix and letting LiQuer know where it is...
url_prefix = '/liquer'
app.register_blueprint(bp.app, url_prefix=url_prefix)
set_var("api_path", url_prefix + "/q/")
set_var("server", "http://localhost:5000")
# Setting the cache
set_cache(FileCache("../cache"))
#set_cache(MemoryCache())
wfp_url = "https://data.humdata.org/dataset/4fdcd4dc-5c2f-43af-a1e4-93c9b6539a27/resource/12d7c8e3-eff9-4db0-93b7-726825c4fe9a/download/wfpvam_foodprices.csv"
#wfp_url = "https://raw.githubusercontent.com/orest-d/liquer/master/tests/test.csv"
wfp_query = "df_from-" + encode_token(wfp_url)
@command
def datemy(df, y="mp_year", m="mp_month", target="date"):
df.loc[:, target] = [
"%04d-%02d-01" % (int(year), int(month))
for year, month in zip(df[y], df[m])
]
return df
@command
def count(df, *groupby_columns):
df.loc[:, "count"] = 1
return df.groupby(
