def get(self, query):
"""Shortcut to the 'web' directory in the store.
Similar to /store/data/web, except the index.html is automatically added if query is a directory.
The 'web' directory hosts web applications and visualization tools, e.g. liquer-pcv or liquer-gui.
"""
store = get_store()
try:
query = "web/" + query
if query.endswith("/"):
query += "index.html"
if store.is_dir(query):
query += "/index.html"
metadata = store.get_metadata(query)
mimetype = metadata.get("mimetype", "application/octet-stream")
b = store.get_bytes(query)
except:
mimetype = "application/json"
b = json.dumps(
dict(query=query,
message=traceback.format_exc(),
status="ERROR"))
header = "Content-Type"
body = mimetype
self.set_header(header, body)
self.write(b)
def post(self, query):
"""Set data from store. Equivalent to Store.store.
Unlike store method, which stores both data and metadata in one call,
the api/store/data POST only stores the data. The metadata needs to be set in a separate POST of api/store/metadata
either before or after the api/store/data POST.
"""
store = get_store()
try:
metadata = store.get_metadata(query)
except KeyNotFoundStoreException:
metadata = {}
try:
data = self.request.body
store.store(query, data, metadata)
response = dict(query=query, message="Data stored", status="OK")
except:
response = dict(query=query,
message=traceback.format_exc(),
status="ERROR")
mimetype = "application/json"
header = "Content-Type"
body = mimetype
self.set_header(header, body)
self.write(json.dumps(response))
def test_error_message3(self):
import traceback
import liquer.store as st
reset_command_registry()
st.set_store(st.MemoryStore())
store = st.get_store()
store.store("a/b", b"hello", {})
@command
def world(data):
return data.decode("utf-8") + " world"
@command
def expected(data, arg):
return f"{data} {arg}"
assert evaluate("a/b/-/world/expected-x").get() == "hello world x"
try:
evaluate("a/b/-/expected").get()
assert False
except Exception as e:
traceback.print_exc()
assert e.query == "a/b/-/expected"
assert e.position.offset == 6
try:
evaluate("a/b/-/expected-x-y").get()
assert False
except Exception as e:
assert e.query == "a/b/-/expected-x-y"
assert e.position.offset == 6
def store_remove(query):
store = get_store()
try:
store.remove(query)
return jsonify(
dict(query=query, message=f"Removed {query}", status="OK"))
except:
return jsonify(
dict(query=query, message=traceback.format_exc(), status="ERROR"))
def store_keys():
store = get_store()
try:
keys = store.keys()
return jsonify(
dict(query=None, message=f"Keys obtained", keys=keys, status="OK"))
except:
return jsonify(
dict(query=None, message=traceback.format_exc(), status="ERROR"))
def store_makedir(query):
store = get_store()
try:
store.makedir(query)
return jsonify(
dict(query=query, message=f"Makedir succeeded", status="OK"))
except:
return jsonify(
dict(query=query, message=traceback.format_exc(), status="ERROR"))
def store_contains(query):
store = get_store()
try:
contains = store.contains(query)
return jsonify(
dict(query=query,
message=f"Contains {query}",
contains=contains,
status="OK"))
except:
return jsonify(
dict(query=query, message=traceback.format_exc(), status="ERROR"))
def store_set_metadata(query):
store = get_store()
try:
metadata = request.get_json(force=True)
store.store_metadata(query, metadata)
return jsonify(
dict(query=query, message="Metadata stored", status="OK"))
except:
response = jsonify(
dict(query=query, message=traceback.format_exc(), status="ERROR"))
response.status = "404"
return response
def store_listdir(query):
store = get_store()
try:
listdir = store.listdir(query)
return jsonify(
dict(query=query,
message=f"Keys obtained",
listdir=listdir,
status="OK"))
except:
return jsonify(
dict(query=query, message=traceback.format_exc(), status="ERROR"))
def get(self, query):
"""Get data from store. Equivalent to Store.get_bytes.
Content type (MIME) is obtained from the metadata.
"""
store = get_store()
metadata = store.get_metadata(query)
mimetype = "application/json"
header = "Content-Type"
body = mimetype
self.set_header(header, body)
self.write(json.dumps(metadata))
def store_is_dir(query):
store = get_store()
try:
is_dir = store.is_dir(query)
return jsonify(
dict(query=query,
message=f"Is directory {query}",
is_dir=is_dir,
status="OK"))
except:
return jsonify(
dict(query=query, message=traceback.format_exc(), status="ERROR"))
def test_store(self):
import liquer.store as st
reset_command_registry()
st.set_store(st.MemoryStore())
store = st.get_store()
store.store("a/b", b"hello", {})
@command
def world(data):
return data.decode("utf-8") + " world"
assert evaluate("a/b/-/world").get() == "hello world"
def get(self, query):
store = get_store()
try:
store.remove(query)
self.write(
json.dumps(
dict(query=query, message=f"Removed {query}",
status="OK")))
except:
self.write(
json.dumps(
dict(query=query,
message=traceback.format_exc(),
status="ERROR")))
def test_clean_recipes(self):
import importlib
from liquer import evaluate
import liquer.ext.basic
import liquer.ext.meta
import liquer.store as st
from liquer.commands import reset_command_registry
reset_command_registry() # prevent double-registration
# Hack to enforce registering of the commands
importlib.reload(liquer.ext.basic)
importlib.reload(liquer.ext.meta)
@first_command
def hello(x):
return f"Hello, {x}"
substore = st.MemoryStore()
substore.store(
"recipes.yaml",
"""
RECIPES:
- hello-RECIPES/hello1.txt
subdir:
- hello-subdir/hello2.txt
""",
{},
)
store = RecipeSpecStore(substore)
store_backup = st.get_store()
st.set_store(store)
try:
assert store.get_metadata("hello1.txt")["status"] == Status.RECIPE.value
assert store.get_metadata("subdir/hello2.txt")["status"] == Status.RECIPE.value
assert store.get_bytes("hello1.txt") == b"Hello, RECIPES"
assert store.get_bytes("subdir/hello2.txt") == b"Hello, subdir"
assert store.get_metadata("hello1.txt")["status"] == Status.READY.value
assert store.get_metadata("subdir/hello2.txt")["status"] == Status.READY.value
assert evaluate("-R-meta/subdir/-/ns-meta/clean_recipes").get()["removed"] == ["subdir/hello2.txt"]
assert store.get_metadata("hello1.txt")["status"] == Status.READY.value
assert store.get_metadata("subdir/hello2.txt")["status"] == Status.RECIPE.value
finally:
st.set_store(store_backup)
def test_store_evaluate_and_save1(self):
import liquer.store as st
reset_command_registry()
st.set_store(st.MemoryStore())
store = st.get_store()
store.store("a/b", b"hello", {})
@command
def world(data):
return data.decode("utf-8") + " world"
evaluate_and_save("a/b/-/world/hello.txt",
target_resource_directory="results")
assert store.get_bytes("results/hello.txt") == b"hello world"
def get_stored_metadata(query):
"""Get metadata for a query - if it is stored in cache or a store.
"""
if not isinstance(query, Query):
query = parse(query)
if query.is_resource_query():
rq = query.resource_query()
header = rq.header
key = rq.path()
try:
return get_store().get_metadata(key)
except KeyNotFoundStoreException:
return None
else:
return get_cache().get_metadata(str(query))
def get(self, query):
store = get_store()
try:
store.makedir(query)
self.write(
json.dumps(
dict(query=query,
message=f"Makedir succeeded",
status="OK")))
except:
self.write(
json.dumps(
dict(query=query,
message=traceback.format_exc(),
status="ERROR")))
def get(self, query):
store = get_store()
try:
listdir = store.listdir(query)
self.write(
json.dumps(
dict(query=query,
message=f"Keys obtained",
listdir=listdir,
status="OK")))
except:
self.write(
json.dumps(
dict(query=query,
message=traceback.format_exc(),
status="ERROR")))
def get(self, query):
store = get_store()
try:
is_dir = store.is_dir(query)
self.write(
json.dumps(
dict(query=query,
message=f"Is directory {query}",
is_dir=is_dir,
status="OK")))
except:
self.write(
json.dumps(
dict(query=query,
message=traceback.format_exc(),
status="ERROR")))
def get(self, query):
store = get_store()
try:
keys = store.keys()
self.write(
json.dumps(
dict(query=None,
message=f"Keys obtained",
keys=keys,
status="OK")))
except:
self.write(
json.dumps(
dict(query=query,
message=traceback.format_exc(),
status="ERROR")))
def store_get(query):
"""Get data from store. Equivalent to Store.get_bytes.
Content type (MIME) is obtained from the metadata.
"""
store = get_store()
try:
metadata = store.get_metadata(query)
mimetype = metadata.get("mimetype", "application/octet-stream")
r = make_response(store.get_bytes(query))
r.headers.set("Content-Type", mimetype)
return r
except:
response = jsonify(
dict(query=query, message=traceback.format_exc(), status="ERROR"))
response.status = "404"
return response
def get(self, query):
store = get_store()
try:
contains = store.contains(query)
self.write(
json.dumps(
dict(query=query,
message=f"Contains {query}",
contains=contains,
status="OK")))
except:
self.write(
json.dumps(
dict(query=query,
message=traceback.format_exc(),
status="ERROR")))
def test_resource_link(self, tmpdir):
import liquer.store as st
reset_command_registry()
st.set_store(st.MemoryStore())
store = st.get_store()
store.store("a/b", b"hello", {})
@command
def world(data):
return data.decode("utf-8") + " world"
@first_command
def value(x):
return f"<{x}>"
assert evaluate("a/b/-/world").get() == "hello world"
assert evaluate("-R/a/b/-/world").get() == "hello world"
assert evaluate("value-~X~-R/a/b/-/world~E").get() == "<hello world>"
def __init__(
self,
key,
item_keys=None,
extension="parquet",
number_format="%04d",
batch_number=0,
store=None,
):
self.key = key
self.item_keys = item_keys or []
self.extension = extension
self.number_format = number_format
self.batch_number = batch_number
self.state_type = state_types_registry().get("dataframe")
if store is None:
store = get_store()
self.store = store
def test_meta_store(self):
import liquer.store as st
reset_command_registry()
st.set_store(st.MemoryStore())
store = st.get_store()
store.store("a/b", b"hello", {"x": 123})
@command
def world(data):
return data.decode("utf-8") + " world"
@command
def get_x(metadata):
return metadata.get("x")
assert evaluate("-R/a/b/-/world").get() == "hello world"
assert evaluate("-R-meta/a/b/-/get_x").get() == 123
print(evaluate("-R-meta/a/b").get())
assert evaluate("-R-meta/a/b").get()["key"] == "a/b"
def test_error_message1(self):
import traceback
import liquer.store as st
reset_command_registry()
st.set_store(st.MemoryStore())
store = st.get_store()
store.store("a/b", b"hello", {})
@command
def world(data):
return data.decode("utf-8") + " world"
assert evaluate("a/b/-/world").get() == "hello world"
try:
evaluate("a/b/-/undefined").get()
assert False
except Exception as e:
assert e.query == "a/b/-/undefined"
assert e.position.offset == 6
def web_store_get(query):
"""Shortcut to the 'web' directory in the store.
Similar to /store/data/web, except the index.html is automatically added if query is a directory.
The 'web' directory hosts web applications and visualization tools, e.g. liquer-pcv or liquer-gui.
"""
store = get_store()
try:
query = "web/" + query
if query.endswith("/"):
query += "index.html"
if store.is_dir(query):
query += "/index.html"
metadata = store.get_metadata(query)
mimetype = metadata.get("mimetype", "application/octet-stream")
r = make_response(store.get_bytes(query))
r.headers.set("Content-Type", mimetype)
return r
except:
return jsonify(
dict(query=query, message=traceback.format_exc(), status="ERROR"))
def store_set(query):
"""Set data from store. Equivalent to Store.store.
Unlike store method, which stores both data and metadata in one call,
the api/store/data POST only stores the data. The metadata needs to be set in a separate POST of api/store/metadata
either before or after the api/store/data POST.
"""
store = get_store()
try:
metadata = store.get_metadata(query)
except KeyNotFoundStoreException:
metadata = {}
try:
data = request.get_data()
store.store(query, data, metadata)
return jsonify(dict(query=query, message="Data stored", status="OK"))
except:
response = jsonify(
dict(query=query, message=traceback.format_exc(), status="ERROR"))
response.status = "404"
return response
def get(self, query):
"""Get data from store. Equivalent to Store.get_bytes.
Content type (MIME) is obtained from the metadata.
"""
store = get_store()
metadata = store.get_metadata(query)
try:
mimetype = metadata.get("mimetype", "application/octet-stream")
b = store.get_bytes(query)
except:
mimetype = "application/json"
b = json.dumps(
dict(query=query,
message=traceback.format_exc(),
status="ERROR"))
header = "Content-Type"
body = mimetype
self.set_header(header, body)
self.write(b)
from liquer import *
from liquer.store import web_mount, MemoryStore, get_store
import liquer.ext.meta
import liquer.ext.lq_pandas
### Create Flask app and register LiQuer blueprint
from flask import Flask
import liquer.server.blueprint as bp
app = Flask(__name__)
app.register_blueprint(bp.app, url_prefix='/liquer')
# Mount a new store into the web/hello. This will be just in memory (MemoryStore).
web_mount("hello", MemoryStore())
# Create a "file" in the web store
get_store().store("web/hello/index.html", "Hello from web store!", {})
@command
def display(txt):
if type(
txt
) == bytes: # Data from store come as bytes, so they need to be decoded
txt = txt.decode("utf-8")
return f"""<html>
<body>
<h1>Display</h1>
<pre>{txt}</pre>
</body>
</html>"""
