def cached_part(query, cache=None):
"""Get cached part of the query.
Use either supplied cache object or global cache object (default).
In the process, query is into two parts: the beginning of the query
and the remainder. Function tries to find longest possible beginning of the query
which is cached, then returns the cached state and the remainder of the query.
(query == state.query + "/" + remainder)
"""
if cache is None:
cache = get_cache()
if isinstance(
cache, NoCache
): # Just an optimization - to avoid looping over all query splits
return State(), encode(decode(query))
for key, remainder in all_splits(query):
if key == "":
return State(), remainder
if cache.contains(key):
state = cache.get(key)
if state is None:
continue
return state, remainder
# Should never get here, but this is a sensible default:
return State(), encode(decode(query))
def state_factory():
global url_prefix
state = State()
state.vars["server"]="http://localhost:5000"
state.vars["api_path"]=url_prefix+"/q/"
return state
def test_filecache(self):
state = State().with_data(123)
state.query = "abc"
with tempfile.TemporaryDirectory() as cachepath:
cache = FileCache(cachepath)
cache.remove("abc") # Try to remove key from empty cache
assert not cache.contains("abc")
assert list(cache.keys()) == []
cache.store(state)
assert cache.contains("abc")
assert list(cache.keys()) == ["abc"]
assert cache.get("abc").get() == 123
assert cache.get_metadata("abc")["query"] == "abc"
assert cache.store_metadata(dict(query="abc", mymetafield="Hello"))
assert cache.get_metadata("abc")["mymetafield"] == "Hello"
assert not cache.contains("xyz")
assert cache.get("xyz") == None
assert not cache.contains("xxx")
assert cache.store_metadata(dict(query="xxx", mymetafield="Hello"))
assert cache.contains("xxx")
assert sorted(cache.keys()) == ["abc", "xxx"]
cache.clean()
assert not cache.contains("abc")
assert list(cache.keys()) == []
assert cache.get("abc") == None
def test_storecache(self):
from liquer.store import MemoryStore
state = State().with_data(123)
state.query = "abc"
for cache in [
StoreCache(MemoryStore(), path=""),
StoreCache(MemoryStore(), path="xx"),
StoreCache(MemoryStore(), path="xx", flat=True),
]:
cache.remove("abc") # Try to remove key from empty cache
assert not cache.contains("abc")
assert list(cache.keys()) == []
cache.store(state)
assert cache.contains("abc")
assert list(cache.keys()) == ["abc"]
assert cache.get("abc").get() == 123
assert cache.get_metadata("abc")["query"] == "abc"
assert cache.store_metadata(dict(query="abc", mymetafield="Hello"))
assert cache.get_metadata("abc")["mymetafield"] == "Hello"
assert not cache.contains("xyz")
assert cache.get("xyz") == None
assert not cache.contains("xxx")
assert cache.store_metadata(dict(query="xxx", mymetafield="Hello"))
assert cache.contains("xxx")
assert sorted(cache.keys()) == ["abc", "xxx"]
cache.clean()
assert not cache.contains("abc")
assert list(cache.keys()) == []
assert cache.get("abc") == None
def test_sqlite(self):
state = State().with_data(123)
state.query = "abc"
cache = SQLCache.from_sqlite()
cache.remove("abc") # Try to remove key from empty cache
assert not cache.contains("abc")
assert list(cache.keys()) == []
cache.store(state)
assert cache.contains("abc")
assert list(cache.keys()) == ["abc"]
assert cache.get("abc").get() == 123
assert cache.get_metadata("abc")["query"] == "abc"
assert cache.store_metadata(dict(query="abc", mymetafield="Hello"))
assert cache.get_metadata("abc")["mymetafield"] == "Hello"
assert not cache.contains("xyz")
assert cache.get("xyz") == None
assert not cache.contains("xxx")
assert cache.store_metadata(dict(query="xxx", mymetafield="Hello"))
assert cache.contains("xxx")
assert sorted(cache.keys()) == ["abc", "xxx"]
cache.clean()
assert not cache.contains("abc")
assert list(cache.keys()) == []
assert cache.get("abc") == None
def get(self, key):
c = self.connection.cursor()
c.execute(
f"""
SELECT
metadata,
state_data
FROM {self.table}
WHERE query=?
""",
[key],
)
try:
metadata, data = c.fetchone()
metadata = json.loads(metadata)
if metadata.get("status") != "ready":
return None
except:
return None
try:
state = State()
state = state.from_dict(metadata)
t = state_types_registry().get(state.type_identifier)
state.data = t.from_bytes(self.decode(data))
return state
except:
logging.exception(f"Cache failed to recover {key}")
return None
def get(self, key):
state_path = self.to_path(key) + ".json"
if os.path.exists(state_path):
state = State()
state = state.from_state(json.loads(open(state_path).read()))
else:
return None
path = self.to_path(key)
if os.path.exists(path):
return state.from_bytes(open(path, "rb").read())
def get(self, key):
state_path = self.to_path(key) + ".json"
if os.path.exists(state_path):
state = State()
state = state.from_state(json.loads(open(state_path).read()))
else:
return None
csv_path = self.to_path(key) + ".csv"
if os.path.exists(csv_path):
df = pd.read_csv(csv_path)
return state.with_df(df)
def test_memory(self):
state = State().with_data(123)
state.query = "abc"
cache = MemoryCache()
assert not cache.contains("abc")
cache.store(state)
assert cache.contains("abc")
assert cache.get("abc").get() == 123
assert not cache.contains("xyz")
assert cache.get("xyz") == None
def test_nocache(self):
state = State().with_data(123)
state.query = "abc"
cache = NoCache()
assert not cache.contains("abc")
cache.store(state)
assert not cache.contains("abc")
assert cache.get("abc") == None
assert not cache.contains("xyz")
assert cache.get("xyz") == None
def test_filecache(self):
state = State().with_data(123)
state.query = "abc"
with tempfile.TemporaryDirectory() as cachepath:
cache = FileCache(cachepath)
assert not cache.contains("abc")
cache.store(state)
assert cache.contains("abc")
assert cache.get("abc").get() == 123
assert not cache.contains("xyz")
assert cache.get("xyz") == None
def contains(self, key):
state_path = self.to_path(key)
if os.path.exists(state_path):
state = State()
state = state.from_dict(json.loads(open(state_path).read()))
else:
return False
t = state_types_registry().get(state.type_identifier)
path = self.to_path(key,
prefix="data_",
extension=t.default_extension())
if os.path.exists(path):
return True
else:
return False
def test_cached_part_nocache(self):
cache = NoCache()
state, remainder = cached_part("abc", cache)
assert remainder == "abc"
assert state.get() == None
state = State().with_data(123)
state.query = "abc"
cache.store(state)
state, remainder = cached_part("/abc/", cache)
assert remainder == "abc"
assert state.get() == None
state, remainder = cached_part("/abc/def/", cache)
assert remainder == "abc/def"
assert state.get() == None
def test_sqlite_store_metadata_disabled(self):
state = State().with_data(123)
state.query = "abc"
cache = SQLCache.from_sqlite()
cache.remove("abc") # Try to remove key from empty cache
cache.store_metadata_enabled = False
assert not cache.contains("abc")
assert list(cache.keys()) == []
cache.store(state)
assert cache.contains("abc")
assert list(cache.keys()) == ["abc"]
assert cache.get("abc").get() == 123
assert cache.get_metadata("abc")["query"] == "abc"
assert not cache.store_metadata(dict(query="abc", mymetafield="Hello"))
assert cache.get_metadata("abc").get("mymetafield") is None
def test_sqlite_string(self):
state = State().with_data(123)
state.query = "abc"
cache = SQLStringCache.from_sqlite()
cache.remove("abc") # Try to remove key from empty cache
assert not cache.contains("abc")
cache.store(state)
assert cache.contains("abc")
assert cache.get("abc").get() == 123
assert not cache.contains("xyz")
assert cache.get("xyz") == None
cache.clean()
assert not cache.contains("abc")
assert cache.get("abc") == None
def get(self, key):
state_path = self.to_path(key)
if os.path.exists(state_path):
state = State()
state = state.from_dict(json.loads(open(state_path).read()))
else:
return None
t = state_types_registry().get(state.type_identifier)
path = self.to_path(key,
prefix="data_",
extension=t.default_extension())
if os.path.exists(path):
try:
state.data = t.from_bytes(open(path, "rb").read())
return state
except:
logging.exception(f"Cache failed to recover {key}")
return None
def test_xor_file_cache(self):
state = State().with_data(123)
state.query = "abc"
with tempfile.TemporaryDirectory() as cachepath:
cache = XORFileCache(cachepath, b"**")
cache.remove("abc") # Try to remove key from empty cache
assert not cache.contains("abc")
cache.store(state)
assert cache.contains("abc")
assert cache.get("abc").get() == 123
assert not cache.contains("xyz")
assert cache.get("xyz") == None
cache.clean()
assert not cache.contains("abc")
assert cache.get("abc") == None
def test_nocache(self):
state = State().with_data(123)
state.query = "abc"
cache = NoCache()
cache.remove("abc") # Try to remove key from empty cache
assert not cache.contains("abc")
cache.store(state)
assert not cache.contains("abc")
assert cache.get("abc") == None
assert not cache.contains("xyz")
assert cache.get("xyz") == None
cache.clean()
assert not cache.contains("abc")
assert cache.get("abc") == None
def test_first_command(self):
reset_command_registry()
@first_command
def test_callable(a: int, b=123):
return a + b
result = command_registry().executables["root"]["test_callable"](
State(), "1")
assert result.get() == 124
def test_nonstate_command(self):
reset_command_registry()
@command
def nonstatecommand(x: int): # has state as a first argument
assert x == 1
return 123 + x
assert command_registry().evaluate_command(
State().with_data(1), ["nonstatecommand"]).get() == 124
def test_state_command(self):
reset_command_registry()
@command
def statecommand(state): # has state as a first argument
assert isinstance(state, State)
return 123 + state.get()
assert command_registry().evaluate_command(
State().with_data(1), ["statecommand"]).get() == 124
def test_command(self):
reset_command_registry()
@command
def test_callable(state, a: int, b=123): # has state as a first argument
return a + b
ns, _command, command_metadata = command_registry().resolve_command(State(),"test_callable")
m = Dependencies()
m.add_command_dependency(ns, command_metadata)
assert "ns-root/test_callable" in m.as_dict()["commands"]
def test_fernet_file_cache(self):
from cryptography.fernet import Fernet
fernet_key = Fernet.generate_key()
state = State().with_data(123)
state.query = "abc"
with tempfile.TemporaryDirectory() as cachepath:
cache = FernetFileCache(cachepath, fernet_key)
cache.remove("abc") # Try to remove key from empty cache
assert not cache.contains("abc")
cache.store(state)
assert cache.contains("abc")
assert cache.get("abc").get() == 123
assert not cache.contains("xyz")
assert cache.get("xyz") == None
cache.clean()
assert not cache.contains("abc")
assert cache.get("abc") == None
def get(self, key):
metadata = self.get_metadata(key)
if metadata is None:
print(f"(FileCache) Metadata missing: {key}")
return None
if metadata.get("status") != "ready":
print(f"(FileCache) Not ready {key}; ", metadata.get("status"))
return None
state = State()
state.metadata = metadata
t = state_types_registry().get(metadata["type_identifier"])
path = self.to_path(key, prefix="data_", extension=t.default_extension())
if os.path.exists(path):
try:
state.data = t.from_bytes(self.decode(open(path, "rb").read()))
return state
except:
traceback.print_exc()
logging.exception(f"Cache failed to recover {key}")
return None
def contains(self, key):
state_path = self.to_path(key)
if os.path.exists(state_path):
state = State()
metadata = self._load_metadata(state_path)
if metadata is None:
return False
else:
return metadata.get("query") == key
else:
return False
return True
def test_evaluate_action_with_arguments(self):
reset_command_registry()
@command
def test_callable(state, a: int, b=123): # has state as a first argument
return a + b
context = get_context()
action = ActionRequest.from_arguments("test_callable", "1")
result = context.evaluate_action(State(), action, extra_parameters=[234])
assert result.is_volatile()
assert result.get() == 235
assert result.metadata["commands"][-1] == ["test_callable", "1"]
def test_evaluate_command_with_attributes(self):
reset_command_registry()
@command(ABC="def")
def test_callable(state, a: int, b=123): # has state as a first argument
return a + b
context = get_context()
action = ActionRequest.from_arguments("test_callable", "1")
result = context.evaluate_action(State(), action)
assert result.get() == 124
assert result.metadata["commands"][-1] == ["test_callable", "1"]
assert result.metadata["attributes"]["ABC"] == "def"
def test_evaluate_command(self):
reset_command_registry()
@command
def test_callable(state,
a: int,
b=123): # has state as a first argument
return a + b
cmd = ["test_callable", "1"]
result = command_registry().evaluate_command(State(), cmd)
assert result.get() == 124
assert result.commands[-1] == cmd
def test_state_command(self):
reset_command_registry()
@command
def statecommand(state): # has state as a first argument
assert isinstance(state, State)
return 123 + state.get()
assert (
get_context()
.evaluate_action(State().with_data(1), ActionRequest("statecommand"))
.get()
== 124
)
def get(self, key):
print(f"GET {key}")
metadata = self.get_metadata(key)
print(f" METADATA {metadata}")
if metadata is None:
print(f"(StoreCache) Metadata missing: {key}")
return None
if metadata.get("status") != "ready":
print(f"(StoreCache) Not ready {key}; ", metadata.get("status"))
return None
state = State()
state.metadata = metadata
t = state_types_registry().get(metadata["type_identifier"])
path = self.to_path(key)
if self.storage.contains(path):
try:
state.data = t.from_bytes(self.decode(self.storage.get_bytes(path)))
return state
except:
traceback.print_exc()
logging.exception(f"Cache failed to recover {key}")
return None
