def test_invalidate_cascade(flowmachine_connect):
"""
Test that invalidation does not cascade if cascade=False.
"""
dl1 = daily_location("2016-01-01")
dl1.store().result()
hl1 = ModalLocation(daily_location("2016-01-01"),
daily_location("2016-01-02"))
hl1.store().result()
hl2 = ModalLocation(daily_location("2016-01-03"),
daily_location("2016-01-04"))
flow = Flows(hl1, hl2)
flow.store().result()
assert dl1.is_stored
assert hl1.is_stored
assert flow.is_stored
dl1.invalidate_db_cache(cascade=False)
assert not dl1.is_stored
assert hl1.is_stored
assert flow.is_stored
assert not cache_table_exists(get_db(), dl1.query_id)
assert cache_table_exists(get_db(), hl1.query_id)
has_deps = bool(get_db().fetch("SELECT * FROM cache.dependencies"))
assert has_deps
def test_invalidate_cache_midchain(flowmachine_connect):
"""
Test that invalidating a query in the middle of a chain drops the
top of the chain and this link, but not the bottom.
"""
dl1 = daily_location("2016-01-01")
dl1.store().result()
hl1 = ModalLocation(daily_location("2016-01-01"),
daily_location("2016-01-02"))
hl1.store().result()
hl2 = ModalLocation(daily_location("2016-01-03"),
daily_location("2016-01-04"))
flow = Flows(hl1, hl2)
flow.store().result()
assert dl1.is_stored
assert hl1.is_stored
assert flow.is_stored
hl1.invalidate_db_cache()
assert dl1.is_stored
assert not hl1.is_stored
assert not flow.is_stored
assert cache_table_exists(get_db(), dl1.query_id)
assert not cache_table_exists(get_db(), hl1.query_id)
assert not cache_table_exists(get_db(), flow.query_id)
has_deps = bool(get_db().fetch("SELECT * FROM cache.dependencies"))
assert has_deps # Daily location deps should remain
def test_cache_table_exists(flowmachine_connect):
"""
Test that cache_table_exists reports accurately.
"""
assert not cache_table_exists(flowmachine_connect, "NONEXISTENT_CACHE_ID")
assert cache_table_exists(
flowmachine_connect,
daily_location("2016-01-01").store().result().query_id)
def test_table_records_removed(flowmachine_connect):
"""Test that removing a query from cache removes any Tables in cache that pointed to it."""
dl = daily_location("2016-01-01")
dl.store().result()
assert dl.is_stored
table = dl.get_table()
assert cache_table_exists(get_db(), table.query_id)
dl.invalidate_db_cache()
assert not cache_table_exists(get_db(), table.query_id)
def test_do_cache_simple(flowmachine_connect):
"""
Test that a simple object can be cached.
"""
dl1 = daily_location("2016-01-01")
write_cache_metadata(get_db(), dl1)
assert cache_table_exists(get_db(), dl1.query_id)
def test_do_cache_simple(flowmachine_connect):
"""
Test that a simple object can be cached.
"""
dl1 = daily_location("2016-01-01")
dl1._db_store_cache_metadata()
assert cache_table_exists(flowmachine_connect, dl1.md5)
def test_store_cache_simple(flowmachine_connect):
"""
Test that storing a simple object also caches it.
"""
dl1 = daily_location("2016-01-01")
dl1.store().result()
# Should be stored
assert dl1.is_stored
assert cache_table_exists(get_db(), dl1.query_id)
def test_do_cache_multi(flowmachine_connect):
"""
Test that a query containing subqueries can be cached.
"""
hl1 = ModalLocation(daily_location("2016-01-01"),
daily_location("2016-01-02"))
write_cache_metadata(get_db(), hl1)
assert cache_table_exists(get_db(), hl1.query_id)
def test_do_cache_multi(flowmachine_connect):
"""
Test that a query containing subqueries can be cached.
"""
hl1 = ModalLocation(daily_location("2016-01-01"),
daily_location("2016-01-02"))
hl1._db_store_cache_metadata()
assert cache_table_exists(flowmachine_connect, hl1.md5)
def test_store_cache_multi(flowmachine_connect):
"""
Test that storing a query containing subqueries also caches it.
"""
hl1 = ModalLocation(daily_location("2016-01-01"),
daily_location("2016-01-02"))
hl1.store().result()
# Should be stored
assert hl1.is_stored
assert cache_table_exists(get_db(), hl1.query_id)
def test_invalidate_cache_multi(flowmachine_connect):
"""
Test that invalidating a simple query that is part of
a bigger one drops both tables, cleans up dependencies
and removes both from cache.
"""
dl1 = daily_location("2016-01-01")
dl1.store().result()
hl1 = ModalLocation(daily_location("2016-01-01"),
daily_location("2016-01-02"))
hl1.store().result()
assert dl1.is_stored
assert hl1.is_stored
dl1.invalidate_db_cache()
assert not dl1.is_stored
assert not hl1.is_stored
assert not cache_table_exists(get_db(), dl1.query_id)
assert not cache_table_exists(get_db(), hl1.query_id)
has_deps = bool(get_db().fetch("SELECT * FROM cache.dependencies"))
assert has_deps # the remaining dependencies are due to underlying Table objects
def test_invalidate_cache_simple(flowmachine_connect):
"""
Test that invalidating a simple object drops the table,
and removes it from cache.
"""
dl1 = daily_location("2016-01-01")
dl1.store().result()
assert dl1.is_stored
dl1.invalidate_db_cache()
assert not dl1.is_stored
assert not cache_table_exists(get_db(), dl1.query_id)
def test_do_cache_nested(flowmachine_connect):
"""
Test that a query containing nested subqueries can be cached.
"""
hl1 = ModalLocation(daily_location("2016-01-01"),
daily_location("2016-01-02"))
hl2 = ModalLocation(daily_location("2016-01-03"),
daily_location("2016-01-04"))
flow = Flows(hl1, hl2)
write_cache_metadata(get_db(), flow)
assert cache_table_exists(get_db(), flow.query_id)
def __call__(self, value) -> Union[None, str]:
from flowmachine.core.server.query_schemas import FlowmachineQuerySchema
if (value is not None) and (value is not missing):
try:
(FlowmachineQuerySchema().load(
QueryInfoLookup(get_redis()).get_query_params(
value))._flowmachine_query_obj)
except UnkownQueryIdError:
if not cache_table_exists(get_db(), value):
raise ValidationError("Must be None or a valid query id.")
return value
def test_store_cache_nested(flowmachine_connect):
"""
Test that storing a query with nested subqueries also caches it.
"""
hl1 = ModalLocation(daily_location("2016-01-01"),
daily_location("2016-01-02"))
hl2 = ModalLocation(daily_location("2016-01-03"),
daily_location("2016-01-04"))
flow = Flows(hl1, hl2)
flow.store().result()
# Should be stored
assert flow.is_stored
assert cache_table_exists(get_db(), flow.query_id)
def poll(self):
"""
Return the status of a submitted query.
Returns
-------
str
"""
query_id = self._get_query_id_from_redis()
logger.debug(
f"Getting status for query {query_id} of kind {self.query_kind}")
if self.redis_interface.has_lock(query_id):
status = "running"
else:
if cache_table_exists(Query.connection, query_id):
status = "done"
else:
status = "awol"
return status
