def test_invalidate_cascade(flowmachine_connect): """ Test that invalidation does not cascade if cascade=False. """ dl1 = daily_location("2016-01-01") dl1.store().result() hl1 = ModalLocation(daily_location("2016-01-01"), daily_location("2016-01-02")) hl1.store().result() hl2 = ModalLocation(daily_location("2016-01-03"), daily_location("2016-01-04")) flow = Flows(hl1, hl2) flow.store().result() assert dl1.is_stored assert hl1.is_stored assert flow.is_stored dl1.invalidate_db_cache(cascade=False) assert not dl1.is_stored assert hl1.is_stored assert flow.is_stored assert not cache_table_exists(get_db(), dl1.query_id) assert cache_table_exists(get_db(), hl1.query_id) has_deps = bool(get_db().fetch("SELECT * FROM cache.dependencies")) assert has_deps
def test_invalidate_cache_midchain(flowmachine_connect): """ Test that invalidating a query in the middle of a chain drops the top of the chain and this link, but not the bottom. """ dl1 = daily_location("2016-01-01") dl1.store().result() hl1 = ModalLocation(daily_location("2016-01-01"), daily_location("2016-01-02")) hl1.store().result() hl2 = ModalLocation(daily_location("2016-01-03"), daily_location("2016-01-04")) flow = Flows(hl1, hl2) flow.store().result() assert dl1.is_stored assert hl1.is_stored assert flow.is_stored hl1.invalidate_db_cache() assert dl1.is_stored assert not hl1.is_stored assert not flow.is_stored assert cache_table_exists(get_db(), dl1.query_id) assert not cache_table_exists(get_db(), hl1.query_id) assert not cache_table_exists(get_db(), flow.query_id) has_deps = bool(get_db().fetch("SELECT * FROM cache.dependencies")) assert has_deps # Daily location deps should remain
def test_cache_table_exists(flowmachine_connect): """ Test that cache_table_exists reports accurately. """ assert not cache_table_exists(flowmachine_connect, "NONEXISTENT_CACHE_ID") assert cache_table_exists( flowmachine_connect, daily_location("2016-01-01").store().result().query_id)
def test_table_records_removed(flowmachine_connect): """Test that removing a query from cache removes any Tables in cache that pointed to it.""" dl = daily_location("2016-01-01") dl.store().result() assert dl.is_stored table = dl.get_table() assert cache_table_exists(get_db(), table.query_id) dl.invalidate_db_cache() assert not cache_table_exists(get_db(), table.query_id)
def test_do_cache_simple(flowmachine_connect): """ Test that a simple object can be cached. """ dl1 = daily_location("2016-01-01") write_cache_metadata(get_db(), dl1) assert cache_table_exists(get_db(), dl1.query_id)
def test_do_cache_simple(flowmachine_connect): """ Test that a simple object can be cached. """ dl1 = daily_location("2016-01-01") dl1._db_store_cache_metadata() assert cache_table_exists(flowmachine_connect, dl1.md5)
def test_store_cache_simple(flowmachine_connect): """ Test that storing a simple object also caches it. """ dl1 = daily_location("2016-01-01") dl1.store().result() # Should be stored assert dl1.is_stored assert cache_table_exists(get_db(), dl1.query_id)
def test_do_cache_multi(flowmachine_connect): """ Test that a query containing subqueries can be cached. """ hl1 = ModalLocation(daily_location("2016-01-01"), daily_location("2016-01-02")) write_cache_metadata(get_db(), hl1) assert cache_table_exists(get_db(), hl1.query_id)
def test_do_cache_multi(flowmachine_connect): """ Test that a query containing subqueries can be cached. """ hl1 = ModalLocation(daily_location("2016-01-01"), daily_location("2016-01-02")) hl1._db_store_cache_metadata() assert cache_table_exists(flowmachine_connect, hl1.md5)
def test_store_cache_multi(flowmachine_connect): """ Test that storing a query containing subqueries also caches it. """ hl1 = ModalLocation(daily_location("2016-01-01"), daily_location("2016-01-02")) hl1.store().result() # Should be stored assert hl1.is_stored assert cache_table_exists(get_db(), hl1.query_id)
def test_invalidate_cache_multi(flowmachine_connect): """ Test that invalidating a simple query that is part of a bigger one drops both tables, cleans up dependencies and removes both from cache. """ dl1 = daily_location("2016-01-01") dl1.store().result() hl1 = ModalLocation(daily_location("2016-01-01"), daily_location("2016-01-02")) hl1.store().result() assert dl1.is_stored assert hl1.is_stored dl1.invalidate_db_cache() assert not dl1.is_stored assert not hl1.is_stored assert not cache_table_exists(get_db(), dl1.query_id) assert not cache_table_exists(get_db(), hl1.query_id) has_deps = bool(get_db().fetch("SELECT * FROM cache.dependencies")) assert has_deps # the remaining dependencies are due to underlying Table objects
def test_invalidate_cache_simple(flowmachine_connect): """ Test that invalidating a simple object drops the table, and removes it from cache. """ dl1 = daily_location("2016-01-01") dl1.store().result() assert dl1.is_stored dl1.invalidate_db_cache() assert not dl1.is_stored assert not cache_table_exists(get_db(), dl1.query_id)
def test_do_cache_nested(flowmachine_connect): """ Test that a query containing nested subqueries can be cached. """ hl1 = ModalLocation(daily_location("2016-01-01"), daily_location("2016-01-02")) hl2 = ModalLocation(daily_location("2016-01-03"), daily_location("2016-01-04")) flow = Flows(hl1, hl2) write_cache_metadata(get_db(), flow) assert cache_table_exists(get_db(), flow.query_id)
def __call__(self, value) -> Union[None, str]: from flowmachine.core.server.query_schemas import FlowmachineQuerySchema if (value is not None) and (value is not missing): try: (FlowmachineQuerySchema().load( QueryInfoLookup(get_redis()).get_query_params( value))._flowmachine_query_obj) except UnkownQueryIdError: if not cache_table_exists(get_db(), value): raise ValidationError("Must be None or a valid query id.") return value
def test_store_cache_nested(flowmachine_connect): """ Test that storing a query with nested subqueries also caches it. """ hl1 = ModalLocation(daily_location("2016-01-01"), daily_location("2016-01-02")) hl2 = ModalLocation(daily_location("2016-01-03"), daily_location("2016-01-04")) flow = Flows(hl1, hl2) flow.store().result() # Should be stored assert flow.is_stored assert cache_table_exists(get_db(), flow.query_id)
def poll(self): """ Return the status of a submitted query. Returns ------- str """ query_id = self._get_query_id_from_redis() logger.debug( f"Getting status for query {query_id} of kind {self.query_kind}") if self.redis_interface.has_lock(query_id): status = "running" else: if cache_table_exists(Query.connection, query_id): status = "done" else: status = "awol" return status