def enable_time_to_live(self, ttl_config: StateTtlConfig):
self._ttl_config = ttl_config
if ttl_config.get_state_visibility(
) == StateTtlConfig.StateVisibility.NeverReturnExpired:
self._cache_type = SynchronousKvRuntimeState.CacheType.DISABLE_CACHE
elif ttl_config.get_update_type(
) == StateTtlConfig.UpdateType.OnReadAndWrite:
self._cache_type = SynchronousKvRuntimeState.CacheType.ENABLE_WRITE_CACHE
if self._cache_type != SynchronousKvRuntimeState.CacheType.ENABLE_READ_WRITE_CACHE:
# disable read cache
self._remote_state_backend._state_handler._state_cache._cache._max_entries = 0
def test_state_ttl_config_proto(self):
from pyflink.datastream.state import StateTtlConfig
from pyflink.common.time import Time
state_ttl_config = StateTtlConfig \
.new_builder(Time.milliseconds(1000)) \
.set_update_type(StateTtlConfig.UpdateType.OnCreateAndWrite) \
.set_state_visibility(StateTtlConfig.StateVisibility.NeverReturnExpired) \
.cleanup_full_snapshot() \
.cleanup_incrementally(10, True) \
.cleanup_in_rocksdb_compact_filter(1000) \
.build()
state_ttl_config_proto = state_ttl_config._to_proto()
state_ttl_config = StateTtlConfig._from_proto(state_ttl_config_proto)
self.assertEqual(state_ttl_config.get_ttl(), Time.milliseconds(1000))
self.assertEqual(
state_ttl_config.get_update_type(), StateTtlConfig.UpdateType.OnCreateAndWrite)
self.assertEqual(
state_ttl_config.get_state_visibility(),
StateTtlConfig.StateVisibility.NeverReturnExpired)
self.assertEqual(
state_ttl_config.get_ttl_time_characteristic(),
StateTtlConfig.TtlTimeCharacteristic.ProcessingTime)
cleanup_strategies = state_ttl_config.get_cleanup_strategies()
self.assertTrue(cleanup_strategies.is_cleanup_in_background())
self.assertTrue(cleanup_strategies.in_full_snapshot())
incremental_cleanup_strategy = cleanup_strategies.get_incremental_cleanup_strategy()
self.assertEqual(incremental_cleanup_strategy.get_cleanup_size(), 10)
self.assertTrue(incremental_cleanup_strategy.run_cleanup_for_every_record())
rocksdb_compact_filter_cleanup_strategy = \
cleanup_strategies.get_rocksdb_compact_filter_cleanup_strategy()
self.assertEqual(
rocksdb_compact_filter_cleanup_strategy.get_query_time_after_num_entries(), 1000)
def to_java_state_ttl_config(ttl_config: StateTtlConfig):
j_ttl_config_builder = JStateTtlConfig.newBuilder(
JTime.milliseconds(ttl_config.get_ttl().to_milliseconds()))
update_type = ttl_config.get_update_type()
if update_type == StateTtlConfig.UpdateType.Disabled:
j_ttl_config_builder.setUpdateType(JUpdateType.Disabled)
elif update_type == StateTtlConfig.UpdateType.OnCreateAndWrite:
j_ttl_config_builder.setUpdateType(JUpdateType.OnCreateAndWrite)
elif update_type == StateTtlConfig.UpdateType.OnReadAndWrite:
j_ttl_config_builder.setUpdateType(JUpdateType.OnReadAndWrite)
state_visibility = ttl_config.get_state_visibility()
if state_visibility == StateTtlConfig.StateVisibility.ReturnExpiredIfNotCleanedUp:
j_ttl_config_builder.setStateVisibility(
JStateVisibility.ReturnExpiredIfNotCleanedUp)
elif state_visibility == StateTtlConfig.StateVisibility.NeverReturnExpired:
j_ttl_config_builder.setStateVisibility(
JStateVisibility.NeverReturnExpired)
cleanup_strategies = ttl_config.get_cleanup_strategies()
if not cleanup_strategies.is_cleanup_in_background():
j_ttl_config_builder.disableCleanupInBackground()
if cleanup_strategies.in_full_snapshot():
j_ttl_config_builder.cleanupFullSnapshot()
incremental_cleanup_strategy = cleanup_strategies.get_incremental_cleanup_strategy(
)
if incremental_cleanup_strategy:
j_ttl_config_builder.cleanupIncrementally(
incremental_cleanup_strategy.get_cleanup_size(),
incremental_cleanup_strategy.run_cleanup_for_every_record())
rocksdb_compact_filter_cleanup_strategy = \
cleanup_strategies.get_rocksdb_compact_filter_cleanup_strategy()
if rocksdb_compact_filter_cleanup_strategy:
j_ttl_config_builder.cleanupInRocksdbCompactFilter(
rocksdb_compact_filter_cleanup_strategy.
get_query_time_after_num_entries())
return j_ttl_config_builder.build()