def _initCache(self):
"""
# try to load region_include from cache
"""
if self.filter_region is not None:
self.region_include = [self.filter_region]
self.regionInclude_ready = True
return
# need to use the profile name
# because a profile could have ec2 in us-east-1
# whereas another could have ec2 in us-west-1
import boto3
profile_name = boto3.session.Session().profile_name
# cache filename and key to use
from ...dotMan import DotMan
import os
self.cache_filename = 'iterator_cache-%s-%s.pkl'%(profile_name, self.service_name)
self.cache_filename = os.path.join(DotMan().get_dotisitfit(), self.cache_filename)
self.cache_key = 'iterator-region_include'
# https://github.com/barisumog/simple_cache
import simple_cache
ri_cached = simple_cache.load_key(filename=self.cache_filename, key=self.cache_key)
if ri_cached is not None:
logger.debug("Loading regions containing EC2 from cache file")
self.region_include = ri_cached
self.regionInclude_ready = True
def wrapper(**kwargs):
# key = (args, tuple_kwargs(kwargs))
filename = f"data/.simple.{kwargs['personality']}.cache"
tokenizer = kwargs["tokenizer"]
# We must use immutaable, hashable args as keys, so no lists, sets, or tokenizer
key = simple_cache.tuple_kwargs(
dict(
personality=kwargs["personality"],
mimic_op=kwargs["mimic_op"],
max_seq_len=kwargs["max_seq_len"],
files=tuple(sorted([str(f) for f in kwargs["files"]])),
tokenizer_name=type(tokenizer).__name__,
vocab_size=len(tokenizer.encoder),
special_tokens=tuple(sorted(tokenizer.special_tokens)),
num_candidates=kwargs["num_candidates"],
))
value = simple_cache.load_key(filename, key)
if value is None:
value = func(**kwargs)
simple_cache.save_key(filename, key, value, ttl)
else:
logger.info(
f'Loaded utturances from cache for {kwargs["personality"]}'
)
return value
def test_save_load_key():
filename = "testing.cache"
key = "answer"
value = 42
ttl = 3
simple_cache.save_key(filename, key, value, ttl)
returned = simple_cache.load_key(filename, key)
assert returned == value
def test_save_load_key():
filename = "testing.cache"
key = "answer"
value = 42
ttl = 3
simple_cache.save_key(filename, key, value, ttl)
returned = simple_cache.load_key(filename, key)
assert returned == value
def test_expiry():
filename = "testing.cache"
key = "answer"
value = 42
ttl = 1
simple_cache.save_key(filename, key, value, ttl)
simple_cache.time.sleep(3)
returned = simple_cache.load_key(filename, key)
assert returned is None
def test_expiry():
filename = "testing.cache"
key = "answer"
value = 42
ttl = 1
simple_cache.save_key(filename, key, value, ttl)
simple_cache.time.sleep(3)
returned = simple_cache.load_key(filename, key)
assert returned is None
def load_key(self, key):
k2 = self.key_with_namespace(key)
return simple_cache.load_key(filename=self.filename, key=k2)
def test_absent_key():
filename = "testing.cache"
key = "this key does not exist"
returned = simple_cache.load_key(filename, key)
assert returned is None
def test_absent_key():
filename = "testing.cache"
key = "this key does not exist"
returned = simple_cache.load_key(filename, key)
assert returned is None