def wrapper(**kwargs):
# key = (args, tuple_kwargs(kwargs))
filename = f"data/.simple.{kwargs['personality']}.cache"
tokenizer = kwargs["tokenizer"]
# We must use immutaable, hashable args as keys, so no lists, sets, or tokenizer
key = simple_cache.tuple_kwargs(
dict(
personality=kwargs["personality"],
mimic_op=kwargs["mimic_op"],
max_seq_len=kwargs["max_seq_len"],
files=tuple(sorted([str(f) for f in kwargs["files"]])),
tokenizer_name=type(tokenizer).__name__,
vocab_size=len(tokenizer.encoder),
special_tokens=tuple(sorted(tokenizer.special_tokens)),
num_candidates=kwargs["num_candidates"],
))
value = simple_cache.load_key(filename, key)
if value is None:
value = func(**kwargs)
simple_cache.save_key(filename, key, value, ttl)
else:
logger.info(
f'Loaded utturances from cache for {kwargs["personality"]}'
)
return value
def test_save_load_key():
filename = "testing.cache"
key = "answer"
value = 42
ttl = 3
simple_cache.save_key(filename, key, value, ttl)
returned = simple_cache.load_key(filename, key)
assert returned == value
def test_save_load_key():
filename = "testing.cache"
key = "answer"
value = 42
ttl = 3
simple_cache.save_key(filename, key, value, ttl)
returned = simple_cache.load_key(filename, key)
assert returned == value
def test_expiry():
filename = "testing.cache"
key = "answer"
value = 42
ttl = 1
simple_cache.save_key(filename, key, value, ttl)
simple_cache.time.sleep(3)
returned = simple_cache.load_key(filename, key)
assert returned is None
def test_expiry():
filename = "testing.cache"
key = "answer"
value = 42
ttl = 1
simple_cache.save_key(filename, key, value, ttl)
simple_cache.time.sleep(3)
returned = simple_cache.load_key(filename, key)
assert returned is None
def test_prune():
filename = "testing.cache"
key = "answer"
value = 42
ttl = 1
simple_cache.save_key(filename, key, value, ttl)
simple_cache.time.sleep(3)
simple_cache.prune_cache(filename)
cache = simple_cache.read_cache(filename)
assert cache == {}
def test_prune():
filename = "testing.cache"
key = "answer"
value = 42
ttl = 1
simple_cache.save_key(filename, key, value, ttl)
simple_cache.time.sleep(3)
simple_cache.prune_cache(filename)
cache = simple_cache.read_cache(filename)
assert cache == {}
def save_key(self, key, value):
k2 = self.key_with_namespace(key)
simple_cache.save_key(filename=self.filename,
key=k2,
value=value,
ttl=SECONDS_PER_HOUR)
def iterate_core(self, display_tqdm=False):
fx_l = ['service_name', 'service_description', 'paginator_name', 'paginator_entryJmespath', 'paginator_exception', 'entry_keyId', 'entry_keyCreated']
for fx_i in fx_l:
# https://stackoverflow.com/a/9058315/4126114
if fx_i not in self.__class__.__dict__.keys():
raise Exception("Derived class should set %s"%fx_i)
# iterate on regions
import botocore
import boto3
import jmespath
redshift_regions = boto3.Session().get_available_regions(self.service_name)
# redshift_regions = ['us-west-2'] # FIXME
if self.filter_region is not None:
if self.filter_region not in redshift_regions:
msg_err = "Invalid region specified: %s. Supported values: %s"
msg_err = msg_err%(self.filter_region, ", ".join(redshift_regions))
raise IsitfitCliError(msg_err, None) # passing None for click context
# over-ride
redshift_regions = [self.filter_region]
# iterate
region_iterator = redshift_regions
if display_tqdm:
from tqdm import tqdm
region_iterator = tqdm(region_iterator, total = len(redshift_regions), desc="%s, counting in all regions"%self.service_description)
for region_name in region_iterator:
if self.regionInclude_ready and self.filter_region is None:
if region_name not in self.region_include:
# skip since already failed to use it
continue
logger.debug("Region %s"%region_name)
boto3.setup_default_session(region_name = region_name)
# boto3 clients
# https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/redshift.html#Redshift.Client.describe_logging_status
redshift_client = boto3.client(self.service_name)
# https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/cloudwatch.html#metric
self.cloudwatch_resource = boto3.resource('cloudwatch')
# iterate on redshift clusters
paginator = redshift_client.get_paginator(self.paginator_name)
rc_iterator = paginator.paginate()
try:
region_anyClusterFound = False
for rc_describe_page in rc_iterator:
rc_describe_entries = jmespath.search(self.paginator_entryJmespath, rc_describe_page)
for rc_describe_entry in rc_describe_entries:
region_anyClusterFound = True
# add field for region
rc_describe_entry['Region'] = region_name
# yield
yield rc_describe_entry
if not self.regionInclude_ready and self.filter_region is None:
if region_anyClusterFound:
# only include if found clusters in this region
self.region_include.append(region_name)
except botocore.exceptions.ClientError as e:
# Exception that means "no access to region"
if e.response['Error']['Code']==self.paginator_exception:
continue
# all other exceptions raised
raise e
# before exiting, check if a count just completed, and mark region_include as usable
if not self.regionInclude_ready and self.filter_region is None:
self.regionInclude_ready = True
# save to cache
import simple_cache
SECONDS_PER_HOUR = 60*60
simple_cache.save_key(filename=self.cache_filename, key=self.cache_key, value=self.region_include, ttl=SECONDS_PER_HOUR)