示例#1
0
  def _initCache(self):
    """
    # try to load region_include from cache
    """
    if self.filter_region is not None:
      self.region_include = [self.filter_region]
      self.regionInclude_ready = True
      return

    # need to use the profile name
    # because a profile could have ec2 in us-east-1
    # whereas another could have ec2 in us-west-1
    import boto3
    profile_name = boto3.session.Session().profile_name

    # cache filename and key to use
    from ...dotMan import DotMan
    import os
    self.cache_filename = 'iterator_cache-%s-%s.pkl'%(profile_name, self.service_name)
    self.cache_filename = os.path.join(DotMan().get_dotisitfit(), self.cache_filename)

    self.cache_key = 'iterator-region_include'

    # https://github.com/barisumog/simple_cache
    import simple_cache
    ri_cached = simple_cache.load_key(filename=self.cache_filename, key=self.cache_key)
    if ri_cached is not None:
      logger.debug("Loading regions containing EC2 from cache file")
      self.region_include = ri_cached
      self.regionInclude_ready = True
 def wrapper(**kwargs):
     # key = (args, tuple_kwargs(kwargs))
     filename = f"data/.simple.{kwargs['personality']}.cache"
     tokenizer = kwargs["tokenizer"]
     # We must use immutaable, hashable args as keys, so no lists, sets, or tokenizer
     key = simple_cache.tuple_kwargs(
         dict(
             personality=kwargs["personality"],
             mimic_op=kwargs["mimic_op"],
             max_seq_len=kwargs["max_seq_len"],
             files=tuple(sorted([str(f) for f in kwargs["files"]])),
             tokenizer_name=type(tokenizer).__name__,
             vocab_size=len(tokenizer.encoder),
             special_tokens=tuple(sorted(tokenizer.special_tokens)),
             num_candidates=kwargs["num_candidates"],
         ))
     value = simple_cache.load_key(filename, key)
     if value is None:
         value = func(**kwargs)
         simple_cache.save_key(filename, key, value, ttl)
     else:
         logger.info(
             f'Loaded utturances from cache for {kwargs["personality"]}'
         )
     return value
示例#3
0
def test_save_load_key():
    filename = "testing.cache"
    key = "answer"
    value = 42
    ttl = 3
    simple_cache.save_key(filename, key, value, ttl)
    returned = simple_cache.load_key(filename, key)
    assert returned == value
示例#4
0
def test_save_load_key():
    filename = "testing.cache"
    key = "answer"
    value = 42
    ttl = 3
    simple_cache.save_key(filename, key, value, ttl)
    returned = simple_cache.load_key(filename, key)
    assert returned == value
示例#5
0
def test_expiry():
    filename = "testing.cache"
    key = "answer"
    value = 42
    ttl = 1
    simple_cache.save_key(filename, key, value, ttl)
    simple_cache.time.sleep(3)
    returned = simple_cache.load_key(filename, key)
    assert returned is None
示例#6
0
def test_expiry():
    filename = "testing.cache"
    key = "answer"
    value = 42
    ttl = 1
    simple_cache.save_key(filename, key, value, ttl)
    simple_cache.time.sleep(3)
    returned = simple_cache.load_key(filename, key)
    assert returned is None
示例#7
0
 def load_key(self, key):
     k2 = self.key_with_namespace(key)
     return simple_cache.load_key(filename=self.filename, key=k2)
示例#8
0
def test_absent_key():
    filename = "testing.cache"
    key = "this key does not exist"
    returned = simple_cache.load_key(filename, key)
    assert returned is None
示例#9
0
def test_absent_key():
    filename = "testing.cache"
    key = "this key does not exist"
    returned = simple_cache.load_key(filename, key)
    assert returned is None