def test_memoize(self):
def func(a, b):
return a + b + random.randrange(0, 100000)
config = get_cache_config('simple')
cache = Cache(namespace=self.namespace, **config)
cache_key1 = self.cache._memoize_make_cache_key()(func)
cache_key2 = cache._memoize_make_cache_key()(func)
nt.assert_equal(cache_key1, cache_key2)
def test_memoize(self):
def func(a, b):
return a + b + random.randrange(0, 100000)
config = get_cache_config('simple')
cache = Cache(namespace=self.namespace, **config)
cache_key1 = self.cache._memoize_make_cache_key()(func)
cache_key2 = cache._memoize_make_cache_key()(func)
nt.assert_equal(cache_key1, cache_key2)
def test_timeout(self):
config = get_cache_config('simple', CACHE_DEFAULT_TIMEOUT=1)
self.cache = Cache(**config)
@self.cache.memoize(50)
def func(a, b):
return a + b + random.randrange(0, 100000)
result = func(5, 2)
time.sleep(2)
nt.assert_equal(func(5, 2), result)
def memoize(*args, **kwargs):
_cache_type = kwargs.pop('cache_type', 'simple')
namespace = kwargs.pop('namespace', DEF_NS)
cache_type = get_cache_type() if _cache_type == 'auto' else _cache_type
config = merge([MEMOIZE_DEFAULTS, CACHE_CONFIGS[cache_type]])
if 'CACHE_TIMEOUT' in kwargs:
config['CACHE_TIMEOUT'] = kwargs.pop('CACHE_TIMEOUT')
if 'CACHE_THRESHOLD' in kwargs:
config['CACHE_THRESHOLD'] = kwargs.pop('CACHE_THRESHOLD')
cache = Cache(namespace=namespace, **config)
return cache.memoize(*args, **kwargs)
def test_06a_memoize(self):
self.config['CACHE_DEFAULT_TIMEOUT'] = 1
self.cache = Cache(**self.config)
@self.cache.memoize(50)
def big_foo(a, b):
return a + b + random.randrange(0, 100000)
result = big_foo(5, 2)
time.sleep(2)
assert big_foo(5, 2) == result
def test_timeout(self):
config = get_cache_config('simple', CACHE_DEFAULT_TIMEOUT=1)
self.cache = Cache(**config)
@self.cache.memoize(50)
def func(a, b):
return a + b + random.randrange(0, 100000)
result = func(5, 2)
time.sleep(2)
nt.assert_equal(func(5, 2), result)
def setup_func(*args, **kwargs):
namespace = kwargs.pop('namespace', None)
client_name = kwargs.pop('client_name', None)
if client_name:
CACHE_OPTIONS = kwargs.get('CACHE_OPTIONS', {})
CACHE_OPTIONS['preferred_memcache'] = client_name
kwargs['CACHE_OPTIONS'] = CACHE_OPTIONS
config = get_cache_config(*args, **kwargs)
cache = Cache(namespace=namespace, **config)
return cache
import random
from os import environ
from mezmorize import Cache
if True:
config = {
'DEBUG': True,
'CACHE_TYPE': 'memcached',
'CACHE_MEMCACHED_SERVERS': [environ.get('MEMCACHE_SERVERS')]
}
else:
config = {'CACHE_TYPE': 'simple'}
cache = Cache(**config)
@cache.memoize(60)
def add(a, b):
return a + b + random.randrange(0, 1000)
@cache.memoize(60)
def sub(a, b):
return a - b - random.randrange(0, 1000)
def delete_cache():
cache.delete_memoized(add)
cache.delete_memoized(sub)
def test_23_init_app_sets_app_attribute(self):
cache = Cache()
cache.init_app(self.app)
assert cache.app == self.app
class TestCache(object):
def setup(self):
self.cache = setup_func('simple')
def teardown(self):
self.cache.clear()
def test_dict_config(self):
check_cache_type(self.cache, 'simple')
check_cache_instance(self.cache, SimpleCache)
def test_000_set(self):
self.cache.set('hi', 'hello')
nt.assert_equal(self.cache.get('hi'), 'hello')
def test_add(self):
self.cache.add('hi', 'hello')
nt.assert_equal(self.cache.get('hi'), 'hello')
self.cache.add('hi', 'foobar')
nt.assert_equal(self.cache.get('hi'), 'hello')
def test_add_unicode(self):
self.cache.add('ȟį', 'ƕɛĺłö')
nt.assert_equal(self.cache.get('ȟį'), 'ƕɛĺłö')
self.cache.add('ȟį', 'fööƀåř')
nt.assert_equal(self.cache.get('ȟį'), 'ƕɛĺłö')
def test_add_bytes(self):
self.cache.add(b'hi', b'hello')
nt.assert_equal(self.cache.get(b'hi'), b'hello')
self.cache.add(b'hi', b'foobar')
nt.assert_equal(self.cache.get(b'hi'), b'hello')
def test_delete(self):
check_set_delete(self.cache, 'hi', 'hello')
def test_delete_unicode(self):
check_set_delete(self.cache, 'ȟį', 'ƕɛĺłö')
def test_delete_bytes(self):
check_set_delete(self.cache, b'hi', b'foobar')
def test_memoize(self):
@self.cache.memoize(5)
def func(a, b):
return a + b + random.randrange(0, 100000)
result = func(5, 2)
time.sleep(1)
nt.assert_equal(func(5, 2), result)
result2 = func(5, 3)
nt.assert_not_equal(result2, result)
time.sleep(6)
nt.assert_not_equal(func(5, 2), result)
time.sleep(1)
nt.assert_not_equal(func(5, 3), result2)
def test_timeout(self):
config = get_cache_config('simple', CACHE_DEFAULT_TIMEOUT=1)
self.cache = Cache(**config)
@self.cache.memoize(50)
def func(a, b):
return a + b + random.randrange(0, 100000)
result = func(5, 2)
time.sleep(2)
nt.assert_equal(func(5, 2), result)
def test_delete_timeout(self):
@self.cache.memoize(5)
def func(a, b):
return a + b + random.randrange(0, 100000)
result = func(5, 2)
result2 = func(5, 3)
time.sleep(1)
nt.assert_equal(func(5, 2), result)
nt.assert_equal(func(5, 2), result)
nt.assert_not_equal(func(5, 3), result)
nt.assert_equal(func(5, 3), result2)
self.cache.delete_memoized(func)
nt.assert_not_equal(func(5, 2), result)
nt.assert_not_equal(func(5, 3), result2)
def test_delete_verhash(self):
@self.cache.memoize(5)
def func(a, b):
return a + b + random.randrange(0, 100000)
result = func(5, 2)
result2 = func(5, 3)
time.sleep(1)
nt.assert_equal(func(5, 2), result)
nt.assert_equal(func(5, 2), result)
nt.assert_not_equal(func(5, 3), result)
nt.assert_equal(func(5, 3), result2)
fname = function_namespace(func)[0]
version_key = self.cache._memvname(fname)
nt.assert_is_not_none(self.cache.get(version_key))
self.cache.delete_memoized_verhash(func)
nt.assert_is_none(self.cache.get(version_key))
nt.assert_not_equal(func(5, 2), result)
nt.assert_not_equal(func(5, 3), result2)
nt.assert_is_not_none(self.cache.get(version_key))
def test_delete_rand(self):
@self.cache.memoize()
def func(a, b):
return a + b + random.randrange(0, 100000)
result_a = func(5, 1)
result_b = func(5, 2)
nt.assert_equal(func(5, 1), result_a)
nt.assert_equal(func(5, 2), result_b)
self.cache.delete_memoized(func, 5, 2)
nt.assert_equal(func(5, 1), result_a)
nt.assert_not_equal(func(5, 2), result_b)
def test_args(self):
@self.cache.memoize()
def func(a, b):
return sum(a) + sum(b) + random.randrange(0, 100000)
result_a = func([5, 3, 2], [1])
result_b = func([3, 3], [3, 1])
nt.assert_equal(func([5, 3, 2], [1]), result_a)
nt.assert_equal(func([3, 3], [3, 1]), result_b)
self.cache.delete_memoized(func, [5, 3, 2], [1])
nt.assert_not_equal(func([5, 3, 2], [1]), result_a)
nt.assert_equal(func([3, 3], [3, 1]), result_b)
def test_kwargs(self):
@self.cache.memoize()
def func(a, b=None):
return a + sum(b.values()) + random.randrange(0, 100000)
result_a = func(1, {'one': 1, 'two': 2})
result_b = func(5, {'three': 3, 'four': 4})
nt.assert_equal(func(1, {'one': 1, 'two': 2}), result_a)
nt.assert_equal(func(5, {'three': 3, 'four': 4}), result_b)
self.cache.delete_memoized(func, 1, {'one': 1, 'two': 2})
nt.assert_not_equal(func(1, {'one': 1, 'two': 2}), result_a)
nt.assert_equal(func(5, {'three': 3, 'four': 4}), result_b)
def test_kwargonly(self):
@self.cache.memoize()
def func(a=None):
if a is None:
a = 0
return a + random.random()
result_a = func()
result_b = func(5)
nt.assert_equal(func(), result_a)
nt.assert_less(func(), 1)
nt.assert_equal(func(5), result_b)
nt.assert_greater_equal(func(5), 5)
nt.assert_less(func(5), 6)
def test_arg_kwarg(self):
@self.cache.memoize()
def func(a, b, c=1):
return a + b + c + random.randrange(0, 100000)
nt.assert_equal(func(1, 2), func(1, 2, c=1))
nt.assert_equal(func(1, 2), func(1, 2, 1))
nt.assert_equal(func(1, 2), func(1, 2))
nt.assert_not_equal(func(1, 2, 3), func(1, 2))
with nt.assert_raises(TypeError):
func(1)
def test_classarg(self):
@self.cache.memoize()
def func(a):
return a.value + random.random()
class Adder(object):
def __init__(self, value):
self.value = value
adder = Adder(15)
adder2 = Adder(20)
y = func(adder)
z = func(adder2)
nt.assert_not_equal(y, z)
nt.assert_equal(func(adder), y)
nt.assert_not_equal(func(adder), z)
adder.value = 14
nt.assert_equal(func(adder), y)
nt.assert_not_equal(func(adder), z)
nt.assert_not_equal(func(adder), func(adder2))
nt.assert_equal(func(adder2), z)
def test_classfunc(self):
class Adder(object):
def __init__(self, initial):
self.initial = initial
@self.cache.memoize()
def add(self, b):
return self.initial + b
adder1 = Adder(1)
adder2 = Adder(2)
x = adder1.add(3)
nt.assert_equal(adder1.add(3), x)
nt.assert_not_equal(adder1.add(4), x)
nt.assert_not_equal(adder1.add(3), adder2.add(3))
def test_delete_classfunc(self):
class Adder(object):
def __init__(self, initial):
self.initial = initial
@self.cache.memoize()
def add(self, b):
return self.initial + b + random.random()
adder1 = Adder(1)
adder2 = Adder(2)
a1 = adder1.add(3)
a2 = adder2.add(3)
nt.assert_not_equal(a1, a2)
nt.assert_equal(adder1.add(3), a1)
nt.assert_equal(adder2.add(3), a2)
self.cache.delete_memoized(adder1.add)
a3 = adder1.add(3)
a4 = adder2.add(3)
nt.assert_not_equal(a1, a3)
nt.assert_not_equal(a1, a3)
nt.assert_equal(a2, a4)
self.cache.delete_memoized(Adder.add)
a5 = adder1.add(3)
a6 = adder2.add(3)
nt.assert_not_equal(a5, a6)
nt.assert_not_equal(a3, a5)
nt.assert_not_equal(a4, a6)
def test_delete_classmethod(self):
class Mock(object):
@classmethod
@self.cache.memoize(5)
def func(cls, a, b):
return a + b + random.randrange(0, 100000)
result = Mock.func(5, 2)
result2 = Mock.func(5, 3)
time.sleep(1)
nt.assert_equal(Mock.func(5, 2), result)
nt.assert_equal(Mock.func(5, 2), result)
nt.assert_not_equal(Mock.func(5, 3), result)
nt.assert_equal(Mock.func(5, 3), result2)
self.cache.delete_memoized(Mock.func)
nt.assert_not_equal(Mock.func(5, 2), result)
nt.assert_not_equal(Mock.func(5, 3), result2)
def test_multiple_arg_kwarg_calls(self):
@self.cache.memoize()
def func(a, b, c=[1, 1], d=[1, 1]):
rand = random.randrange(0, 100000)
return sum(a) + sum(b) + sum(c) + sum(d) + rand
expected = func([5, 3, 2], [1], c=[3, 3], d=[3, 3])
nt.assert_equal(func([5, 3, 2], [1], d=[3, 3], c=[3, 3]), expected)
result = func(b=[1], a=[5, 3, 2], c=[3, 3], d=[3, 3])
nt.assert_equal(result, expected)
nt.assert_equal(func([5, 3, 2], [1], [3, 3], [3, 3]), expected)
def test_delete_multiple_arg_kwarg(self):
@self.cache.memoize()
def func(a, b, c=[1, 1], d=[1, 1]):
rand = random.randrange(0, 100000)
return sum(a) + sum(b) + sum(c) + sum(d) + rand
result_a = func([5, 3, 2], [1], c=[3, 3], d=[3, 3])
self.cache.delete_memoized(func, [5, 3, 2], [1], [3, 3], [3, 3])
result_b = func([5, 3, 2], [1], c=[3, 3], d=[3, 3])
nt.assert_not_equal(result_a, result_b)
self.cache.delete_memoized(
func, [5, 3, 2], b=[1], c=[3, 3], d=[3, 3])
result_b = func([5, 3, 2], [1], c=[3, 3], d=[3, 3])
nt.assert_not_equal(result_a, result_b)
self.cache.delete_memoized(func, [5, 3, 2], [1], c=[3, 3], d=[3, 3])
result_a = func([5, 3, 2], [1], c=[3, 3], d=[3, 3])
nt.assert_not_equal(result_a, result_b)
self.cache.delete_memoized(
func, [5, 3, 2], b=[1], c=[3, 3], d=[3, 3])
result_a = func([5, 3, 2], [1], c=[3, 3], d=[3, 3])
nt.assert_not_equal(result_a, result_b)
self.cache.delete_memoized(func, [5, 3, 2], [1], c=[3, 3], d=[3, 3])
result_b = func([5, 3, 2], [1], c=[3, 3], d=[3, 3])
nt.assert_not_equal(result_a, result_b)
self.cache.delete_memoized(func, [5, 3, 2], [1], [3, 3], [3, 3])
result_a = func([5, 3, 2], [1], c=[3, 3], d=[3, 3])
nt.assert_not_equal(result_a, result_b)
def test_kwargs_to_args(self):
def func(a, b, c=None, d=None):
return sum(a) + sum(b) + random.randrange(0, 100000)
expected = (1, 2, 'foo', 'bar')
args = self.cache._gen_args(func, 1, 2, 'foo', 'bar')
nt.assert_equal(tuple(args), expected)
args = self.cache._gen_args(func, 2, 'foo', 'bar', a=1)
nt.assert_equal(tuple(args), expected)
args = self.cache._gen_args(func, a=1, b=2, c='foo', d='bar')
nt.assert_equal(tuple(args), expected)
args = self.cache._gen_args(func, d='bar', b=2, a=1, c='foo')
nt.assert_equal(tuple(args), expected)
args = self.cache._gen_args(func, 1, 2, d='bar', c='foo')
nt.assert_equal(tuple(args), expected)
class CacheTestCase(unittest.TestCase):
def _get_config(self):
return {'CACHE_TYPE': 'simple'}
def setUp(self):
self.config = self._get_config()
self.cache = Cache(**self.config)
def tearDown(self):
self.cache = {}
def test_00_set(self):
self.cache.set('hi', 'hello')
assert self.cache.get('hi') == 'hello'
def test_01_add(self):
self.cache.add('hi', 'hello')
assert self.cache.get('hi') == 'hello'
self.cache.add('hi', 'foobar')
assert self.cache.get('hi') == 'hello'
def test_02_delete(self):
self.cache.set('hi', 'hello')
self.cache.delete('hi')
assert self.cache.get('hi') is None
def test_06_memoize(self):
@self.cache.memoize(5)
def big_foo(a, b):
return a + b + random.randrange(0, 100000)
result = big_foo(5, 2)
time.sleep(1)
assert big_foo(5, 2) == result
result2 = big_foo(5, 3)
assert result2 != result
time.sleep(6)
assert big_foo(5, 2) != result
time.sleep(1)
assert big_foo(5, 3) != result2
def test_06a_memoize(self):
self.config['CACHE_DEFAULT_TIMEOUT'] = 1
self.cache = Cache(**self.config)
@self.cache.memoize(50)
def big_foo(a, b):
return a + b + random.randrange(0, 100000)
result = big_foo(5, 2)
time.sleep(2)
assert big_foo(5, 2) == result
def test_07_delete_memoize(self):
@self.cache.memoize(5)
def big_foo(a, b):
return a + b + random.randrange(0, 100000)
result = big_foo(5, 2)
result2 = big_foo(5, 3)
time.sleep(1)
assert big_foo(5, 2) == result
assert big_foo(5, 2) == result
assert big_foo(5, 3) != result
assert big_foo(5, 3) == result2
self.cache.delete_memoized(big_foo)
assert big_foo(5, 2) != result
assert big_foo(5, 3) != result2
def test_07b_delete_memoized_verhash(self):
@self.cache.memoize(5)
def big_foo(a, b):
return a + b + random.randrange(0, 100000)
result = big_foo(5, 2)
result2 = big_foo(5, 3)
time.sleep(1)
assert big_foo(5, 2) == result
assert big_foo(5, 2) == result
assert big_foo(5, 3) != result
assert big_foo(5, 3) == result2
self.cache.delete_memoized_verhash(big_foo)
_fname, _fname_instance = function_namespace(big_foo)
version_key = self.cache._memvname(_fname)
assert self.cache.get(version_key) is None
assert big_foo(5, 2) != result
assert big_foo(5, 3) != result2
assert self.cache.get(version_key) is not None
def test_08_delete_memoize(self):
@self.cache.memoize()
def big_foo(a, b):
return a + b + random.randrange(0, 100000)
result_a = big_foo(5, 1)
result_b = big_foo(5, 2)
assert big_foo(5, 1) == result_a
assert big_foo(5, 2) == result_b
self.cache.delete_memoized(big_foo, 5, 2)
assert big_foo(5, 1) == result_a
assert big_foo(5, 2) != result_b
## Cleanup bigfoo 5, 1; 5, 2 or it might conflict with
## following run if it also uses memcache
self.cache.delete_memoized(big_foo, 5, 2)
self.cache.delete_memoized(big_foo, 5, 1)
def test_09_args_memoize(self):
@self.cache.memoize()
def big_foo(a, b):
return sum(a) + sum(b) + random.randrange(0, 100000)
result_a = big_foo([5, 3, 2], [1])
result_b = big_foo([3, 3], [3, 1])
assert big_foo([5, 3, 2], [1]) == result_a
assert big_foo([3, 3], [3, 1]) == result_b
self.cache.delete_memoized(big_foo, [5, 3, 2], [1])
assert big_foo([5, 3, 2], [1]) != result_a
assert big_foo([3, 3], [3, 1]) == result_b
## Cleanup bigfoo 5, 1; 5,2 or it might conflict with
## following run if it also uses memecache
self.cache.delete_memoized(big_foo, [5, 3, 2], [1])
self.cache.delete_memoized(big_foo, [3, 3], [1])
def test_10_kwargs_memoize(self):
@self.cache.memoize()
def big_foo(a, b=None):
return a + sum(b.values()) + random.randrange(0, 100000)
result_a = big_foo(1, dict(one=1, two=2))
result_b = big_foo(5, dict(three=3, four=4))
assert big_foo(1, dict(one=1, two=2)) == result_a
assert big_foo(5, dict(three=3, four=4)) == result_b
self.cache.delete_memoized(big_foo, 1, dict(one=1, two=2))
assert big_foo(1, dict(one=1, two=2)) != result_a
assert big_foo(5, dict(three=3, four=4)) == result_b
def test_10a_kwargonly_memoize(self):
@self.cache.memoize()
def big_foo(a=None):
if a is None:
a = 0
return a + random.random()
result_a = big_foo()
result_b = big_foo(5)
assert big_foo() == result_a
assert big_foo() < 1
assert big_foo(5) == result_b
assert big_foo(5) >= 5 and big_foo(5) < 6
def test_10a_arg_kwarg_memoize(self):
@self.cache.memoize()
def f(a, b, c=1):
return a + b + c + random.randrange(0, 100000)
assert f(1, 2) == f(1, 2, c=1)
assert f(1, 2) == f(1, 2, 1)
assert f(1, 2) == f(1, 2)
assert f(1, 2, 3) != f(1, 2)
with self.assertRaises(TypeError):
f(1)
def test_10b_classarg_memoize(self):
@self.cache.memoize()
def bar(a):
return a.value + random.random()
class Adder(object):
def __init__(self, value):
self.value = value
adder = Adder(15)
adder2 = Adder(20)
y = bar(adder)
z = bar(adder2)
assert y != z
assert bar(adder) == y
assert bar(adder) != z
adder.value = 14
assert bar(adder) == y
assert bar(adder) != z
assert bar(adder) != bar(adder2)
assert bar(adder2) == z
def test_10c_classfunc_memoize(self):
class Adder(object):
def __init__(self, initial):
self.initial = initial
@self.cache.memoize()
def add(self, b):
return self.initial + b
adder1 = Adder(1)
adder2 = Adder(2)
x = adder1.add(3)
assert adder1.add(3) == x
assert adder1.add(4) != x
assert adder1.add(3) != adder2.add(3)
def test_10d_classfunc_memoize_delete(self):
class Adder(object):
def __init__(self, initial):
self.initial = initial
@self.cache.memoize()
def add(self, b):
return self.initial + b + random.random()
adder1 = Adder(1)
adder2 = Adder(2)
a1 = adder1.add(3)
a2 = adder2.add(3)
assert a1 != a2
assert adder1.add(3) == a1
assert adder2.add(3) == a2
self.cache.delete_memoized(adder1.add)
a3 = adder1.add(3)
a4 = adder2.add(3)
self.assertNotEqual(a1, a3)
assert a1 != a3
self.assertEqual(a2, a4)
self.cache.delete_memoized(Adder.add)
a5 = adder1.add(3)
a6 = adder2.add(3)
self.assertNotEqual(a5, a6)
self.assertNotEqual(a3, a5)
self.assertNotEqual(a4, a6)
def test_10e_delete_memoize_classmethod(self):
class Mock(object):
@classmethod
@self.cache.memoize(5)
def big_foo(cls, a, b):
return a + b + random.randrange(0, 100000)
result = Mock.big_foo(5, 2)
result2 = Mock.big_foo(5, 3)
time.sleep(1)
assert Mock.big_foo(5, 2) == result
assert Mock.big_foo(5, 2) == result
assert Mock.big_foo(5, 3) != result
assert Mock.big_foo(5, 3) == result2
self.cache.delete_memoized(Mock.big_foo)
assert Mock.big_foo(5, 2) != result
assert Mock.big_foo(5, 3) != result2
def test_14_memoized_multiple_arg_kwarg_calls(self):
@self.cache.memoize()
def big_foo(a, b, c=[1, 1], d=[1, 1]):
rand = random.randrange(0, 100000)
return sum(a) + sum(b) + sum(c) + sum(d) + rand
result_a = big_foo([5, 3, 2], [1], c=[3, 3], d=[3, 3])
assert big_foo([5, 3, 2], [1], d=[3, 3], c=[3, 3]) == result_a
assert big_foo(b=[1], a=[5, 3, 2], c=[3, 3], d=[3, 3]) == result_a
assert big_foo([5, 3, 2], [1], [3, 3], [3, 3]) == result_a
def test_15_memoize_multiple_arg_kwarg_delete(self):
@self.cache.memoize()
def big_foo(a, b, c=[1, 1], d=[1, 1]):
rand = random.randrange(0, 100000)
return sum(a) + sum(b) + sum(c) + sum(d) + rand
result_a = big_foo([5, 3, 2], [1], c=[3, 3], d=[3, 3])
self.cache.delete_memoized(big_foo, [5, 3, 2], [1], [3, 3], [3, 3])
result_b = big_foo([5, 3, 2], [1], c=[3, 3], d=[3, 3])
assert result_a != result_b
self.cache.delete_memoized(
big_foo, [5, 3, 2], b=[1], c=[3, 3], d=[3, 3])
result_b = big_foo([5, 3, 2], [1], c=[3, 3], d=[3, 3])
assert result_a != result_b
self.cache.delete_memoized(big_foo, [5, 3, 2], [1], c=[3, 3], d=[3, 3])
result_a = big_foo([5, 3, 2], [1], c=[3, 3], d=[3, 3])
assert result_a != result_b
self.cache.delete_memoized(
big_foo, [5, 3, 2], b=[1], c=[3, 3], d=[3, 3])
result_a = big_foo([5, 3, 2], [1], c=[3, 3], d=[3, 3])
assert result_a != result_b
self.cache.delete_memoized(big_foo, [5, 3, 2], [1], c=[3, 3], d=[3, 3])
result_b = big_foo([5, 3, 2], [1], c=[3, 3], d=[3, 3])
assert result_a != result_b
self.cache.delete_memoized(big_foo, [5, 3, 2], [1], [3, 3], [3, 3])
result_a = big_foo([5, 3, 2], [1], c=[3, 3], d=[3, 3])
assert result_a != result_b
def test_16_memoize_kwargs_to_args(self):
def big_foo(a, b, c=None, d=None):
return sum(a) + sum(b) + random.randrange(0, 100000)
expected = (1, 2, 'foo', 'bar')
args, kwargs = self.cache._memoize_kwargs_to_args(
big_foo, 1, 2, 'foo', 'bar')
assert (args == expected)
args, kwargs = self.cache._memoize_kwargs_to_args(
big_foo, 2, 'foo', 'bar', a=1)
assert (args == expected)
args, kwargs = self.cache._memoize_kwargs_to_args(
big_foo, a=1, b=2, c='foo', d='bar')
assert (args == expected)
args, kwargs = self.cache._memoize_kwargs_to_args(
big_foo, d='bar', b=2, a=1, c='foo')
assert (args == expected)
args, kwargs = self.cache._memoize_kwargs_to_args(
big_foo, 1, 2, d='bar', c='foo')
assert (args == expected)
def test_17_dict_config(self):
from werkzeug.contrib.cache import SimpleCache
cache = Cache(CACHE_TYPE='simple')
assert cache.config['CACHE_TYPE'] == 'simple'
assert isinstance(cache.cache, SimpleCache)
def test_21_cached_view_forced_update(self):
forced_update = False
def forced_update_fn():
return forced_update
@self.app.route('/a')
@self.cache.cached(5, forced_update=lambda: forced_update)
def view():
return str(time.time())
tc = self.app.test_client()
rv = tc.get('/a')
the_time = rv.data.decode('utf-8')
time.sleep(1)
rv = tc.get('/a')
assert the_time == rv.data.decode('utf-8')
forced_update = True
rv = tc.get('/a')
new_time = rv.data.decode('utf-8')
assert new_time != the_time
forced_update = False
time.sleep(1)
rv = tc.get('/a')
assert new_time == rv.data.decode('utf-8')
def test_22_memoize_forced_update(self):
with self.app.test_request_context():
forced_update = False
@self.cache.memoize(5, forced_update=lambda: forced_update)
def big_foo(a, b):
return a+b+random.randrange(0, 100000)
result = big_foo(5, 2)
time.sleep(1)
assert big_foo(5, 2) == result
forced_update = True
new_result = big_foo(5, 2)
assert new_result != result
forced_update = False
time.sleep(1)
assert big_foo(5, 2) == new_result
def test_23_init_app_sets_app_attribute(self):
cache = Cache()
cache.init_app(self.app)
assert cache.app == self.app
def setUp(self):
self.config = self._get_config()
self.cache = Cache(**self.config)
DATETIME_FORMAT = '{0} %H:%M:%S'.format(DATE_FORMAT)
URL_SAFE = "%/:=&?~#+!$,;'@()*[]"
ALTERNATIVE_DATE_FORMATS = (
"%m-%d-%Y",
"%m/%d/%y",
"%m/%d/%Y",
"%m-%d-%y",
"%Y-%m-%dt%H:%M:%Sz",
# todo more: whatever Yahoo can accept
)
# leave option to substitute with multiprocessing
_map_func = imap
combine_dicts = lambda *d: dict(chain.from_iterable(imap(dict.iteritems, d)))
cache = Cache(**cache_config)
timeout = 60 * 60 * 1
class Objectify:
def __init__(self, **entries):
self.__dict__.update(entries)
def __iter__(self):
return self.__dict__.itervalues()
def iteritems(self):
return self.__dict__.iteritems()
def _apply_func(funcs, items, map_func=starmap):
from epifilter import filterVideo
os.environ['GOOGLE_APPLICATION_CREDENTIALS'] = 'auth.json'
os.environ['AWS_ACCESS_KEY_ID'] = ''
os.environ['AWS_SECRET_ACCESS_KEY'] = ''
os.environ['AWS_ROLE'] = ''
os.environ['AWS_DEFAULT_REGION'] = ''
config = Config(retries=dict(max_attempts=1000))
app = Flask(__name__)
CORS(app)
cache = Cache(CACHE_TYPE='filesystem',
CACHE_DIR='cache',
CACHE_DEFAULT_TIMEOUT=3600 * 24)
def downloadVideo(videoCode):
ydl_opts = {"format": "135", 'outtmpl': f'{videoCode}.%(ext)s'}
with youtube_dl.YoutubeDL(ydl_opts) as ydl:
ydl.download([f'https://www.youtube.com/watch?v={videoCode}'])
ydl_opts = {"format": "251", 'outtmpl': f'{videoCode}.%(ext)s'}
with youtube_dl.YoutubeDL(ydl_opts) as ydl:
ydl.download([f'https://www.youtube.com/watch?v={videoCode}'])
def extractFrames(videoCode, n):
try:
class TestCache(object):
def setup(self):
self.cache = setup_func('simple')
def teardown(self):
self.cache.clear()
def test_dict_config(self):
check_cache_type(self.cache, 'simple')
check_cache_instance(self.cache, SimpleCache)
def test_000_set(self):
self.cache.set('hi', 'hello')
nt.assert_equal(self.cache.get('hi'), 'hello')
def test_add(self):
self.cache.add('hi', 'hello')
nt.assert_equal(self.cache.get('hi'), 'hello')
self.cache.add('hi', 'foobar')
nt.assert_equal(self.cache.get('hi'), 'hello')
def test_add_unicode(self):
self.cache.add('ȟį', 'ƕɛĺłö')
nt.assert_equal(self.cache.get('ȟį'), 'ƕɛĺłö')
self.cache.add('ȟį', 'fööƀåř')
nt.assert_equal(self.cache.get('ȟį'), 'ƕɛĺłö')
def test_add_bytes(self):
self.cache.add(b'hi', b'hello')
nt.assert_equal(self.cache.get(b'hi'), b'hello')
self.cache.add(b'hi', b'foobar')
nt.assert_equal(self.cache.get(b'hi'), b'hello')
def test_delete(self):
check_set_delete(self.cache, 'hi', 'hello')
def test_delete_unicode(self):
check_set_delete(self.cache, 'ȟį', 'ƕɛĺłö')
def test_delete_bytes(self):
check_set_delete(self.cache, b'hi', b'foobar')
def test_memoize(self):
@self.cache.memoize(5)
def func(a, b):
return a + b + random.randrange(0, 100000)
result = func(5, 2)
time.sleep(1)
nt.assert_equal(func(5, 2), result)
result2 = func(5, 3)
nt.assert_not_equal(result2, result)
time.sleep(6)
nt.assert_not_equal(func(5, 2), result)
time.sleep(1)
nt.assert_not_equal(func(5, 3), result2)
def test_timeout(self):
config = get_cache_config('simple', CACHE_DEFAULT_TIMEOUT=1)
self.cache = Cache(**config)
@self.cache.memoize(50)
def func(a, b):
return a + b + random.randrange(0, 100000)
result = func(5, 2)
time.sleep(2)
nt.assert_equal(func(5, 2), result)
def test_delete_timeout(self):
@self.cache.memoize(5)
def func(a, b):
return a + b + random.randrange(0, 100000)
result = func(5, 2)
result2 = func(5, 3)
time.sleep(1)
nt.assert_equal(func(5, 2), result)
nt.assert_equal(func(5, 2), result)
nt.assert_not_equal(func(5, 3), result)
nt.assert_equal(func(5, 3), result2)
self.cache.delete_memoized(func)
nt.assert_not_equal(func(5, 2), result)
nt.assert_not_equal(func(5, 3), result2)
def test_delete_verhash(self):
@self.cache.memoize(5)
def func(a, b):
return a + b + random.randrange(0, 100000)
result = func(5, 2)
result2 = func(5, 3)
time.sleep(1)
nt.assert_equal(func(5, 2), result)
nt.assert_equal(func(5, 2), result)
nt.assert_not_equal(func(5, 3), result)
nt.assert_equal(func(5, 3), result2)
fname = function_namespace(func)[0]
version_key = self.cache._memvname(fname)
nt.assert_is_not_none(self.cache.get(version_key))
self.cache.delete_memoized_verhash(func)
nt.assert_is_none(self.cache.get(version_key))
nt.assert_not_equal(func(5, 2), result)
nt.assert_not_equal(func(5, 3), result2)
nt.assert_is_not_none(self.cache.get(version_key))
def test_delete_rand(self):
@self.cache.memoize()
def func(a, b):
return a + b + random.randrange(0, 100000)
result_a = func(5, 1)
result_b = func(5, 2)
nt.assert_equal(func(5, 1), result_a)
nt.assert_equal(func(5, 2), result_b)
self.cache.delete_memoized(func, 5, 2)
nt.assert_equal(func(5, 1), result_a)
nt.assert_not_equal(func(5, 2), result_b)
def test_args(self):
@self.cache.memoize()
def func(a, b):
return sum(a) + sum(b) + random.randrange(0, 100000)
result_a = func([5, 3, 2], [1])
result_b = func([3, 3], [3, 1])
nt.assert_equal(func([5, 3, 2], [1]), result_a)
nt.assert_equal(func([3, 3], [3, 1]), result_b)
self.cache.delete_memoized(func, [5, 3, 2], [1])
nt.assert_not_equal(func([5, 3, 2], [1]), result_a)
nt.assert_equal(func([3, 3], [3, 1]), result_b)
def test_kwargs(self):
@self.cache.memoize()
def func(a, b=None):
return a + sum(b.values()) + random.randrange(0, 100000)
result_a = func(1, {'one': 1, 'two': 2})
result_b = func(5, {'three': 3, 'four': 4})
nt.assert_equal(func(1, {'one': 1, 'two': 2}), result_a)
nt.assert_equal(func(5, {'three': 3, 'four': 4}), result_b)
self.cache.delete_memoized(func, 1, {'one': 1, 'two': 2})
nt.assert_not_equal(func(1, {'one': 1, 'two': 2}), result_a)
nt.assert_equal(func(5, {'three': 3, 'four': 4}), result_b)
def test_kwargonly(self):
@self.cache.memoize()
def func(a=None):
if a is None:
a = 0
return a + random.random()
result_a = func()
result_b = func(5)
nt.assert_equal(func(), result_a)
nt.assert_less(func(), 1)
nt.assert_equal(func(5), result_b)
nt.assert_greater_equal(func(5), 5)
nt.assert_less(func(5), 6)
def test_arg_kwarg(self):
@self.cache.memoize()
def func(a, b, c=1):
return a + b + c + random.randrange(0, 100000)
nt.assert_equal(func(1, 2), func(1, 2, c=1))
nt.assert_equal(func(1, 2), func(1, 2, 1))
nt.assert_equal(func(1, 2), func(1, 2))
nt.assert_not_equal(func(1, 2, 3), func(1, 2))
with nt.assert_raises(TypeError):
func(1)
def test_classarg(self):
@self.cache.memoize()
def func(a):
return a.value + random.random()
class Adder(object):
def __init__(self, value):
self.value = value
adder = Adder(15)
adder2 = Adder(20)
y = func(adder)
z = func(adder2)
nt.assert_not_equal(y, z)
nt.assert_equal(func(adder), y)
nt.assert_not_equal(func(adder), z)
adder.value = 14
nt.assert_equal(func(adder), y)
nt.assert_not_equal(func(adder), z)
nt.assert_not_equal(func(adder), func(adder2))
nt.assert_equal(func(adder2), z)
def test_classfunc(self):
class Adder(object):
def __init__(self, initial):
self.initial = initial
@self.cache.memoize()
def add(self, b):
return self.initial + b
adder1 = Adder(1)
adder2 = Adder(2)
x = adder1.add(3)
nt.assert_equal(adder1.add(3), x)
nt.assert_not_equal(adder1.add(4), x)
nt.assert_not_equal(adder1.add(3), adder2.add(3))
def test_delete_classfunc(self):
class Adder(object):
def __init__(self, initial):
self.initial = initial
@self.cache.memoize()
def add(self, b):
return self.initial + b + random.random()
adder1 = Adder(1)
adder2 = Adder(2)
a1 = adder1.add(3)
a2 = adder2.add(3)
nt.assert_not_equal(a1, a2)
nt.assert_equal(adder1.add(3), a1)
nt.assert_equal(adder2.add(3), a2)
self.cache.delete_memoized(adder1.add)
a3 = adder1.add(3)
a4 = adder2.add(3)
nt.assert_not_equal(a1, a3)
nt.assert_not_equal(a1, a3)
nt.assert_equal(a2, a4)
self.cache.delete_memoized(Adder.add)
a5 = adder1.add(3)
a6 = adder2.add(3)
nt.assert_not_equal(a5, a6)
nt.assert_not_equal(a3, a5)
nt.assert_not_equal(a4, a6)
def test_delete_classmethod(self):
class Mock(object):
@classmethod
@self.cache.memoize(5)
def func(cls, a, b):
return a + b + random.randrange(0, 100000)
result = Mock.func(5, 2)
result2 = Mock.func(5, 3)
time.sleep(1)
nt.assert_equal(Mock.func(5, 2), result)
nt.assert_equal(Mock.func(5, 2), result)
nt.assert_not_equal(Mock.func(5, 3), result)
nt.assert_equal(Mock.func(5, 3), result2)
self.cache.delete_memoized(Mock.func)
nt.assert_not_equal(Mock.func(5, 2), result)
nt.assert_not_equal(Mock.func(5, 3), result2)
def test_multiple_arg_kwarg_calls(self):
@self.cache.memoize()
def func(a, b, c=[1, 1], d=[1, 1]):
rand = random.randrange(0, 100000)
return sum(a) + sum(b) + sum(c) + sum(d) + rand
expected = func([5, 3, 2], [1], c=[3, 3], d=[3, 3])
nt.assert_equal(func([5, 3, 2], [1], d=[3, 3], c=[3, 3]), expected)
result = func(b=[1], a=[5, 3, 2], c=[3, 3], d=[3, 3])
nt.assert_equal(result, expected)
nt.assert_equal(func([5, 3, 2], [1], [3, 3], [3, 3]), expected)
def test_delete_multiple_arg_kwarg(self):
@self.cache.memoize()
def func(a, b, c=[1, 1], d=[1, 1]):
rand = random.randrange(0, 100000)
return sum(a) + sum(b) + sum(c) + sum(d) + rand
result_a = func([5, 3, 2], [1], c=[3, 3], d=[3, 3])
self.cache.delete_memoized(func, [5, 3, 2], [1], [3, 3], [3, 3])
result_b = func([5, 3, 2], [1], c=[3, 3], d=[3, 3])
nt.assert_not_equal(result_a, result_b)
self.cache.delete_memoized(
func, [5, 3, 2], b=[1], c=[3, 3], d=[3, 3])
result_b = func([5, 3, 2], [1], c=[3, 3], d=[3, 3])
nt.assert_not_equal(result_a, result_b)
self.cache.delete_memoized(func, [5, 3, 2], [1], c=[3, 3], d=[3, 3])
result_a = func([5, 3, 2], [1], c=[3, 3], d=[3, 3])
nt.assert_not_equal(result_a, result_b)
self.cache.delete_memoized(
func, [5, 3, 2], b=[1], c=[3, 3], d=[3, 3])
result_a = func([5, 3, 2], [1], c=[3, 3], d=[3, 3])
nt.assert_not_equal(result_a, result_b)
self.cache.delete_memoized(func, [5, 3, 2], [1], c=[3, 3], d=[3, 3])
result_b = func([5, 3, 2], [1], c=[3, 3], d=[3, 3])
nt.assert_not_equal(result_a, result_b)
self.cache.delete_memoized(func, [5, 3, 2], [1], [3, 3], [3, 3])
result_a = func([5, 3, 2], [1], c=[3, 3], d=[3, 3])
nt.assert_not_equal(result_a, result_b)
def test_kwargs_to_args(self):
def func(a, b, c=None, d=None):
return sum(a) + sum(b) + random.randrange(0, 100000)
expected = (1, 2, 'foo', 'bar')
args = self.cache._gen_args(func, 1, 2, 'foo', 'bar')
nt.assert_equal(tuple(args), expected)
args = self.cache._gen_args(func, 2, 'foo', 'bar', a=1)
nt.assert_equal(tuple(args), expected)
args = self.cache._gen_args(func, a=1, b=2, c='foo', d='bar')
nt.assert_equal(tuple(args), expected)
args = self.cache._gen_args(func, d='bar', b=2, a=1, c='foo')
nt.assert_equal(tuple(args), expected)
args = self.cache._gen_args(func, 1, 2, d='bar', c='foo')
nt.assert_equal(tuple(args), expected)
import subprocess
import os
from mezmorize import Cache
import pandas as pd
import json
import requests
from concurrent.futures import ThreadPoolExecutor, as_completed
cache = Cache(DEBUG=True, CACHE_TYPE='filesystem', CACHE_DIR='cache', CACHE_THRESHOLD=1024*1024, CACHE_DEFAULT_TIMEOUT=9999)
def get_products(query, count=50):
params = ["products", "--keyword=" + query, "-n=" + str(count)]
products_dict = run_scraper(params)
products_df = pd.json_normalize(products_dict)
adjust_reviews(products_df)
products_df['title'] = products_df['title'].str.lstrip("Sponsored Ad - ")
return products_df
@cache.memoize()
def run_scraper(parameters):
print("Running scraper: ", parameters)
default_param = ["node", "amazon-product-api/bin/cli.js", "--random-ua", "--filetype=json"]
output = subprocess.check_output(default_param + parameters, universal_newlines=True)
filename = output.split("was saved to: ")[1].rstrip() + ".json"
with open(filename) as f:
result = json.load(f)
os.remove(filename)
return result
def isfloat(value):
'memcached': {
'DEBUG': True,
'CACHE_TYPE': 'memcached',
'CACHE_MEMCACHED_SERVERS': [environ.get('MEMCACHE_SERVERS')]
}
}
return CONFIGS[cache_type]
def set_cache(cache_config):
global CACHE
CACHE = Cache(**cache_config)
CACHE = Cache(**get_cache_config())
# http://api.stackexchange.com/2.2/tags?
# page=1&pagesize=100&order=desc&sort=popular&site=stackoverflow
# http://api.stackexchange.com/2.2/tags?
# page=1&pagesize=100&order=desc&sort=popular&site=graphicdesign
def memoize(*args, **kwargs):
return CACHE.memoize(*args, **kwargs)
def remove_keys(content, *args):
""" Remove keys from a dict and return new dict
Args:
content (dict): The dict to remove keys from
def set_cache(cache_config):
global CACHE
CACHE = Cache(**cache_config)
def memoize(*args, **kwargs):
cache_type = kwargs.pop('cache_type', 'simple')
cache = Cache(**CACHE_CONFIGS[cache_type])
return cache.memoize(*args, **kwargs)