def test_24_delete_memoized_backend_error(self, memoizer_delete):
logging.basicConfig()
# disable logger (to avoid cluttering test output)
logging.disable(logging.ERROR)
memoizer = Memoizer()
@memoizer.memoize()
def f(*args, **kwargs):
return random.randrange(0, 100000)
cache_key = f.make_cache_key(f.uncached, 'a', 'b', c='c', d='d')
old_value = f('a', 'b', c='c', d='d')
memoizer.delete_memoized(f, 'a', 'b', c='c', d='d')
memoizer_delete.assert_called_with(cache_key)
new_value = f('a', 'b', c='c', d='d')
assert new_value == old_value
# re-enable logger
logging.disable(logging.NOTSET)
def __init__(
self,
bridge_ip,
logger,
main_tbl_num,
stats_table,
next_table,
scratch_table_num,
session_rule_version_mapper,
):
self._bridge_ip = bridge_ip
self.logger = logger
self.main_tbl_num = main_tbl_num
self.stats_table = stats_table
self.next_table = next_table
self._scratch_tbl_num = scratch_table_num
self._redirect_dict = RedirectDict()
self._dns_cache = Memoizer({})
self._redirect_port = get_service_config_value(
'redirectd',
'http_port',
8080,
)
self._session_rule_version_mapper = session_rule_version_mapper
self._cwf_args_set = False
self._mac_rewrite_scratch = None
self._internal_ip_allocator = None
self._arpd_controller_fut = None
self._arp_contoller = None
self._egress_table = None
self._bridge_mac = None
def __init__(self, bridge_ip, logger, tbl_num, next_table):
self._bridge_ip = bridge_ip
self.logger = logger
self.tbl_num = tbl_num
self.next_table = next_table
self._redirect_dict = RedirectDict()
self._dns_cache = Memoizer({})
self._redirect_port = get_service_config_value(
'redirectd', 'http_port', 8080)
def __init__(self, bridge_ip, logger, main_tbl_num, next_table,
scratch_table_num, session_rule_version_mapper):
self._bridge_ip = bridge_ip
self.logger = logger
self.main_tbl_num = main_tbl_num
self.next_table = next_table
self._scratch_tbl_num = scratch_table_num
self._redirect_dict = RedirectDict()
self._dns_cache = Memoizer({})
self._redirect_port = get_service_config_value(
'redirectd', 'http_port', 8080)
self._session_rule_version_mapper = session_rule_version_mapper
def test_32_update_memoized_backend_error(self, memoizer_set):
settings.DEBUG = True
logging.basicConfig()
# disable logger (to avoid cluttering test output)
logging.disable(logging.ERROR)
memoizer = Memoizer()
@memoizer.memoize()
def f():
return random.randrange(0, 100000)
exception_raised = False
try:
self.memoizer.update_memoized(f)
except Exception:
exception_raised = True
assert exception_raised
settings.DEBUG = False
memoizer_set.reset_mock()
# re-enable logger
logging.disable(logging.NOTSET)
def test_22_memoize_set_backend_error(self, memoizer_set):
logging.basicConfig()
# disable logger (to avoid cluttering test output)
logging.disable(logging.ERROR)
memoizer = Memoizer()
@memoizer.memoize()
def f():
return random.randrange(0, 100000)
cache_key = f.make_cache_key(f.uncached)
old_value = f()
memoizer_set.assert_called_with(cache_key,
old_value,
timeout=f.cache_timeout)
memoizer_set.reset_mock()
new_value = f()
memoizer_set.assert_called_with(cache_key,
new_value,
timeout=f.cache_timeout)
assert new_value != old_value
# re-enable logger
logging.disable(logging.NOTSET)
def main():
#%%
data0 = {
"x_train": pd.read_pickle(DATA_DIR + "/x_train.pkl"),
"y_train": pd.read_pickle(DATA_DIR + "/y_train.pkl"),
"x_test": pd.read_pickle(DATA_DIR + "/x_test.pkl"),
"y_test": pd.read_pickle(DATA_DIR + "/y_test.pkl")
}
#%%
param_grid_dic = OrderedDict([
("learning_rate", [0.05, 0.10, 0.15, 0.20, 0.25, 0.30]),
("max_depth", [3, 4, 5, 6, 8, 10, 12, 15]),
("min_child_weight", [1, 3, 5, 7]),
("gamma", [0.0, 0.1, 0.2, 0.3, 0.4]),
("colsample_bytree", [0.3, 0.4, 0.5, 0.7]),
])
#%%
train_fraction = 0.05
test_fraction = 0.16
data = subsample(data0, train_fraction, test_fraction)
#%%
memoization_path = DATA_DIR + "/" + "xgboost_memo%g" % train_fraction
print("memoization_path= " + memoization_path)
if not os.path.exists(memoization_path):
os.mkdir(memoization_path)
#%%
fun = Memoizer(lambda param_dic: train_xgb(data, param_dic),
memoization_path)
#%%
grid_search(param_grid_dic, fun)
def test_17_memoize_none_value(self):
self.memoizer = Memoizer()
@self.memoizer.memoize()
def foo():
return None
cache_key = foo.make_cache_key(foo.uncached)
assert (self.memoizer.get(cache_key) is
self.memoizer.default_cache_value)
result = foo()
assert result is None
assert self.memoizer.get(cache_key) is None
self.memoizer.delete_memoized(foo)
cache_key = foo.make_cache_key(foo.uncached)
assert (self.memoizer.get(cache_key) is
self.memoizer.default_cache_value)
def test_26_delete_memoized_verhash_backend_error(self, memoizer_delete):
logging.basicConfig()
# disable logger (to avoid cluttering test output)
logging.disable(logging.ERROR)
memoizer = Memoizer()
@memoizer.memoize()
def f():
return random.randrange(0, 100000)
_fname, _ = function_namespace(f)
version_key = self.memoizer._memvname(_fname)
result = f()
assert f() == result
assert memoizer.get(version_key) is not memoizer.default_cache_value
memoizer.delete_memoized_verhash(f)
memoizer_delete.assert_called_with(version_key)
assert f() == result
assert memoizer.get(version_key) is not memoizer.default_cache_value
# re-enable logger
logging.disable(logging.NOTSET)
def test_19_test_custom_repr_fn(self):
custom_memoizer = Memoizer(repr_fn=lambda x: 'static')
memoizer = Memoizer()
class Test(object):
def fn(self, arg=None):
return 1
cached_fn = memoizer.memoize(60)(fn)
cached_fn_custom_repr = custom_memoizer.memoize(60)(fn)
def __str__(self):
return 'str'
__unicode__ = __str__
def __repr__(self):
return 'repr'
fake_obj = Test()
fake_obj2 = Test()
def _get_keys(obj):
cache_key = obj.cached_fn.make_cache_key(obj.fn, obj, 123)
custom_cache_key = obj.cached_fn_custom_repr.make_cache_key(
obj.fn, obj, 123)
return cache_key, custom_cache_key
cache_key, custom_cache_key = _get_keys(fake_obj)
assert cache_key != custom_cache_key
cache_key2, custom_cache_key2 = _get_keys(fake_obj2)
assert cache_key == cache_key2
assert custom_cache_key == custom_cache_key2
custom_cache_key3 = fake_obj.cached_fn_custom_repr.make_cache_key(
fake_obj.fn, fake_obj, ['test'])
assert custom_cache_key == custom_cache_key3
def test_17_memoize_none_value(self):
self.memoizer = Memoizer()
@self.memoizer.memoize()
def foo():
return None
cache_key = foo.make_cache_key(foo.uncached)
assert (
self.memoizer.get(cache_key) is self.memoizer.default_cache_value)
result = foo()
assert result is None
assert self.memoizer.get(cache_key) is None
self.memoizer.delete_memoized(foo)
cache_key = foo.make_cache_key(foo.uncached)
assert (
self.memoizer.get(cache_key) is self.memoizer.default_cache_value)
def test_31_update_memoized_backend_error(self, memoizer_set):
logging.basicConfig()
# disable logger (to avoid cluttering test output)
logging.disable(logging.ERROR)
memoizer = Memoizer()
@memoizer.memoize()
def f():
return random.randrange(0, 100000)
old_value = self.memoizer.update_memoized(f)
fname, instance_fname = function_namespace(f, args=[])
version_key = self.memoizer._memvname(fname)
version_val = self.memoizer.get(version_key)
memoizer_set.assert_called_with(
version_key,
version_val,
timeout=f.cache_timeout
)
memoizer_set.reset_mock()
new_value = self.memoizer.update_memoized(f)
memoizer_set.assert_called_with(
version_key,
version_val,
timeout=f.cache_timeout
)
assert new_value != old_value
# re-enable logger
logging.disable(logging.NOTSET)
def setUp(self):
self.store = {}
self.records = []
self.memo = Memoizer(self.store, **self.memo_kwargs)
import json
import os
from os import path
from pathlib import Path
import gspread
import pygsheets
from memoize import Memoizer
from oauth2client.service_account import ServiceAccountCredentials
from pygsheets import Spreadsheet
user_spreadsheet_store = {}
memo = Memoizer(user_spreadsheet_store)
class MissingGoogleKeyException(Exception):
pass
def get_credentials():
scope = ['https://spreadsheets.google.com/feeds',
'https://www.googleapis.com/auth/drive']
google_key = os.environ.get("PC_GOOGLE_KEY")
if google_key:
google_key_json_payload = json.loads(google_key)
key_path = '/tmp/data.json'
with open(key_path, 'w') as outfile:
json.dump(google_key_json_payload, outfile)
credentials = ServiceAccountCredentials.from_json_keyfile_name(key_path, scope)
os.remove(key_path)
return credentials
def make_auctions_app(global_conf,
redis_url='redis://localhost:9002/1',
redis_password='',
redis_database='',
sentinel_cluster_name='',
sentinels='',
external_couch_url='http://localhost:5000/auction',
internal_couch_url='http://localhost:9000/',
proxy_internal_couch_url='http://localhost:9000/',
auctions_db='database',
hash_secret_key='',
timezone='Europe/Kiev',
preferred_url_scheme='http',
debug=False,
auto_build=False,
event_source_connection_limit=1000,
limit_replications_progress=99,
limit_replications_func='any'):
"""
[app:main]
use = egg:openprocurement.auction#auctions_server
redis_url = redis://:passwod@localhost:1111/0
external_couch_url = http://localhost:1111/auction
internal_couch_url = http://localhost:9011/
auctions_db = auction
timezone = Europe/Kiev
"""
auctions_server = components.queryUtility(IAuctionsServer)
auctions_server.proxy_connection_pool = ConnectionPool(factory=Connection,
max_size=20,
backend='gevent')
auctions_server.proxy_mappings = Memoizer({})
auctions_server.event_sources_pool = deque([])
auctions_server.config['PREFERRED_URL_SCHEME'] = preferred_url_scheme
auctions_server.config['limit_replications_progress'] = float(
limit_replications_progress)
auctions_server.config['limit_replications_func'] = limit_replications_func
auctions_server.config['REDIS'] = {
'redis': redis_url,
'redis_password': redis_password,
'redis_database': redis_database,
'sentinel_cluster_name': sentinel_cluster_name,
'sentinel': loads(sentinels)
}
auctions_server.config['event_source_connection_limit'] = int(
event_source_connection_limit)
auctions_server.config['EXT_COUCH_DB'] = urljoin(external_couch_url,
auctions_db)
auctions_server.add_url_rule('/' + auctions_db + '/<path:path>',
'couch_server_proxy',
couch_server_proxy,
methods=['GET'])
auctions_server.add_url_rule('/' + auctions_db + '/',
'couch_server_proxy',
couch_server_proxy,
methods=['GET'],
defaults={'path': ''})
auctions_server.add_url_rule('/' + auctions_db + '_secured/<path:path>',
'auth_couch_server_proxy',
auth_couch_server_proxy,
methods=['GET'])
auctions_server.add_url_rule('/' + auctions_db + '_secured/',
'auth_couch_server_proxy',
auth_couch_server_proxy,
methods=['GET'],
defaults={'path': ''})
auctions_server.config['INT_COUCH_URL'] = internal_couch_url
auctions_server.config['PROXY_COUCH_URL'] = proxy_internal_couch_url
auctions_server.config['COUCH_DB'] = auctions_db
auctions_server.config['TIMEZONE'] = tz(timezone)
auctions_server.couch_server = Server(
auctions_server.config.get('INT_COUCH_URL'),
session=Session(retry_delays=range(10)))
if auctions_server.config['COUCH_DB'] not in auctions_server.couch_server:
auctions_server.couch_server.create(auctions_server.config['COUCH_DB'])
auctions_server.db = auctions_server.\
couch_server[auctions_server.config['COUCH_DB']]
auctions_server.config['HASH_SECRET_KEY'] = hash_secret_key
sync_design(auctions_server.db)
for entry_point in iter_entry_points(PKG_NAMESPACE):
plugin = entry_point.load()
plugin(components)
return auctions_server
from memoize import Memoizer
import requests
import utils.errors as errors
from utils import config
memoKeys = {}
memo = Memoizer(memoKeys)
@memo(max_age=3600)
def validate_token(token):
if not isinstance(token, str) or len(token) == 0:
raise errors.InvalidAuth()
headers = {'Authorization'.encode(): token.encode()}
host = config.get_auth_server_url()
port = config.get_auth_server_port()
auth_url = ':'.join([host, str(port)])
response = requests.get('/'.join(['http:/', auth_url, 'v1/users/current']),
headers=headers)
if response.status_code != 200:
raise errors.InvalidAuth()
result = response.json()
if not result:
raise errors.InvalidAuth()
class MemoizeTestCase(SimpleTestCase):
def setUp(self):
self.memoizer = Memoizer()
def test_00_set(self):
self.memoizer.set('hi', 'hello')
assert self.memoizer.get('hi') == 'hello'
def test_01_add(self):
self.memoizer.add('hi', 'hello')
assert self.memoizer.get('hi') == 'hello'
self.memoizer.add('hi', 'foobar')
assert self.memoizer.get('hi') == 'hello'
def test_02_delete(self):
self.memoizer.set('hi', 'hello')
self.memoizer.delete('hi')
assert self.memoizer.get('hi') is self.memoizer.default_cache_value
def test_06_memoize(self):
@self.memoizer.memoize(5)
def big_foo(a, b):
return a + b + random.randrange(0, 100000)
result = big_foo(5, 2)
time.sleep(1)
assert big_foo(5, 2) == result
result2 = big_foo(5, 3)
assert result2 != result
time.sleep(6)
assert big_foo(5, 2) != result
time.sleep(1)
assert big_foo(5, 3) != result2
def test_06a_memoize(self):
@self.memoizer.memoize(50)
def big_foo(a, b):
return a + b + random.randrange(0, 100000)
result = big_foo(5, 2)
time.sleep(2)
assert big_foo(5, 2) == result
def test_07_delete_memoize(self):
@self.memoizer.memoize(5)
def big_foo(a, b):
return a + b + random.randrange(0, 100000)
result = big_foo(5, 2)
result2 = big_foo(5, 3)
time.sleep(1)
assert big_foo(5, 2) == result
assert big_foo(5, 2) == result
assert big_foo(5, 3) != result
assert big_foo(5, 3) == result2
self.memoizer.delete_memoized(big_foo)
assert big_foo(5, 2) != result
assert big_foo(5, 3) != result2
def test_07b_delete_memoized_verhash(self):
@self.memoizer.memoize(5)
def big_foo(a, b):
return a + b + random.randrange(0, 100000)
result = big_foo(5, 2)
result2 = big_foo(5, 3)
time.sleep(1)
assert big_foo(5, 2) == result
assert big_foo(5, 2) == result
assert big_foo(5, 3) != result
assert big_foo(5, 3) == result2
self.memoizer.delete_memoized_verhash(big_foo)
_fname, _fname_instance = function_namespace(big_foo)
version_key = self.memoizer._memvname(_fname)
assert (self.memoizer.get(version_key) is
self.memoizer.default_cache_value)
assert big_foo(5, 2) != result
assert big_foo(5, 3) != result2
assert (self.memoizer.get(version_key)
is not self.memoizer.default_cache_value)
def test_08_delete_memoize(self):
@self.memoizer.memoize()
def big_foo(a, b):
return a + b + random.randrange(0, 100000)
result_a = big_foo(5, 1)
result_b = big_foo(5, 2)
assert big_foo(5, 1) == result_a
assert big_foo(5, 2) == result_b
self.memoizer.delete_memoized(big_foo, 5, 2)
assert big_foo(5, 1) == result_a
assert big_foo(5, 2) != result_b
# Cleanup bigfoo 5,1 5,2 or it might conflict with
# following run if it also uses memecache
self.memoizer.delete_memoized(big_foo, 5, 2)
self.memoizer.delete_memoized(big_foo, 5, 1)
def test_09_args_memoize(self):
@self.memoizer.memoize()
def big_foo(a, b):
return sum(a) + sum(b) + random.randrange(0, 100000)
result_a = big_foo([5, 3, 2], [1])
result_b = big_foo([3, 3], [3, 1])
assert big_foo([5, 3, 2], [1]) == result_a
assert big_foo([3, 3], [3, 1]) == result_b
self.memoizer.delete_memoized(big_foo, [5, 3, 2], [1])
assert big_foo([5, 3, 2], [1]) != result_a
assert big_foo([3, 3], [3, 1]) == result_b
# Cleanup bigfoo 5,1 5,2 or it might conflict with
# following run if it also uses memecache
self.memoizer.delete_memoized(big_foo, [5, 3, 2], [1])
self.memoizer.delete_memoized(big_foo, [3, 3], [1])
def test_10_kwargs_memoize(self):
@self.memoizer.memoize()
def big_foo(a, b=None):
return a + sum(b.values()) + random.randrange(0, 100000)
result_a = big_foo(1, dict(one=1, two=2))
result_b = big_foo(5, dict(three=3, four=4))
assert big_foo(1, dict(one=1, two=2)) == result_a
assert big_foo(5, dict(three=3, four=4)) == result_b
self.memoizer.delete_memoized(big_foo, 1, dict(one=1, two=2))
assert big_foo(1, dict(one=1, two=2)) != result_a
assert big_foo(5, dict(three=3, four=4)) == result_b
def test_10a_kwargonly_memoize(self):
@self.memoizer.memoize()
def big_foo(a=None):
if a is None:
a = 0
return a + random.random()
result_a = big_foo()
result_b = big_foo(5)
assert big_foo() == result_a
assert big_foo() < 1
assert big_foo(5) == result_b
assert big_foo(5) >= 5 and big_foo(5) < 6
def test_10a_arg_kwarg_memoize(self):
@self.memoizer.memoize()
def f(a, b, c=1):
return a + b + c + random.randrange(0, 100000)
assert f(1, 2) == f(1, 2, c=1)
assert f(1, 2) == f(1, 2, 1)
assert f(1, 2) == f(1, 2)
assert f(1, 2) != f(2, 1)
assert f(1, 2, 3) != f(1, 2)
with self.assertRaises(TypeError):
f(1)
def test_10ab_arg_kwarg_memoize(self):
@self.memoizer.memoize()
def f(a, **kwargs):
return (a, kwargs, random.randrange(0, 100000000))
f1 = f(1, b=2)
assert f1 == f(1, b=2)
assert f1 != f(1, b=4)
f2 = f(5, c=6)
assert f2 == f(5, c=6)
assert f2 != f(7, c=6)
self.memoizer.delete_memoized(f, 1, b=2)
assert f1 != f(1, b=2)
assert f2 == f(5, c=6)
assert f2 != f(7, c=6)
def test_10ac_arg_kwarg_memoize(self):
@self.memoizer.memoize()
def f(a, *args, **kwargs):
return (a, args, kwargs, random.randrange(0, 100000000))
f1 = f(1, 2, b=3)
assert f1 == f(1, 2, b=3)
assert f1 != f(1, 3, b=3)
f2 = f(5, 6, c=7)
assert f2 == f(5, 6, c=7)
assert f2 != f(7, 6, c=7)
self.memoizer.delete_memoized(f, 1, 2, b=3)
assert f1 != f(1, 2, b=3)
assert f2 == f(5, 6, c=7)
assert f2 != f(7, 6, c=7)
@self.memoizer.memoize()
def f(a, b=1, *args, **kwargs):
return (a, b, args, kwargs, random.randrange(0, 100000000))
f3 = f(1, 3, 4)
assert f3 == f(1, 3, 4)
assert f3 == f(*(1, 3, 4))
assert f3 != f(1, 3)
self.memoizer.delete_memoized(f, 1, 3, 4)
assert f3 != f(1, 3, 4)
def test_10b_classarg_memoize(self):
@self.memoizer.memoize()
def bar(a, *args):
return a.value + random.random() + sum(args)
class Adder(object):
def __init__(self, value, *args):
self.value = (value + sum(args))
adder = Adder(15)
adder2 = Adder(20)
adder3 = Adder(16, 5)
adder4 = Adder(21, 6)
w = bar(adder)
x = bar(adder2)
y = bar(adder3)
z = bar(adder4)
assert w != x
assert y != z
assert bar(adder) == w
assert bar(adder) != x
assert bar(adder3) == y
assert bar(adder3) != z
adder.value = 14
adder3.value = 15
assert bar(adder) == w
assert bar(adder) != x
assert bar(adder3) == y
assert bar(adder3) != z
assert bar(adder) != bar(adder2)
assert bar(adder3) != bar(adder4)
assert bar(adder2) == x
assert bar(adder4) == z
def test_10c_classfunc_memoize(self):
class Adder(object):
def __init__(self, initial):
self.initial = initial
@self.memoizer.memoize()
def add(self, b, *args):
return self.initial + b + sum(args)
adder1 = Adder(1)
adder2 = Adder(2)
x = adder1.add(3)
y = adder1.add(3, 4)
assert adder1.add(3) == x
assert adder1.add(4) != x
assert adder1.add(3, 4) == y
assert adder1.add(4, 4) != y
assert adder1.add(3) != adder2.add(3)
def test_10d_classfunc_memoize_delete(self):
class Adder(object):
def __init__(self, initial):
self.initial = initial
@self.memoizer.memoize()
def add(self, b, *args):
return self.initial + b + sum(args) + random.random()
adder1 = Adder(1)
adder2 = Adder(2)
a1 = adder1.add(3)
a2 = adder2.add(3)
b1 = adder1.add(3, 1)
b2 = adder2.add(3, 1)
assert a1 != a2
assert a1 != b1
assert a2 != b2
assert adder1.add(3) == a1
assert adder2.add(3) == a2
assert adder1.add(3, 1) == b1
assert adder2.add(3, 1) == b2
self.memoizer.delete_memoized(adder1.add)
a3 = adder1.add(3)
a4 = adder2.add(3)
b3 = adder1.add(3, 1)
b4 = adder2.add(3, 1)
self.assertNotEqual(a1, a3)
self.assertNotEqual(b1, b3)
assert a1 != a3
assert b1 != b3
self.assertEqual(a2, a4)
self.assertEqual(b2, b4)
self.memoizer.delete_memoized(Adder.add)
a5 = adder1.add(3)
a6 = adder2.add(3)
b5 = adder1.add(3, 1)
b6 = adder2.add(3, 1)
self.assertNotEqual(a5, a6)
self.assertNotEqual(b5, b6)
self.assertNotEqual(a3, a5)
self.assertNotEqual(b3, b5)
self.assertNotEqual(a4, a6)
self.assertNotEqual(b4, b6)
def test_10e_delete_memoize_classmethod(self):
class Mock(object):
@classmethod
@self.memoizer.memoize(5)
def big_foo(cls, a, b, *args):
return a + b + sum(args) + random.randrange(0, 100000)
result = Mock.big_foo(5, 2)
result2 = Mock.big_foo(5, 3)
result3 = Mock.big_foo(5, 2, 1)
result4 = Mock.big_foo(5, 3, 1)
time.sleep(1)
assert Mock.big_foo(5, 2) == result
assert Mock.big_foo(5, 2) == result
assert Mock.big_foo(5, 3) != result
assert Mock.big_foo(5, 3) == result2
assert Mock.big_foo(5, 2, 1) == result3
assert Mock.big_foo(5, 2, 1) == result3
assert Mock.big_foo(5, 3, 1) != result3
assert Mock.big_foo(5, 3, 1) == result4
self.memoizer.delete_memoized(Mock.big_foo)
assert Mock.big_foo(5, 2) != result
assert Mock.big_foo(5, 3) != result2
assert Mock.big_foo(5, 2, 1) != result3
assert Mock.big_foo(5, 3, 1) != result4
def test_14_memoized_multiple_arg_kwarg_calls(self):
@self.memoizer.memoize()
def big_foo(a, b, c=[1, 1], d=[1, 1]):
return sum(a) + sum(b) + sum(c) + sum(d) + random.randrange(
0, 100000)
result_a = big_foo([5, 3, 2], [1], c=[3, 3], d=[3, 3])
assert big_foo([5, 3, 2], [1], d=[3, 3], c=[3, 3]) == result_a
assert big_foo(b=[1], a=[5, 3, 2], c=[3, 3], d=[3, 3]) == result_a
assert big_foo([5, 3, 2], [1], [3, 3], [3, 3]) == result_a
def test_15_memoize_multiple_arg_kwarg_delete(self):
@self.memoizer.memoize()
def big_foo(a, b, c=[1, 1], d=[1, 1]):
return sum(a) + sum(b) + sum(c) + sum(d) + random.randrange(
0, 100000)
result_a = big_foo([5, 3, 2], [1], c=[3, 3], d=[3, 3])
self.memoizer.delete_memoized(big_foo, [5, 3, 2], [1], [3, 3], [3, 3])
result_b = big_foo([5, 3, 2], [1], c=[3, 3], d=[3, 3])
assert result_a != result_b
self.memoizer.delete_memoized(big_foo, [5, 3, 2],
b=[1],
c=[3, 3],
d=[3, 3])
result_b = big_foo([5, 3, 2], [1], c=[3, 3], d=[3, 3])
assert result_a != result_b
self.memoizer.delete_memoized(big_foo, [5, 3, 2], [1],
c=[3, 3],
d=[3, 3])
result_a = big_foo([5, 3, 2], [1], c=[3, 3], d=[3, 3])
assert result_a != result_b
self.memoizer.delete_memoized(big_foo, [5, 3, 2],
b=[1],
c=[3, 3],
d=[3, 3])
result_a = big_foo([5, 3, 2], [1], c=[3, 3], d=[3, 3])
assert result_a != result_b
self.memoizer.delete_memoized(big_foo, [5, 3, 2], [1],
c=[3, 3],
d=[3, 3])
result_b = big_foo([5, 3, 2], [1], c=[3, 3], d=[3, 3])
assert result_a != result_b
self.memoizer.delete_memoized(big_foo, [5, 3, 2], [1], [3, 3], [3, 3])
result_a = big_foo([5, 3, 2], [1], c=[3, 3], d=[3, 3])
assert result_a != result_b
def test_16_memoize_kwargs_to_args(self):
def big_foo(a, b, c=None, d=None):
return sum(a) + sum(b) + random.randrange(0, 100000)
expected = (1, 2, 'foo', 'bar')
args, kwargs = self.memoizer._memoize_kwargs_to_args(
big_foo, 1, 2, 'foo', 'bar')
assert (args == expected)
args, kwargs = self.memoizer._memoize_kwargs_to_args(big_foo,
2,
'foo',
'bar',
a=1)
assert (args == expected)
args, kwargs = self.memoizer._memoize_kwargs_to_args(big_foo,
a=1,
b=2,
c='foo',
d='bar')
assert (args == expected)
args, kwargs = self.memoizer._memoize_kwargs_to_args(big_foo,
d='bar',
b=2,
a=1,
c='foo')
assert (args == expected)
args, kwargs = self.memoizer._memoize_kwargs_to_args(big_foo,
1,
2,
d='bar',
c='foo')
assert (args == expected)
def test_17_memoize_none_value(self):
self.memoizer = Memoizer()
@self.memoizer.memoize()
def foo():
return None
cache_key = foo.make_cache_key(foo.uncached)
assert (self.memoizer.get(cache_key) is
self.memoizer.default_cache_value)
result = foo()
assert result is None
assert self.memoizer.get(cache_key) is None
self.memoizer.delete_memoized(foo)
cache_key = foo.make_cache_key(foo.uncached)
assert (self.memoizer.get(cache_key) is
self.memoizer.default_cache_value)
def test_17_delete_memoized_instancemethod_with_mutable_param(self):
class Foo(object):
def __init__(self, id):
self.id = id
@self.memoizer.memoize(5)
def foo(self, bar_obj):
return random.randrange(0, 100000) + bar_obj.id
def __repr__(self):
return ('{}({})'.format(self.__class__.__name__, self.id))
class Bar(object):
def __init__(self, id):
self.id = id
def __repr__(self):
return ('{}({})'.format(self.__class__.__name__, self.id))
a = Foo(1)
b = Bar(1)
c = Bar(2)
result1 = a.foo(b)
result2 = a.foo(c)
time.sleep(1)
assert (a.foo(b) == result1)
assert (a.foo(c) == result2)
self.memoizer.delete_memoized(a.foo, a, b)
assert (a.foo(b) != result1)
assert (a.foo(c) == result2)
class MemoizeTestCase(TestCase):
def setUp(self):
self.memoizer = Memoizer()
def test_00_set(self):
self.memoizer.set('hi', 'hello')
assert self.memoizer.get('hi') == 'hello'
def test_01_add(self):
self.memoizer.add('hi', 'hello')
assert self.memoizer.get('hi') == 'hello'
self.memoizer.add('hi', 'foobar')
assert self.memoizer.get('hi') == 'hello'
def test_02_delete(self):
self.memoizer.set('hi', 'hello')
self.memoizer.delete('hi')
assert self.memoizer.get('hi') is None
def test_06_memoize(self):
@self.memoizer.memoize(5)
def big_foo(a, b):
return a + b + random.randrange(0, 100000)
result = big_foo(5, 2)
time.sleep(1)
assert big_foo(5, 2) == result
result2 = big_foo(5, 3)
assert result2 != result
time.sleep(6)
assert big_foo(5, 2) != result
time.sleep(1)
assert big_foo(5, 3) != result2
def test_06a_memoize(self):
@self.memoizer.memoize(50)
def big_foo(a, b):
return a + b + random.randrange(0, 100000)
result = big_foo(5, 2)
time.sleep(2)
assert big_foo(5, 2) == result
def test_07_delete_memoize(self):
@self.memoizer.memoize(5)
def big_foo(a, b):
return a + b + random.randrange(0, 100000)
result = big_foo(5, 2)
result2 = big_foo(5, 3)
time.sleep(1)
assert big_foo(5, 2) == result
assert big_foo(5, 2) == result
assert big_foo(5, 3) != result
assert big_foo(5, 3) == result2
self.memoizer.delete_memoized(big_foo)
assert big_foo(5, 2) != result
assert big_foo(5, 3) != result2
def test_07b_delete_memoized_verhash(self):
@self.memoizer.memoize(5)
def big_foo(a, b):
return a + b + random.randrange(0, 100000)
result = big_foo(5, 2)
result2 = big_foo(5, 3)
time.sleep(1)
assert big_foo(5, 2) == result
assert big_foo(5, 2) == result
assert big_foo(5, 3) != result
assert big_foo(5, 3) == result2
self.memoizer.delete_memoized_verhash(big_foo)
_fname, _fname_instance = function_namespace(big_foo)
version_key = self.memoizer._memvname(_fname)
assert self.memoizer.get(version_key) is None
assert big_foo(5, 2) != result
assert big_foo(5, 3) != result2
assert self.memoizer.get(version_key) is not None
def test_08_delete_memoize(self):
@self.memoizer.memoize()
def big_foo(a, b):
return a + b + random.randrange(0, 100000)
result_a = big_foo(5, 1)
result_b = big_foo(5, 2)
assert big_foo(5, 1) == result_a
assert big_foo(5, 2) == result_b
self.memoizer.delete_memoized(big_foo, 5, 2)
assert big_foo(5, 1) == result_a
assert big_foo(5, 2) != result_b
## Cleanup bigfoo 5,1 5,2 or it might conflict with
## following run if it also uses memecache
self.memoizer.delete_memoized(big_foo, 5, 2)
self.memoizer.delete_memoized(big_foo, 5, 1)
def test_09_args_memoize(self):
@self.memoizer.memoize()
def big_foo(a, b):
return sum(a) + sum(b) + random.randrange(0, 100000)
result_a = big_foo([5, 3, 2], [1])
result_b = big_foo([3, 3], [3, 1])
assert big_foo([5, 3, 2], [1]) == result_a
assert big_foo([3, 3], [3, 1]) == result_b
self.memoizer.delete_memoized(big_foo, [5, 3, 2], [1])
assert big_foo([5, 3, 2], [1]) != result_a
assert big_foo([3, 3], [3, 1]) == result_b
## Cleanup bigfoo 5,1 5,2 or it might conflict with
## following run if it also uses memecache
self.memoizer.delete_memoized(big_foo, [5, 3, 2], [1])
self.memoizer.delete_memoized(big_foo, [3, 3], [1])
def test_10_kwargs_memoize(self):
@self.memoizer.memoize()
def big_foo(a, b=None):
return a + sum(b.values()) + random.randrange(0, 100000)
result_a = big_foo(1, dict(one=1, two=2))
result_b = big_foo(5, dict(three=3, four=4))
assert big_foo(1, dict(one=1, two=2)) == result_a
assert big_foo(5, dict(three=3, four=4)) == result_b
self.memoizer.delete_memoized(big_foo, 1, dict(one=1, two=2))
assert big_foo(1, dict(one=1, two=2)) != result_a
assert big_foo(5, dict(three=3, four=4)) == result_b
def test_10a_kwargonly_memoize(self):
@self.memoizer.memoize()
def big_foo(a=None):
if a is None:
a = 0
return a + random.random()
result_a = big_foo()
result_b = big_foo(5)
assert big_foo() == result_a
assert big_foo() < 1
assert big_foo(5) == result_b
assert big_foo(5) >= 5 and big_foo(5) < 6
def test_10a_arg_kwarg_memoize(self):
@self.memoizer.memoize()
def f(a, b, c=1):
return a + b + c + random.randrange(0, 100000)
assert f(1, 2) == f(1, 2, c=1)
assert f(1, 2) == f(1, 2, 1)
assert f(1, 2) == f(1, 2)
assert f(1, 2, 3) != f(1, 2)
with self.assertRaises(TypeError):
f(1)
def test_10b_classarg_memoize(self):
@self.memoizer.memoize()
def bar(a):
return a.value + random.random()
class Adder(object):
def __init__(self, value):
self.value = value
adder = Adder(15)
adder2 = Adder(20)
y = bar(adder)
z = bar(adder2)
assert y != z
assert bar(adder) == y
assert bar(adder) != z
adder.value = 14
assert bar(adder) == y
assert bar(adder) != z
assert bar(adder) != bar(adder2)
assert bar(adder2) == z
def test_10c_classfunc_memoize(self):
class Adder(object):
def __init__(self, initial):
self.initial = initial
@self.memoizer.memoize()
def add(self, b):
return self.initial + b
adder1 = Adder(1)
adder2 = Adder(2)
x = adder1.add(3)
assert adder1.add(3) == x
assert adder1.add(4) != x
assert adder1.add(3) != adder2.add(3)
def test_10d_classfunc_memoize_delete(self):
class Adder(object):
def __init__(self, initial):
self.initial = initial
@self.memoizer.memoize()
def add(self, b):
return self.initial + b + random.random()
adder1 = Adder(1)
adder2 = Adder(2)
a1 = adder1.add(3)
a2 = adder2.add(3)
assert a1 != a2
assert adder1.add(3) == a1
assert adder2.add(3) == a2
self.memoizer.delete_memoized(adder1.add)
a3 = adder1.add(3)
a4 = adder2.add(3)
self.assertNotEqual(a1, a3)
assert a1 != a3
self.assertEqual(a2, a4)
self.memoizer.delete_memoized(Adder.add)
a5 = adder1.add(3)
a6 = adder2.add(3)
self.assertNotEqual(a5, a6)
self.assertNotEqual(a3, a5)
self.assertNotEqual(a4, a6)
def test_10e_delete_memoize_classmethod(self):
class Mock(object):
@classmethod
@self.memoizer.memoize(5)
def big_foo(cls, a, b):
return a + b + random.randrange(0, 100000)
result = Mock.big_foo(5, 2)
result2 = Mock.big_foo(5, 3)
time.sleep(1)
assert Mock.big_foo(5, 2) == result
assert Mock.big_foo(5, 2) == result
assert Mock.big_foo(5, 3) != result
assert Mock.big_foo(5, 3) == result2
self.memoizer.delete_memoized(Mock.big_foo)
assert Mock.big_foo(5, 2) != result
assert Mock.big_foo(5, 3) != result2
def test_14_memoized_multiple_arg_kwarg_calls(self):
@self.memoizer.memoize()
def big_foo(a, b, c=[1, 1], d=[1, 1]):
return sum(a) + sum(b) + sum(c) + sum(d) + random.randrange(
0, 100000)
result_a = big_foo([5, 3, 2], [1], c=[3, 3], d=[3, 3])
assert big_foo([5, 3, 2], [1], d=[3, 3], c=[3, 3]) == result_a
assert big_foo(b=[1], a=[5, 3, 2], c=[3, 3], d=[3, 3]) == result_a
assert big_foo([5, 3, 2], [1], [3, 3], [3, 3]) == result_a
def test_15_memoize_multiple_arg_kwarg_delete(self):
@self.memoizer.memoize()
def big_foo(a, b, c=[1, 1], d=[1, 1]):
return sum(a) + sum(b) + sum(c) + sum(d) + random.randrange(
0, 100000)
result_a = big_foo([5, 3, 2], [1], c=[3, 3], d=[3, 3])
self.memoizer.delete_memoized(big_foo, [5, 3, 2], [1], [3, 3], [3, 3])
result_b = big_foo([5, 3, 2], [1], c=[3, 3], d=[3, 3])
assert result_a != result_b
self.memoizer.delete_memoized(big_foo, [5, 3, 2],
b=[1],
c=[3, 3],
d=[3, 3])
result_b = big_foo([5, 3, 2], [1], c=[3, 3], d=[3, 3])
assert result_a != result_b
self.memoizer.delete_memoized(big_foo, [5, 3, 2], [1],
c=[3, 3],
d=[3, 3])
result_a = big_foo([5, 3, 2], [1], c=[3, 3], d=[3, 3])
assert result_a != result_b
self.memoizer.delete_memoized(big_foo, [5, 3, 2],
b=[1],
c=[3, 3],
d=[3, 3])
result_a = big_foo([5, 3, 2], [1], c=[3, 3], d=[3, 3])
assert result_a != result_b
self.memoizer.delete_memoized(big_foo, [5, 3, 2], [1],
c=[3, 3],
d=[3, 3])
result_b = big_foo([5, 3, 2], [1], c=[3, 3], d=[3, 3])
assert result_a != result_b
self.memoizer.delete_memoized(big_foo, [5, 3, 2], [1], [3, 3], [3, 3])
result_a = big_foo([5, 3, 2], [1], c=[3, 3], d=[3, 3])
assert result_a != result_b
def test_16_memoize_kwargs_to_args(self):
def big_foo(a, b, c=None, d=None):
return sum(a) + sum(b) + random.randrange(0, 100000)
expected = (1, 2, 'foo', 'bar')
args, kwargs = self.memoizer._memoize_kwargs_to_args(
big_foo, 1, 2, 'foo', 'bar')
assert (args == expected)
args, kwargs = self.memoizer._memoize_kwargs_to_args(big_foo,
2,
'foo',
'bar',
a=1)
assert (args == expected)
args, kwargs = self.memoizer._memoize_kwargs_to_args(big_foo,
a=1,
b=2,
c='foo',
d='bar')
assert (args == expected)
args, kwargs = self.memoizer._memoize_kwargs_to_args(big_foo,
d='bar',
b=2,
a=1,
c='foo')
assert (args == expected)
args, kwargs = self.memoizer._memoize_kwargs_to_args(big_foo,
1,
2,
d='bar',
c='foo')
assert (args == expected)
def setUp(self):
self.memoizer = Memoizer()
from memoize import Memoizer
PYPI_URL_PATTERN = 'https://pypi.python.org/pypi/{package}/json'
CHANNEL_URL_PATTERN = 'https://conda.anaconda.org/{channel}/{platform}/repodata.json'
REDIS_CONN = redis.StrictRedis(host=os.environ.get('REDIS_HOST', '127.0.0.1'),
port=os.environ.get('REDIS_PORT', '6379'),
password=os.environ.get('REDIS_PASSWORD',
'¯\_(ツ)_/¯'))
CHANNELS = ['conda-forge', 'anaconda', 'c3i_test']
PKG_INFO = {}
memo = Memoizer({})
# https://stackoverflow.com/a/34366589/1005215
def get_pypi_version(package, url_pattern=PYPI_URL_PATTERN):
"""Return version of package on pypi.python.org using json."""
req = requests.get(url_pattern.format(package=package))
version = parse('0')
if req.status_code == requests.codes.ok:
j = json.loads(req.text.encode(req.encoding))
if 'releases' in j:
versions = [parse(s) for s in j['releases']]
filtered = [v for v in versions if not v.is_prerelease]
if len(filtered) == 0:
return max(versions)
else:
class RedirectionManager:
"""
RedirectionManager
The redirection manager handles subscribers who have redirection enabled,
it adds the flows into ovs for redirecting user to the redirection server
"""
DNS_TIMEOUT_SECS = 15
REDIRECT_NOT_PROCESSED = REG_ZERO_VAL
REDIRECT_PROCESSED = 0x1
RedirectRequest = namedtuple(
'RedirectRequest',
['imsi', 'ip_addr', 'rule', 'rule_num', 'rule_version', 'priority'],
)
def __init__(self, bridge_ip, logger, main_tbl_num, next_table,
scratch_table_num, session_rule_version_mapper):
self._bridge_ip = bridge_ip
self.logger = logger
self.main_tbl_num = main_tbl_num
self.next_table = next_table
self._scratch_tbl_num = scratch_table_num
self._redirect_dict = RedirectDict()
self._dns_cache = Memoizer({})
self._redirect_port = get_service_config_value('redirectd',
'http_port', 8080)
self._session_rule_version_mapper = session_rule_version_mapper
self._cwf_args_set = False
self._mac_rewrite_scratch = None
self._internal_ip_allocator = None
self._arpd_controller_fut = None
self._arp_contoller = None
self._egress_table = None
self._bridge_mac = None
def set_cwf_args(self, internal_ip_allocator, arp, mac_rewrite,
bridge_name, egress_table):
self._mac_rewrite_scratch = mac_rewrite
self._internal_ip_allocator = internal_ip_allocator
self._arpd_controller_fut = arp
self._arp_contoller = None
self._egress_table = egress_table
def get_virtual_iface_mac(iface):
virt_ifaddresses = netifaces.ifaddresses(iface)
return virt_ifaddresses[netifaces.AF_LINK][0]['addr']
self._bridge_mac = get_virtual_iface_mac(bridge_name)
self._cwf_args_set = True
return self
def setup_lte_redirect(self, datapath, loop, redirect_request):
"""
Depending on redirection server address type install redirection rules
"""
imsi = redirect_request.imsi
ip_addr = redirect_request.ip_addr
rule = redirect_request.rule
rule_num = redirect_request.rule_num
rule_version = redirect_request.rule_version
priority = redirect_request.priority
# TODO IMPORTANT check that redirectd service is running, as its a
# dynamic service its not on by default. Will save you some sanity :)
# TODO figure out what to do with SIP_URI
if rule.redirect.address_type == rule.redirect.SIP_URI:
raise RedirectException("SIP_URIs redirection isn't setup")
if rule.redirect.address_type == rule.redirect.IPv6:
raise RedirectException("No ipv6 support, so no ipv6 redirect")
self._save_redirect_entry(ip_addr, rule.redirect)
self._install_redirect_flows(datapath, loop, imsi, ip_addr, rule,
rule_num, rule_version, priority)
return
def _install_redirect_flows(self, datapath, loop, imsi, ip_addr, rule,
rule_num, rule_version, priority):
"""
Add flows to forward traffic to the redirection server.
1) Intercept tcp traffic to the web to the redirection server, which
completes the tcp handshake. This is done by adding an OVS flow
with a learn action (flow catches inbound tcp packets, while learn
action creates another flow that sends packets back from server)
2) Add flows to allow UDP traffic so DNS queries can go through.
Finally add flows with a higher priority that allow traffic to and
from the address provided in redirect rule.
"""
if rule.redirect.address_type == rule.redirect.URL:
self._install_url_bypass_flows(datapath,
loop,
imsi,
rule,
rule_num,
rule_version,
priority,
ue_ip=ip_addr)
elif rule.redirect.address_type == rule.redirect.IPv4:
self._install_ipv4_bypass_flows(datapath,
imsi,
rule,
rule_num,
rule_version,
priority,
[rule.redirect.server_address],
ue_ip=ip_addr)
self._install_dns_flows(datapath, imsi, rule, rule_num, rule_version,
priority)
self._install_server_flows(datapath, imsi, ip_addr, rule, rule_num,
rule_version, priority)
def _install_scratch_table_flows(self, datapath, imsi, rule, rule_num,
rule_version, priority):
"""
The flow action for subscribers that need to be redirected does 2 things
* Forward requests from subscriber to the internal http server
* Instantiate a flow that matches response packets from the server
and sends them back to subscriber
Match: incoming tcp traffic with port 80, direction out
Action:
1) Set reg2 to rule_num
2) Set ip dst to server ip
3) Output to table 20
4) Apply LearnAction:
LearnAction(adds new flow for every pkt flow that hits this rule)
1) Match ip packets
2) Match tcp protocol
3) Match packets from LOCAL port
4) Match ip src = server ip
5) Match ip dst = current flow ip src
6) Match tcp src = current flow tcp dst
7) Match tcp dst = current flow tcp src
8) Load ip src = current flow ip dst
9) Output through gtp0
"""
parser = datapath.ofproto_parser
match_http = MagmaMatch(eth_type=ether_types.ETH_TYPE_IP,
ip_proto=IPPROTO_TCP,
tcp_dst=80,
imsi=encode_imsi(imsi),
direction=Direction.OUT)
of_note = parser.NXActionNote(list(rule.id.encode()))
actions = [
parser.NXActionLearn(
table_id=self.main_tbl_num,
priority=priority,
cookie=rule_num,
specs=[
parser.NXFlowSpecMatch(src=ether_types.ETH_TYPE_IP,
dst=('eth_type_nxm', 0),
n_bits=16),
parser.NXFlowSpecMatch(src=IPPROTO_TCP,
dst=('ip_proto_nxm', 0),
n_bits=8),
parser.NXFlowSpecMatch(src=Direction.IN,
dst=(DIRECTION_REG, 0),
n_bits=32),
parser.NXFlowSpecMatch(src=int(
ipaddress.IPv4Address(self._bridge_ip)),
dst=('ipv4_src_nxm', 0),
n_bits=32),
parser.NXFlowSpecMatch(src=('ipv4_src_nxm', 0),
dst=('ipv4_dst_nxm', 0),
n_bits=32),
parser.NXFlowSpecMatch(src=('tcp_src_nxm', 0),
dst=('tcp_dst_nxm', 0),
n_bits=16),
parser.NXFlowSpecMatch(src=self._redirect_port,
dst=('tcp_src_nxm', 0),
n_bits=16),
parser.NXFlowSpecMatch(src=encode_imsi(imsi),
dst=(IMSI_REG, 0),
n_bits=64),
parser.NXFlowSpecLoad(src=('ipv4_dst_nxm', 0),
dst=('ipv4_src_nxm', 0),
n_bits=32),
parser.NXFlowSpecLoad(src=80,
dst=('tcp_src_nxm', 0),
n_bits=16),
# Learn doesn't support resubmit to table, so send directly
parser.NXFlowSpecOutput(src=('in_port', 0),
dst="",
n_bits=16),
]),
parser.NXActionRegLoad2(dst=SCRATCH_REGS[0],
value=self.REDIRECT_PROCESSED),
parser.OFPActionSetField(ipv4_dst=self._bridge_ip),
parser.OFPActionSetField(tcp_dst=self._redirect_port),
of_note,
]
actions += self._load_rule_actions(parser, rule_num, rule_version)
flows.add_resubmit_current_service_flow(
datapath,
self._scratch_tbl_num,
match_http,
actions,
priority=priority,
cookie=rule_num,
hard_timeout=rule.hard_timeout,
resubmit_table=self.main_tbl_num)
match = MagmaMatch(imsi=encode_imsi(imsi))
action = []
flows.add_drop_flow(datapath,
self._scratch_tbl_num,
match,
action,
priority=flows.MINIMUM_PRIORITY + 1,
cookie=rule_num)
def _install_not_processed_flows(self, datapath, imsi, ip_addr, rule,
rule_num, priority):
"""
Redirect all traffic to the scratch table to only allow redirected
http traffic to go through, the rest will be dropped. reg0 is used as
a boolean to know whether the drop rule was processed.
"""
parser = datapath.ofproto_parser
of_note = parser.NXActionNote(list(rule.id.encode()))
match = MagmaMatch(imsi=encode_imsi(imsi),
direction=Direction.OUT,
reg0=self.REDIRECT_NOT_PROCESSED,
eth_type=ether_types.ETH_TYPE_IP,
ipv4_src=ip_addr)
action = [of_note]
flows.add_resubmit_current_service_flow(
datapath,
self.main_tbl_num,
match,
action,
priority=priority,
cookie=rule_num,
hard_timeout=rule.hard_timeout,
resubmit_table=self._scratch_tbl_num)
match = MagmaMatch(imsi=encode_imsi(imsi),
direction=Direction.OUT,
reg0=self.REDIRECT_PROCESSED,
eth_type=ether_types.ETH_TYPE_IP,
ipv4_src=ip_addr)
action = [of_note]
flows.add_resubmit_next_service_flow(datapath,
self.main_tbl_num,
match,
action,
priority=priority,
cookie=rule_num,
hard_timeout=rule.hard_timeout,
resubmit_table=self.next_table)
def setup_cwf_redirect(self, datapath, loop, redirect_request):
"""
Add flows to forward traffic to the redirection server for cwf networks
1) Intercept tcp traffic to the web to the redirection server, which
completes the tcp handshake. Also overwrite UE src ip to match the
subnet of the redirection server.
This is done by assigning an internal IP per each subscriber.
Add an OVS flow with a learn action (flow catches inbound tcp http
packets, while learn action creates another flow that rewrites
packet back to send to ue)
2) Add flows to allow UDP traffic so DNS queries can go through.
Add flows with a higher priority that allow traffic to and
from the address provided in redirect rule.
TODO we might want to track stats for these rules and report to sessiond
"""
if not self._cwf_args_set:
raise RedirectException("Can't install cwf redirection, missing"
"cwf specific args, call set_cwf_args()")
imsi = redirect_request.imsi
rule = redirect_request.rule
rule_num = redirect_request.rule_num
rule_version = redirect_request.rule_version
priority = redirect_request.priority
if rule.redirect.address_type == rule.redirect.URL:
self._install_url_bypass_flows(datapath, loop, imsi, rule,
rule_num, rule_version, priority)
elif rule.redirect.address_type == rule.redirect.IPv4:
self._install_ipv4_bypass_flows(datapath, imsi, rule, rule_num,
rule_version, priority,
[rule.redirect.server_address])
parser = datapath.ofproto_parser
# TODO use subscriber ip_addr to generate internal IP and release
# internal IP when subscriber disconnects or redirection flow is removed
internal_ip = self._internal_ip_allocator.next_ip()
self._save_redirect_entry(internal_ip, rule.redirect)
#TODO check if we actually need this, dns might already be allowed
self._install_dns_flows(datapath, imsi, rule, rule_num, rule_version,
priority)
match_tcp_80 = MagmaMatch(imsi=encode_imsi(imsi),
eth_type=ether_types.ETH_TYPE_IP,
ip_proto=IPPROTO_TCP,
direction=Direction.OUT,
tcp_dst=80)
match_tcp_8008 = MagmaMatch(imsi=encode_imsi(imsi),
eth_type=ether_types.ETH_TYPE_IP,
ip_proto=IPPROTO_TCP,
direction=Direction.OUT,
tcp_dst=8080)
match_tcp_8080 = MagmaMatch(imsi=encode_imsi(imsi),
eth_type=ether_types.ETH_TYPE_IP,
ip_proto=IPPROTO_TCP,
direction=Direction.OUT,
tcp_dst=8008)
actions = [
parser.NXActionLearn(
table_id=self._mac_rewrite_scratch,
priority=flows.UE_FLOW_PRIORITY,
cookie=rule_num,
specs=[
parser.NXFlowSpecMatch(src=ether_types.ETH_TYPE_IP,
dst=('eth_type_nxm', 0),
n_bits=16),
parser.NXFlowSpecMatch(src=IPPROTO_TCP,
dst=('ip_proto_nxm', 0),
n_bits=8),
parser.NXFlowSpecMatch(src=int(
ipaddress.IPv4Address(self._bridge_ip)),
dst=('ipv4_src_nxm', 0),
n_bits=32),
parser.NXFlowSpecMatch(src=int(internal_ip),
dst=('ipv4_dst_nxm', 0),
n_bits=32),
parser.NXFlowSpecMatch(src=('tcp_src_nxm', 0),
dst=('tcp_dst_nxm', 0),
n_bits=16),
parser.NXFlowSpecMatch(src=self._redirect_port,
dst=('tcp_src_nxm', 0),
n_bits=16),
parser.NXFlowSpecLoad(src=('eth_src_nxm', 0),
dst=('eth_dst_nxm', 0),
n_bits=48),
parser.NXFlowSpecLoad(src=encode_imsi(imsi),
dst=(IMSI_REG, 0),
n_bits=64),
parser.NXFlowSpecLoad(src=('ipv4_src_nxm', 0),
dst=('ipv4_dst_nxm', 0),
n_bits=32),
parser.NXFlowSpecLoad(src=('ipv4_dst_nxm', 0),
dst=('ipv4_src_nxm', 0),
n_bits=32),
parser.NXFlowSpecLoad(src=('tcp_dst_nxm', 0),
dst=('tcp_src_nxm', 0),
n_bits=16),
]),
parser.OFPActionSetField(ipv4_src=str(internal_ip)),
parser.OFPActionSetField(ipv4_dst=self._bridge_ip),
parser.OFPActionSetField(eth_dst=self._bridge_mac),
parser.OFPActionSetField(tcp_dst=self._redirect_port),
]
flows.add_output_flow(datapath,
self.main_tbl_num,
match_tcp_80,
actions,
priority=flows.UE_FLOW_PRIORITY,
cookie=rule_num,
output_port=OFPP_LOCAL)
flows.add_output_flow(datapath,
self.main_tbl_num,
match_tcp_8008,
actions,
priority=flows.UE_FLOW_PRIORITY,
cookie=rule_num,
output_port=OFPP_LOCAL)
flows.add_output_flow(datapath,
self.main_tbl_num,
match_tcp_8080,
actions,
priority=flows.UE_FLOW_PRIORITY,
cookie=rule_num,
output_port=OFPP_LOCAL)
# Add flows for vlan traffic too (we need to pop vlan for flask server)
# In ryu vlan_vid=(0x1000, 0x1000) matches all vlans
match_tcp_80_vlan = MagmaMatch(imsi=encode_imsi(imsi),
eth_type=ether_types.ETH_TYPE_IP,
ip_proto=IPPROTO_TCP,
direction=Direction.OUT,
tcp_dst=80,
vlan_vid=(0x1000, 0x1000))
match_tcp_8008_vlan = MagmaMatch(imsi=encode_imsi(imsi),
eth_type=ether_types.ETH_TYPE_IP,
ip_proto=IPPROTO_TCP,
direction=Direction.OUT,
tcp_dst=8080,
vlan_vid=(0x1000, 0x1000))
match_tcp_8080_vlan = MagmaMatch(imsi=encode_imsi(imsi),
eth_type=ether_types.ETH_TYPE_IP,
ip_proto=IPPROTO_TCP,
direction=Direction.OUT,
tcp_dst=8008,
vlan_vid=(0x1000, 0x1000))
actions.append(parser.OFPActionPopVlan())
flows.add_output_flow(datapath,
self.main_tbl_num,
match_tcp_80_vlan,
actions,
priority=flows.UE_FLOW_PRIORITY + 1,
cookie=rule_num,
output_port=OFPP_LOCAL)
flows.add_output_flow(datapath,
self.main_tbl_num,
match_tcp_8008_vlan,
actions,
priority=flows.UE_FLOW_PRIORITY + 1,
cookie=rule_num,
output_port=OFPP_LOCAL)
flows.add_output_flow(datapath,
self.main_tbl_num,
match_tcp_8080_vlan,
actions,
priority=flows.UE_FLOW_PRIORITY + 1,
cookie=rule_num,
output_port=OFPP_LOCAL)
# TODO cleanup, make this a default rule in the ue_mac table
ue_tbl = 0
ue_next_tbl = 1
# Allows traffic back from the flask server
match = MagmaMatch(in_port=OFPP_LOCAL)
actions = [
parser.NXActionResubmitTable(table_id=self._mac_rewrite_scratch)
]
flows.add_resubmit_next_service_flow(datapath,
ue_tbl,
match,
actions=actions,
priority=flows.DEFAULT_PRIORITY,
resubmit_table=ue_next_tbl)
match = MagmaMatch(imsi=encode_imsi(imsi),
eth_type=ether_types.ETH_TYPE_IP,
ip_proto=IPPROTO_TCP,
direction=Direction.IN,
in_port=OFPP_LOCAL)
flows.add_resubmit_next_service_flow(datapath,
self.main_tbl_num,
match, [],
priority=flows.DEFAULT_PRIORITY,
cookie=rule_num,
resubmit_table=self._egress_table)
# Mac doesn't matter as we rewrite it anwyays
mac_addr = '01:02:03:04:05:06'
if self._arp_contoller or self._arpd_controller_fut.done():
if not self._arp_contoller:
self._arp_contoller = self._arpd_controller_fut.result()
self._arp_contoller.set_incoming_arp_flows(datapath, internal_ip,
mac_addr)
# Drop all other traffic that doesn't match
match = MagmaMatch(imsi=encode_imsi(imsi))
flows.add_drop_flow(datapath,
self.main_tbl_num,
match, [],
priority=flows.MINIMUM_PRIORITY + 1,
cookie=rule_num)
def _install_server_flows(self, datapath, imsi, ip_addr, rule, rule_num,
rule_version, priority):
"""
Install the redirect flows to redirect all HTTP traffic to the captive
portal and to drop all of the rest.
"""
self._install_scratch_table_flows(datapath, imsi, rule, rule_num,
rule_version, priority)
self._install_not_processed_flows(datapath, imsi, ip_addr, rule,
rule_num, priority)
def _install_url_bypass_flows(self,
datapath,
loop,
imsi,
rule,
rule_num,
rule_version,
priority,
ue_ip=None):
"""
Resolve DNS queries to get the ip address of redirect url, this is done
to allow traffic to safely pass through as we want subscribers to have
full access to the url they are redirected to.
First check cache for redirect url, if not in cache submit a DNS query
"""
redirect_addr_host = urlsplit(rule.redirect.server_address).netloc
cached_ips = self._dns_cache.get(redirect_addr_host)
if cached_ips is not None:
self.logger.debug(
"DNS cache hit for {}, entry expires in {} sec".format(
redirect_addr_host,
self._dns_cache.ttl(redirect_addr_host)))
self._install_ipv4_bypass_flows(datapath, imsi, rule, rule_num,
rule_version, priority, cached_ips,
ue_ip)
return
resolver = aiodns.DNSResolver(timeout=self.DNS_TIMEOUT_SECS, loop=loop)
query = resolver.query(redirect_addr_host, 'A')
def add_flows(dns_resolve_future):
"""
Callback for when DNS query is resolved, adds the bypass flows
"""
try:
ips = [entry.host for entry in dns_resolve_future.result()]
ttl = min(entry.ttl for entry in dns_resolve_future.result())
except aiodns.error.DNSError as err:
self.logger.error("Error: ip lookup for {}: {}".format(
redirect_addr_host, err))
return
self._dns_cache.get(redirect_addr_host, lambda: ips, max_age=ttl)
self._install_ipv4_bypass_flows(datapath, imsi, rule, rule_num,
rule_version, priority, ips, ue_ip)
asyncio.ensure_future(query, loop=loop).add_done_callback(add_flows)
def _install_ipv4_bypass_flows(self,
datapath,
imsi,
rule,
rule_num,
rule_version,
priority,
ips,
ue_ip=None):
"""
Installs flows for traffic that is allowed to pass through for
subscriber who has redirection enabled. Allow access to all passed ips.
Allow UDP traffic(for DNS queries), traffic to/from redirection address
"""
parser = datapath.ofproto_parser
of_note = parser.NXActionNote(list(rule.id.encode()))
actions = [
of_note,
]
actions += self._load_rule_actions(parser, rule_num, rule_version)
matches = []
uplink_ip_match = {}
downlink_ip_match = {}
if ue_ip != None:
uplink_ip_match['ipv4_src'] = ue_ip
downlink_ip_match['ipv4_dst'] = ue_ip
for ip in ips:
matches.append(
MagmaMatch(eth_type=ether_types.ETH_TYPE_IP,
direction=Direction.OUT,
ipv4_dst=ip,
imsi=encode_imsi(imsi),
**uplink_ip_match))
matches.append(
MagmaMatch(eth_type=ether_types.ETH_TYPE_IP,
direction=Direction.IN,
ipv4_src=ip,
imsi=encode_imsi(imsi),
**downlink_ip_match))
for match in matches:
flows.add_resubmit_next_service_flow(
datapath,
self.main_tbl_num,
match,
actions,
priority=priority + 1,
cookie=rule_num,
hard_timeout=rule.hard_timeout,
resubmit_table=self.next_table)
def _install_dns_flows(self, datapath, imsi, rule, rule_num, rule_version,
priority):
"""
Installs flows that allow DNS queries to path through.
"""
parser = datapath.ofproto_parser
of_note = parser.NXActionNote(list(rule.id.encode()))
actions = [
of_note,
]
actions += self._load_rule_actions(parser, rule_num, rule_version)
matches = []
# Install UDP flows for DNS
matches.append(
MagmaMatch(eth_type=ether_types.ETH_TYPE_IP,
ip_proto=IPPROTO_UDP,
udp_src=53,
direction=Direction.IN,
imsi=encode_imsi(imsi)))
matches.append(
MagmaMatch(eth_type=ether_types.ETH_TYPE_IP,
ip_proto=IPPROTO_UDP,
udp_dst=53,
direction=Direction.OUT,
imsi=encode_imsi(imsi)))
# Install TCP flows for DNS
matches.append(
MagmaMatch(eth_type=ether_types.ETH_TYPE_IP,
ip_proto=IPPROTO_TCP,
tcp_src=53,
direction=Direction.IN,
imsi=encode_imsi(imsi)))
matches.append(
MagmaMatch(eth_type=ether_types.ETH_TYPE_IP,
ip_proto=IPPROTO_TCP,
tcp_dst=53,
direction=Direction.OUT,
imsi=encode_imsi(imsi)))
for match in matches:
flows.add_resubmit_next_service_flow(
datapath,
self.main_tbl_num,
match,
actions,
priority=priority,
cookie=rule_num,
hard_timeout=rule.hard_timeout,
resubmit_table=self.next_table)
def _save_redirect_entry(self, ip_addr, redirect_info):
"""
Saves the redirect entry in Redis.
Throws:
RedirectException: on error
"""
try:
# Verify if ip_addr is in the correct format, and also
# normalize the variants of the address into a single format
ip_str = str(ipaddress.ip_address(ip_addr))
except ValueError as exp:
raise RedirectException(exp)
try:
self._redirect_dict[ip_str] = redirect_info
except RedisError as exp:
raise RedirectException(exp)
self.logger.info("Saved redirect rule for %s in Redis" % ip_str)
def deactivate_flow_for_rule(self, datapath, imsi, rule_num):
"""
Deactivate a specific rule using the flow cookie for a subscriber
"""
cookie, mask = (rule_num, flows.OVS_COOKIE_MATCH_ALL)
match = MagmaMatch(imsi=encode_imsi(imsi))
flows.delete_flow(datapath,
self._scratch_tbl_num,
match,
cookie=cookie,
cookie_mask=mask)
def deactivate_flows_for_subscriber(self, datapath, imsi):
"""
Deactivate all rules for a subscriber
"""
flows.delete_flow(datapath, self._scratch_tbl_num,
MagmaMatch(imsi=encode_imsi(imsi)))
def _load_rule_actions(self, parser, rule_num, rule_version):
return [
parser.NXActionRegLoad2(dst='reg2', value=rule_num),
parser.NXActionRegLoad2(dst=RULE_VERSION_REG, value=rule_version),
]
def make_auctions_app(global_conf,
redis_url='redis://localhost:7777/0',
external_couch_url='http://localhost:5000/auction',
internal_couch_url='http://localhost:9000/',
proxy_internal_couch_url='http://localhost:9000/',
auctions_db='auctions',
hash_secret_key='',
timezone='Europe/Kiev',
preferred_url_scheme='http',
debug=False,
auto_build=False,
event_source_connection_limit=1000):
"""
[app:main]
use = egg:openprocurement.auction#auctions_server
redis_url = redis://:passwod@localhost:1111/0
external_couch_url = http://localhost:1111/auction
internal_couch_url = http://localhost:9011/
auctions_db = auction
timezone = Europe/Kiev
"""
auctions_server.proxy_connection_pool = ConnectionPool(factory=Connection,
max_size=20,
backend="gevent")
auctions_server.proxy_mappings = Memoizer({})
auctions_server.event_sources_pool = deque([])
auctions_server.config['PREFERRED_URL_SCHEME'] = preferred_url_scheme
auctions_server.config['REDIS_URL'] = redis_url
auctions_server.config['event_source_connection_limit'] = int(
event_source_connection_limit)
auctions_server.config['EXT_COUCH_DB'] = urljoin(external_couch_url,
auctions_db)
auctions_server.add_url_rule('/' + auctions_db + '/<path:path>',
'couch_server_proxy',
couch_server_proxy,
methods=['GET'])
auctions_server.add_url_rule('/' + auctions_db + '/',
'couch_server_proxy',
couch_server_proxy,
methods=['GET'],
defaults={'path': ''})
auctions_server.add_url_rule('/' + auctions_db + '_secured/<path:path>',
'auth_couch_server_proxy',
auth_couch_server_proxy,
methods=['GET'])
auctions_server.add_url_rule('/' + auctions_db + '_secured/',
'auth_couch_server_proxy',
auth_couch_server_proxy,
methods=['GET'],
defaults={'path': ''})
auctions_server.config['INT_COUCH_URL'] = internal_couch_url
auctions_server.config['PROXY_COUCH_URL'] = proxy_internal_couch_url
auctions_server.config['COUCH_DB'] = auctions_db
auctions_server.config['TIMEZONE'] = tz(timezone)
auctions_server.redis = Redis(auctions_server)
auctions_server.couch_server = Server(
auctions_server.config.get('INT_COUCH_URL'),
session=Session(retry_delays=range(10)))
if auctions_server.config['COUCH_DB'] not in auctions_server.couch_server:
auctions_server.couch_server.create(auctions_server.config['COUCH_DB'])
auctions_server.db = auctions_server.couch_server[
auctions_server.config['COUCH_DB']]
auctions_server.config['HASH_SECRET_KEY'] = hash_secret_key
sync_design(auctions_server.db)
auctions_server.config['ASSETS_DEBUG'] = True if debug else False
assets.auto_build = True if auto_build else False
return auctions_server
def mock_auctions_server(request, mocker):
params = getattr(request, 'param', {})
server_config_redis = params.get('server_config_redis', DEFAULT)
connection_limit = params.get('connection_limit', DEFAULT)
get_mapping = params.get('get_mapping', DEFAULT)
proxy_path = params.get('proxy_path', DEFAULT)
event_sources_pool = params.get('event_sources_pool', DEFAULT)
proxy_connection_pool = params.get('proxy_connection_pool', DEFAULT)
stream_proxy = params.get('stream_proxy', DEFAULT)
db = params.get('db', DEFAULT)
request_headers = params.get('request_headers', [])
request_url = params.get('request_url', DEFAULT)
redirect_url = params.get('redirect_url', DEFAULT)
abort = params.get('abort', DEFAULT)
class AuctionsServerAttributesContainer(object):
logger = NotImplemented
proxy_mappings = NotImplemented
config = NotImplemented
event_sources_pool = NotImplemented
proxy_connection_pool = NotImplemented
get_mapping = NotImplemented
db = NotImplemented
request_headers = NotImplemented
class Request(object):
headers = NotImplemented
environ = NotImplemented
url = NotImplemented
class Config(object):
__getitem__ = NotImplemented
def config_getitem(item):
if item == 'REDIS':
return server_config_redis
elif item == 'event_source_connection_limit':
return connection_limit
else:
raise KeyError
mock_path_info = MagicMock()
def environ_setitem(item, value):
if item == 'PATH_INFO':
mock_path_info(value)
return value
else:
raise KeyError
mocker.patch.object(auctions_server_module, 'get_mapping', get_mapping)
patch_pysse = mocker.patch.object(auctions_server_module,
'PySse',
spec_set=PySse)
patch_add_message = patch_pysse.return_value.add_message
patch_request = mocker.patch.object(auctions_server_module,
'request',
spec_set=Request)
patch_request.environ.__setitem__.side_effect = environ_setitem
patch_request.headers = request_headers
patch_request.url = request_url
patch_redirect = mocker.patch.object(auctions_server_module,
'redirect',
return_value=redirect_url)
patch_abort = mocker.patch.object(auctions_server_module,
'abort',
return_value=abort)
patch_StreamProxy = \
mocker.patch.object(auctions_server_module, 'StreamProxy',
return_value=stream_proxy)
auctions_server = NonCallableMock(
spec_set=AuctionsServerAttributesContainer)
logger = MagicMock(spec_set=frontend.logger)
proxy_mappings = MagicMock(spec_set=Memoizer({}))
proxy_mappings.get.return_value = proxy_path
config = MagicMock(spec_set=Config)
config.__getitem__.side_effect = config_getitem
auctions_server.logger = logger
auctions_server.proxy_mappings = proxy_mappings
auctions_server.config = config
auctions_server.event_sources_pool = event_sources_pool
auctions_server.proxy_connection_pool = proxy_connection_pool
auctions_server.db = db
mocker.patch.object(auctions_server_module, 'auctions_server',
auctions_server)
return {
'server': auctions_server,
'proxy_mappings': proxy_mappings,
'mock_path_info': mock_path_info,
'patch_StreamProxy': patch_StreamProxy,
'patch_redirect': patch_redirect,
'patch_abort': patch_abort,
'patch_PySse': patch_pysse,
'patch_add_message': patch_add_message
}
class ep_risk:
#example folio_dict: {’JNJ’:10**6, ’AMGN’:10**6, ’UNH’:10**6, ’MYL’:10**6, ’A’: 10**6, ’XLV’: -(5*10**6)}
def __init__(self, folio_dict):
self.port = folio_dict
memo = Memoizer({})
#caches the data pulls for up to 60 seconds
@memo(max_age=60)
def get_data(self, end_date="01/15/15", window_length=700):
last_date = datetime.datetime.strptime(end_date, "%x")
first_date = last_date - datetime.timedelta(days=window_length)
s = first_date.__str__()
l = last_date.__str__()
#Create List of Dataframes for all stocks
df_s = [self.prices(tick, s, l) for tick in self.port.keys()]
#Merge that List of Dataframes
df_final = df_s[0]
for i in range(1, len(df_s)):
df_final = DataFrame.join(df_final, df_s[i])
return df_final
def prices(self, ticker, start_string, end_string):
#holds key:Date Val:Adj Close
data_hash = dict()
#Create dictionary from yahoo data
tick_test = ystockquote.get_historical_prices(ticker, start_string,
end_string)
#fill data_hash
for date in tick_test.keys():
data_hash[datetime.date(int(date[:4]), int(date[5:7]), int(
date[8:]))] = float(tick_test[date]['Adj Close'])
#make datahash into dataframe called df
df = DataFrame.from_dict(data_hash, orient='index', dtype=float)
#rename first column
df.columns = [ticker]
#sort in date order
df = df.sort(ascending=True, inplace=False)
return df
def data_stats(self, n_days=2, end_date="01/15/15", window_length=700):
price_data = self.get_data(end_date=end_date,
window_length=window_length)
#calculate n-day returns
returns = price_data.pct_change(periods=n_days,
fill_method='pad',
limit=None,
freq=None)
#apply log transformatinon
returns = 1 + returns
returns = returns.applymap(m.log)
#calculate means and volatilities
means = returns.mean(axis=0)
vols = returns.std(axis=0)
#calc correlations
correlation_matrix = returns.corr()
#returns lambda_plus for PCA approaches
lambda_plus = (1 + m.sqrt(
float(len(self.port.keys())) / max([
returns.ix[:, i].count()
for i in range(0, len(self.port.keys()))
])))**2
return correlation_matrix, means, vols, lambda_plus
def pca_risk(self,
alpha,
n_days=2,
end_date="01/15/15",
window_length=700):
correlation_matrix, means, vols, lambda_plus = self.data_stats(
n_days=n_days, end_date=end_date, window_length=window_length)
#return eigenvectors and values (eigenvectors are columns)
e_vals, e_vecs = np.linalg.eigh(correlation_matrix)
#sort (largest to smallest)
indices = e_vals.argsort()[::-1]
sorted_evals = [e_vals[i] for i in indices]
sorted_evecs = e_vecs[:, indices]
significant_num = len(
[value for value in sorted_evals if value > lambda_plus])
cleaned = self.clean_correlation_matrix(significant_num, sorted_evals,
sorted_evecs)
cleaned_df = DataFrame(data=cleaned,
index=correlation_matrix.columns.tolist(),
columns=correlation_matrix.columns.tolist(),
dtype=float,
copy=True)
var, es = self.mc_risk(alpha, cleaned_df, means, vols)
return var, es
def unclean_risk(self,
alpha,
n_days=2,
end_date="01/15/15",
window_length=700):
correlation_matrix, means, vols, lambda_plus = self.data_stats(
n_days=n_days, end_date=end_date, window_length=window_length)
var, es = self.mc_risk(alpha, correlation_matrix, means, vols)
return var, es
def historical_risk(self,
alpha,
n_days=2,
end_date="01/15/15",
window_length=700):
price_data = self.get_data(end_date=end_date,
window_length=window_length)
returns = price_data.pct_change(periods=n_days,
fill_method='pad',
limit=None,
freq=None)
return self.port_rets(returns, alpha)
def mc_risk(self, alpha, corr_df, means, vols):
#generate 50,000 samples from T dist
joint_scenarios = self.multivariatet_sampler(
np.zeros(len(self.port.keys())), corr_df, 4, 50000)
#scale these scenarios by multiplying by scaling vol then shifting by mean
joint_scenarios = m.sqrt(1 / 2.) * np.dot(
joint_scenarios, np.diag(vols)) + np.tile(means, (50000, 1))
#calculate portfolio returns
var_alpha, es_alpha = self.port_rets(joint_scenarios, alpha)
return var_alpha, es_alpha
def port_rets(self, scenarios, alpha):
port_vec = np.array(self.port.values())
sim_returns = np.dot(scenarios, port_vec)
var_alpha = np.percentile(sim_returns, alpha)
es_alpha = np.mean([v for v in sim_returns.tolist() if v < var_alpha])
return var_alpha, es_alpha
def clean_correlation_matrix(self, significant_num, lambdas, vecs):
dimension = len(lambdas)
matrix = np.zeros((dimension, dimension))
for i in range(0, dimension):
for j in range(0, dimension):
ro = 0
for k in range(0, significant_num):
ro += lambdas[k] * vecs[:, k][i] * vecs[:, k][j]
if (i == j):
eii_sq = 1
for k in range(0, significant_num):
eii_sq = eii_sq - lambdas[k] * vecs[:,
k][i] * vecs[:,
k][i]
ro = ro + eii_sq
matrix[i, j] = ro
return matrix
#Known Function to generate samples from multivariate t in python (Kenny Chowdhary - PhD)
def multivariatet_sampler(self, mu, Sigma, N, M):
'''
Output:
Produce M samples of d-dimensional multivariate t distribution
Input:
mu = mean (d dimensional numpy array or scalar)
Sigma = scale matrix (dxd numpy array)
N = degrees of freedom
M = # of samples to produce
'''
d = len(Sigma)
g = np.tile(np.random.gamma(N / 2., 2. / N, M), (d, 1)).T
Z = np.random.multivariate_normal(np.zeros(d), Sigma, M)
return mu + Z / np.sqrt(g)
def setUp(self):
self.memoizer = Memoizer()
class RedirectionManager:
"""
RedirectionManager
The redirection manager handles subscribers who have redirection enabled,
it adds the flows into ovs for redirecting user to the redirection server
"""
DNS_TIMEOUT_SECS = 15
REDIRECT_NOT_PROCESSED = REG_ZERO_VAL
REDIRECT_PROCESSED = 0x1
RedirectRequest = namedtuple(
'RedirectRequest',
['imsi', 'ip_addr', 'rule', 'rule_num', 'priority'],
)
def __init__(self, bridge_ip, logger, main_tbl_num, next_table,
scratch_table_num, session_rule_version_mapper):
self._bridge_ip = bridge_ip
self.logger = logger
self.main_tbl_num = main_tbl_num
self.next_table = next_table
self._scratch_tbl_num = scratch_table_num
self._redirect_dict = RedirectDict()
self._dns_cache = Memoizer({})
self._redirect_port = get_service_config_value(
'redirectd', 'http_port', 8080)
self._session_rule_version_mapper = session_rule_version_mapper
def handle_redirection(self, datapath, loop, redirect_request):
"""
Depending on redirection server address type install redirection rules
"""
imsi = redirect_request.imsi
ip_addr = redirect_request.ip_addr
rule = redirect_request.rule
rule_num = redirect_request.rule_num
priority = redirect_request.priority
# TODO IMPORTANT check that redirectd service is running, as its a
# dynamic service its not on by default. Will save you some sanity :)
# TODO figure out what to do with SIP_URI
if rule.redirect.address_type == rule.redirect.SIP_URI:
raise RedirectException("SIP_URIs redirection isn't setup")
if rule.redirect.address_type == rule.redirect.IPv6:
raise RedirectException("No ipv6 support, so no ipv6 redirect")
self._save_redirect_entry(ip_addr, rule.redirect)
self._install_redirect_flows(datapath, loop, imsi, rule, rule_num,
priority)
return
def _install_redirect_flows(self, datapath, loop, imsi, rule, rule_num,
priority):
"""
Add flows to forward traffic to the redirection server.
1) Intercept tcp traffic to the web to the redirection server, which
completes the tcp handshake. This is done by adding an OVS flow
with a learn action (flow catches inbound tcp packets, while learn
action creates another flow that sends packets back from server)
2) Add flows to allow UDP traffic so DNS queries can go through.
Finally add flows with a higher priority that allow traffic to and
from the address provided in redirect rule.
"""
if rule.redirect.address_type == rule.redirect.URL:
self._install_url_bypass_flows(datapath, loop, imsi, rule,
rule_num, priority)
elif rule.redirect.address_type == rule.redirect.IPv4:
self._install_ipv4_bypass_flows(datapath, imsi, rule,
rule_num, priority,
[rule.redirect.server_address])
self._install_dns_flows(datapath, imsi, rule, rule_num, priority)
self._install_server_flows(datapath, imsi, rule, rule_num, priority)
def _install_scratch_table_flows(self, datapath, imsi, rule, rule_num,
priority):
"""
The flow action for subscribers that need to be redirected does 2 things
* Forward requests from subscriber to the internal http server
* Instantiate a flow that matches response packets from the server
and sends them back to subscriber
Match: incoming tcp traffic with port 80, direction out
Action:
1) Set reg2 to rule_num
2) Set ip dst to server ip
3) Output to table 20
4) Apply LearnAction:
LearnAction(adds new flow for every pkt flow that hits this rule)
1) Match ip packets
2) Match tcp protocol
3) Match packets from LOCAL port
4) Match ip src = server ip
5) Match ip dst = current flow ip src
6) Match tcp src = current flow tcp dst
7) Match tcp dst = current flow tcp src
8) Load ip src = current flow ip dst
9) Output through gtp0
"""
parser = datapath.ofproto_parser
match_http = MagmaMatch(
eth_type=ether_types.ETH_TYPE_IP, ip_proto=IPPROTO_TCP,
tcp_dst=80, imsi=encode_imsi(imsi), direction=Direction.OUT)
of_note = parser.NXActionNote(list(rule.id.encode()))
actions = [
parser.NXActionLearn(
table_id=self.main_tbl_num,
priority=priority,
cookie=rule_num,
specs=[
parser.NXFlowSpecMatch(
src=ether_types.ETH_TYPE_IP, dst=('eth_type_nxm', 0),
n_bits=16
),
parser.NXFlowSpecMatch(
src=IPPROTO_TCP, dst=('ip_proto_nxm', 0), n_bits=8
),
parser.NXFlowSpecMatch(
src=Direction.IN,
dst=(DIRECTION_REG, 0),
n_bits=32
),
parser.NXFlowSpecMatch(
src=int(ipaddress.IPv4Address(self._bridge_ip)),
dst=('ipv4_src_nxm', 0),
n_bits=32
),
parser.NXFlowSpecMatch(
src=('ipv4_src_nxm', 0),
dst=('ipv4_dst_nxm', 0),
n_bits=32
),
parser.NXFlowSpecMatch(
src=('tcp_src_nxm', 0),
dst=('tcp_dst_nxm', 0),
n_bits=16
),
parser.NXFlowSpecMatch(
src=self._redirect_port,
dst=('tcp_src_nxm', 0),
n_bits=16
),
parser.NXFlowSpecMatch(
src=encode_imsi(imsi),
dst=(IMSI_REG, 0),
n_bits=64
),
parser.NXFlowSpecLoad(
src=('ipv4_dst_nxm', 0),
dst=('ipv4_src_nxm', 0),
n_bits=32
),
parser.NXFlowSpecLoad(
src=80,
dst=('tcp_src_nxm', 0),
n_bits=16
),
# Learn doesn't support resubmit to table, so send directly
parser.NXFlowSpecOutput(
src=('in_port', 0), dst="", n_bits=16
),
]
),
parser.NXActionRegLoad2(dst=SCRATCH_REGS[0],
value=self.REDIRECT_PROCESSED),
parser.OFPActionSetField(ipv4_dst=self._bridge_ip),
parser.OFPActionSetField(tcp_dst=self._redirect_port),
of_note,
]
actions += self._load_rule_actions(parser, rule_num, imsi, rule.id)
flows.add_resubmit_current_service_flow(
datapath, self._scratch_tbl_num, match_http, actions,
priority=priority, cookie=rule_num, hard_timeout=rule.hard_timeout,
resubmit_table=self.main_tbl_num)
match = MagmaMatch(imsi=encode_imsi(imsi))
action = []
flows.add_drop_flow(datapath, self._scratch_tbl_num, match, action,
priority=flows.MINIMUM_PRIORITY + 1,
cookie=rule_num)
def _install_not_processed_flows(self, datapath, imsi, rule, rule_num,
priority):
"""
Redirect all traffic to the scratch table to only allow redirected
http traffic to go through, the rest will be dropped. reg0 is used as
a boolean to know whether the drop rule was processed.
"""
parser = datapath.ofproto_parser
of_note = parser.NXActionNote(list(rule.id.encode()))
match = MagmaMatch(imsi=encode_imsi(imsi),
direction=Direction.OUT,
reg0=self.REDIRECT_NOT_PROCESSED,
eth_type=ether_types.ETH_TYPE_IP)
action = [of_note]
flows.add_resubmit_current_service_flow(
datapath, self.main_tbl_num, match, action, priority=priority,
cookie=rule_num, hard_timeout=rule.hard_timeout,
resubmit_table=self._scratch_tbl_num)
match = MagmaMatch(imsi=encode_imsi(imsi),
direction=Direction.OUT,
reg0=self.REDIRECT_PROCESSED)
action = [of_note]
flows.add_resubmit_next_service_flow(
datapath, self.main_tbl_num, match, action, priority=priority,
cookie=rule_num, hard_timeout=rule.hard_timeout,
resubmit_table=self.next_table)
def _install_server_flows(self, datapath, imsi, rule, rule_num, priority):
"""
Install the redirect flows to redirect all HTTP traffic to the captive
portal and to drop all of the rest.
"""
self._install_scratch_table_flows(datapath, imsi, rule, rule_num,
priority)
self._install_not_processed_flows(datapath, imsi, rule, rule_num,
priority)
def _install_url_bypass_flows(self, datapath, loop, imsi, rule, rule_num,
priority):
"""
Resolve DNS queries to get the ip address of redirect url, this is done
to allow traffic to safely pass through as we want subscribers to have
full access to the url they are redirected to.
First check cache for redirect url, if not in cache submit a DNS query
"""
redirect_addr_host = urlsplit(rule.redirect.server_address).netloc
cached_ips = self._dns_cache.get(redirect_addr_host)
if cached_ips is not None:
self.logger.debug(
"DNS cache hit for {}, entry expires in {} sec".format(
redirect_addr_host, self._dns_cache.ttl(redirect_addr_host)
)
)
self._install_ipv4_bypass_flows(datapath, imsi, rule, rule_num,
priority, cached_ips)
return
resolver = aiodns.DNSResolver(timeout=self.DNS_TIMEOUT_SECS, loop=loop)
query = resolver.query(redirect_addr_host, 'A')
def add_flows(dns_resolve_future):
"""
Callback for when DNS query is resolved, adds the bypass flows
"""
try:
ips = [entry.host for entry in dns_resolve_future.result()]
ttl = min(entry.ttl for entry in dns_resolve_future.result())
except aiodns.error.DNSError as err:
self.logger.error("Error: ip lookup for {}: {}".format(
redirect_addr_host, err))
return
self._dns_cache.get(redirect_addr_host, lambda: ips, max_age=ttl)
self._install_ipv4_bypass_flows(datapath, imsi, rule, rule_num,
priority, ips)
asyncio.ensure_future(query, loop=loop).add_done_callback(add_flows)
def _install_ipv4_bypass_flows(self, datapath, imsi, rule, rule_num,
priority, ips):
"""
Installs flows for traffic that is allowed to pass through for
subscriber who has redirection enabled. Allow access to all passed ips.
Allow UDP traffic(for DNS queries), traffic to/from redirection address
"""
parser = datapath.ofproto_parser
of_note = parser.NXActionNote(list(rule.id.encode()))
actions = [
of_note,
]
actions += self._load_rule_actions(parser, rule_num, imsi, rule.id)
matches = []
for ip in ips:
matches.append(MagmaMatch(
eth_type=ether_types.ETH_TYPE_IP, direction=Direction.OUT,
ipv4_dst=ip, imsi=encode_imsi(imsi)
))
matches.append(MagmaMatch(
eth_type=ether_types.ETH_TYPE_IP, direction=Direction.IN,
ipv4_src=ip, imsi=encode_imsi(imsi)
))
for match in matches:
flows.add_resubmit_next_service_flow(
datapath, self.main_tbl_num, match, actions,
priority=priority + 1, cookie=rule_num,
hard_timeout=rule.hard_timeout, resubmit_table=self.next_table)
def _install_dns_flows(self, datapath, imsi, rule, rule_num, priority):
"""
Installs flows that allow DNS queries to path through.
"""
parser = datapath.ofproto_parser
of_note = parser.NXActionNote(list(rule.id.encode()))
actions = [
of_note,
]
actions += self._load_rule_actions(parser, rule_num, imsi, rule.id)
matches = []
# Install UDP flows for DNS
matches.append(MagmaMatch(eth_type=ether_types.ETH_TYPE_IP,
ip_proto=IPPROTO_UDP,
udp_src=53,
direction=Direction.IN,
imsi=encode_imsi(imsi)))
matches.append(MagmaMatch(eth_type=ether_types.ETH_TYPE_IP,
ip_proto=IPPROTO_UDP,
udp_dst=53,
direction=Direction.OUT,
imsi=encode_imsi(imsi)))
# Install TCP flows for DNS
matches.append(MagmaMatch(eth_type=ether_types.ETH_TYPE_IP,
ip_proto=IPPROTO_TCP,
tcp_src=53,
direction=Direction.IN,
imsi=encode_imsi(imsi)))
matches.append(MagmaMatch(eth_type=ether_types.ETH_TYPE_IP,
ip_proto=IPPROTO_TCP,
tcp_dst=53,
direction=Direction.OUT,
imsi=encode_imsi(imsi)))
for match in matches:
flows.add_resubmit_next_service_flow(
datapath, self.main_tbl_num, match, actions, priority=priority,
cookie=rule_num, hard_timeout=rule.hard_timeout,
resubmit_table=self.next_table)
def _save_redirect_entry(self, ip_addr, redirect_info):
"""
Saves the redirect entry in Redis.
Throws:
RedirectException: on error
"""
try:
# Verify if ip_addr is in the correct format, and also
# normalize the variants of the address into a single format
ip_str = str(ipaddress.ip_address(ip_addr))
except ValueError as exp:
raise RedirectException(exp)
try:
self._redirect_dict[ip_str] = redirect_info
except RedisError as exp:
raise RedirectException(exp)
self.logger.info("Saved redirect rule for %s in Redis" % ip_str)
def deactivate_flow_for_rule(self, datapath, imsi, rule_num):
"""
Deactivate a specific rule using the flow cookie for a subscriber
"""
cookie, mask = (rule_num, flows.OVS_COOKIE_MATCH_ALL)
match = MagmaMatch(imsi=encode_imsi(imsi))
flows.delete_flow(datapath, self._scratch_tbl_num, match,
cookie=cookie, cookie_mask=mask)
def deactivate_flows_for_subscriber(self, datapath, imsi):
"""
Deactivate all rules for a subscriber
"""
flows.delete_flow(datapath, self._scratch_tbl_num,
MagmaMatch(imsi=encode_imsi(imsi)))
def _load_rule_actions(self, parser, rule_num, imsi, rule_id):
version = self._session_rule_version_mapper.get_version(imsi, rule_id)
return [
parser.NXActionRegLoad2(dst='reg2', value=rule_num),
parser.NXActionRegLoad2(dst=RULE_VERSION_REG, value=version),
]
from memoize import Memoizer
memoized = Memoizer()
@memoized
def fib(n):
if n <= 1: return n
return fib(n - 1) + fib(n - 2)
@memoized
def choose(n, k):
if k < 0 or k > n: return 0
if k == 0 or k == n: return 1
return choose(n - 1, k - 1) + choose(n - 1, k)
class MemoizeTestCase(SimpleTestCase):
def setUp(self):
self.memoizer = Memoizer()
def test_00_set(self):
self.memoizer.set('hi', 'hello')
assert self.memoizer.get('hi') == 'hello'
def test_01_add(self):
self.memoizer.add('hi', 'hello')
assert self.memoizer.get('hi') == 'hello'
self.memoizer.add('hi', 'foobar')
assert self.memoizer.get('hi') == 'hello'
def test_02_delete(self):
self.memoizer.set('hi', 'hello')
self.memoizer.delete('hi')
assert self.memoizer.get('hi') is self.memoizer.default_cache_value
def test_03_delete_many(self):
self.memoizer.set('hi', 'hello')
self.memoizer.set('bye', 'goodbye')
self.memoizer.delete_many('hi', 'bye')
assert self.memoizer.get('hi') is self.memoizer.default_cache_value
assert self.memoizer.get('bye') is self.memoizer.default_cache_value
def test_04_clear(self):
self.memoizer.set('hi', 'hello')
self.memoizer.set('bye', 'goodbye')
self.memoizer.clear()
assert self.memoizer.get('hi') is self.memoizer.default_cache_value
assert self.memoizer.get('bye') is self.memoizer.default_cache_value
def test_06_memoize(self):
@self.memoizer.memoize(5)
def big_foo(a, b, func=lambda x: x):
func(a)
return a + b
test_func = MagicMock()
big_foo(5, 2, func=test_func)
# Cached call
big_foo(5, 2, func=test_func)
# We are expecting only 1 call here as second one should be cached
assert test_func.call_count == 1, "Count: {}".format(
test_func.call_count)
# different arguments so we should have additional call
big_foo(5, 3, func=test_func)
assert test_func.call_count == 2, "Count: {}".format(
test_func.call_count)
# Expired
now = datetime.datetime.utcfromtimestamp(time.time())
with freeze_time(now) as frozen_datetime:
frozen_datetime.tick(delta=datetime.timedelta(seconds=6))
# Cache should be expired so we have increasing amount of calls
big_foo(5, 2, func=test_func)
assert test_func.call_count == 3, "Count: {}".format(
test_func.call_count)
big_foo(5, 3, func=test_func)
assert test_func.call_count == 4, "Count: {}".format(
test_func.call_count)
def test_06a_memoize(self):
@self.memoizer.memoize(50)
def big_foo(a, b):
return a + b + random.randrange(0, 100000)
result = big_foo(5, 2)
assert big_foo(5, 2) == result
def test_07_delete_memoize(self):
@self.memoizer.memoize(5)
def big_foo(a, b):
return a + b + random.randrange(0, 100000)
result = big_foo(5, 2)
result2 = big_foo(5, 3)
assert big_foo(5, 2) == result
assert big_foo(5, 2) == result
assert big_foo(5, 3) != result
assert big_foo(5, 3) == result2
self.memoizer.delete_memoized(big_foo)
assert big_foo(5, 2) != result
assert big_foo(5, 3) != result2
def test_07b_delete_memoized_verhash(self):
@self.memoizer.memoize(5)
def big_foo(a, b):
return a + b + random.randrange(0, 100000)
result = big_foo(5, 2)
result2 = big_foo(5, 3)
assert big_foo(5, 2) == result
assert big_foo(5, 2) == result
assert big_foo(5, 3) != result
assert big_foo(5, 3) == result2
self.memoizer.delete_memoized_verhash(big_foo)
_fname, _fname_instance = function_namespace(big_foo)
version_key = self.memoizer._memvname(_fname)
assert (self.memoizer.get(version_key) is
self.memoizer.default_cache_value)
assert big_foo(5, 2) != result
assert big_foo(5, 3) != result2
assert (self.memoizer.get(version_key)
is not self.memoizer.default_cache_value)
def test_08_delete_memoize(self):
@self.memoizer.memoize()
def big_foo(a, b):
return a + b + random.randrange(0, 100000)
result_a = big_foo(5, 1)
result_b = big_foo(5, 2)
assert big_foo(5, 1) == result_a
assert big_foo(5, 2) == result_b
self.memoizer.delete_memoized(big_foo, 5, 2)
assert big_foo(5, 1) == result_a
assert big_foo(5, 2) != result_b
# Cleanup bigfoo 5,1 5,2 or it might conflict with
# following run if it also uses memcache
self.memoizer.delete_memoized(big_foo, 5, 2)
self.memoizer.delete_memoized(big_foo, 5, 1)
def test_09_args_memoize(self):
@self.memoizer.memoize()
def big_foo(a, b):
return sum(a) + sum(b) + random.randrange(0, 100000)
result_a = big_foo([5, 3, 2], [1])
result_b = big_foo([3, 3], [3, 1])
assert big_foo([5, 3, 2], [1]) == result_a
assert big_foo([3, 3], [3, 1]) == result_b
self.memoizer.delete_memoized(big_foo, [5, 3, 2], [1])
assert big_foo([5, 3, 2], [1]) != result_a
assert big_foo([3, 3], [3, 1]) == result_b
# Cleanup bigfoo 5,1 5,2 or it might conflict with
# following run if it also uses memcache
self.memoizer.delete_memoized(big_foo, [5, 3, 2], [1])
self.memoizer.delete_memoized(big_foo, [3, 3], [1])
def test_10_kwargs_memoize(self):
@self.memoizer.memoize()
def big_foo(a, b=None):
return a + sum(b.values()) + random.randrange(0, 100000)
result_a = big_foo(1, dict(one=1, two=2))
result_b = big_foo(5, dict(three=3, four=4))
assert big_foo(1, dict(one=1, two=2)) == result_a
assert big_foo(5, dict(three=3, four=4)) == result_b
self.memoizer.delete_memoized(big_foo, 1, dict(one=1, two=2))
assert big_foo(1, dict(one=1, two=2)) != result_a
assert big_foo(5, dict(three=3, four=4)) == result_b
def test_10a_kwargonly_memoize(self):
@self.memoizer.memoize()
def big_foo(a=None):
if a is None:
a = 0
return a + random.random()
result_a = big_foo()
result_b = big_foo(5)
assert big_foo() == result_a
assert big_foo() < 1
assert big_foo(5) == result_b
assert big_foo(5) >= 5 and big_foo(5) < 6
def test_10a_arg_kwarg_memoize(self):
@self.memoizer.memoize()
def f(a, b, c=1):
return a + b + c + random.randrange(0, 100000)
assert f(1, 2) == f(1, 2, c=1)
assert f(1, 2) == f(1, 2, 1)
assert f(1, 2) == f(1, 2)
assert f(1, 2) != f(2, 1)
assert f(1, 2, 3) != f(1, 2)
with self.assertRaises(TypeError):
f(1)
def test_10ab_arg_kwarg_memoize(self):
@self.memoizer.memoize()
def f(a, **kwargs):
return (a, kwargs, random.randrange(0, 100000000))
f1 = f(1, b=2)
assert f1 == f(1, b=2)
assert f1 != f(1, b=4)
f2 = f(5, c=6)
assert f2 == f(5, c=6)
assert f2 != f(7, c=6)
self.memoizer.delete_memoized(f, 1, b=2)
assert f1 != f(1, b=2)
assert f2 == f(5, c=6)
assert f2 != f(7, c=6)
def test_10ac_arg_kwarg_memoize(self):
@self.memoizer.memoize()
def f(a, *args, **kwargs):
return (a, args, kwargs, random.randrange(0, 100000000))
f1 = f(1, 2, b=3)
assert f1 == f(1, 2, b=3)
assert f1 != f(1, 3, b=3)
f2 = f(5, 6, c=7)
assert f2 == f(5, 6, c=7)
assert f2 != f(7, 6, c=7)
self.memoizer.delete_memoized(f, 1, 2, b=3)
assert f1 != f(1, 2, b=3)
assert f2 == f(5, 6, c=7)
assert f2 != f(7, 6, c=7)
@self.memoizer.memoize()
def f(a, b=1, *args, **kwargs):
return (a, b, args, kwargs, random.randrange(0, 100000000))
f3 = f(1, 3, 4)
assert f3 == f(1, 3, 4)
assert f3 == f(*(1, 3, 4))
assert f3 != f(1, 3)
self.memoizer.delete_memoized(f, 1, 3, 4)
assert f3 != f(1, 3, 4)
def test_10b_classarg_memoize(self):
@self.memoizer.memoize()
def bar(a, *args):
return a.value + random.random() + sum(args)
class Adder(object):
def __init__(self, value, *args):
self.value = (value + sum(args))
adder = Adder(15)
adder2 = Adder(20)
adder3 = Adder(16, 5)
adder4 = Adder(21, 6)
w = bar(adder)
x = bar(adder2)
y = bar(adder3)
z = bar(adder4)
assert w != x
assert y != z
assert bar(adder) == w
assert bar(adder) != x
assert bar(adder3) == y
assert bar(adder3) != z
adder.value = 14
adder3.value = 15
assert bar(adder) == w
assert bar(adder) != x
assert bar(adder3) == y
assert bar(adder3) != z
assert bar(adder) != bar(adder2)
assert bar(adder3) != bar(adder4)
assert bar(adder2) == x
assert bar(adder4) == z
def test_10c_classfunc_memoize(self):
class Adder(object):
def __init__(self, initial):
self.initial = initial
@self.memoizer.memoize()
def add(self, b, *args):
return self.initial + b + sum(args)
adder1 = Adder(1)
adder2 = Adder(2)
x = adder1.add(3)
y = adder1.add(3, 4)
assert adder1.add(3) == x
assert adder1.add(4) != x
assert adder1.add(3, 4) == y
assert adder1.add(4, 4) != y
assert adder1.add(3) != adder2.add(3)
def test_10d_classfunc_memoize_delete(self):
class Adder(object):
def __init__(self, initial):
self.initial = initial
@self.memoizer.memoize()
def add(self, b, *args):
return self.initial + b + sum(args) + random.random()
adder1 = Adder(1)
adder2 = Adder(2)
a1 = adder1.add(3)
a2 = adder2.add(3)
b1 = adder1.add(3, 1)
b2 = adder2.add(3, 1)
assert a1 != a2
assert a1 != b1
assert a2 != b2
assert adder1.add(3) == a1
assert adder2.add(3) == a2
assert adder1.add(3, 1) == b1
assert adder2.add(3, 1) == b2
self.memoizer.delete_memoized(adder1.add)
a3 = adder1.add(3)
a4 = adder2.add(3)
b3 = adder1.add(3, 1)
b4 = adder2.add(3, 1)
self.assertNotEqual(a1, a3)
self.assertNotEqual(b1, b3)
assert a1 != a3
assert b1 != b3
self.assertEqual(a2, a4)
self.assertEqual(b2, b4)
self.memoizer.delete_memoized(Adder.add)
a5 = adder1.add(3)
a6 = adder2.add(3)
b5 = adder1.add(3, 1)
b6 = adder2.add(3, 1)
self.assertNotEqual(a5, a6)
self.assertNotEqual(b5, b6)
self.assertNotEqual(a3, a5)
self.assertNotEqual(b3, b5)
self.assertNotEqual(a4, a6)
self.assertNotEqual(b4, b6)
def test_10e_delete_memoize_classmethod(self):
class Mock(object):
@classmethod
@self.memoizer.memoize(5)
def big_foo(cls, a, b, *args):
return a + b + sum(args) + random.randrange(0, 100000)
result = Mock.big_foo(5, 2)
result2 = Mock.big_foo(5, 3)
result3 = Mock.big_foo(5, 2, 1)
result4 = Mock.big_foo(5, 3, 1)
assert Mock.big_foo(5, 2) == result
assert Mock.big_foo(5, 2) == result
assert Mock.big_foo(5, 3) != result
assert Mock.big_foo(5, 3) == result2
assert Mock.big_foo(5, 2, 1) == result3
assert Mock.big_foo(5, 2, 1) == result3
assert Mock.big_foo(5, 3, 1) != result3
assert Mock.big_foo(5, 3, 1) == result4
self.memoizer.delete_memoized(Mock.big_foo)
assert Mock.big_foo(5, 2) != result
assert Mock.big_foo(5, 3) != result2
assert Mock.big_foo(5, 2, 1) != result3
assert Mock.big_foo(5, 3, 1) != result4
def test_14_memoized_multiple_arg_kwarg_calls(self):
@self.memoizer.memoize()
def big_foo(a, b, c=[1, 1], d=[1, 1]):
return (sum(a) + sum(b) + sum(c) + sum(d) +
random.randrange(0, 100000))
result_a = big_foo([5, 3, 2], [1], c=[3, 3], d=[3, 3])
assert big_foo([5, 3, 2], [1], d=[3, 3], c=[3, 3]) == result_a
assert big_foo(b=[1], a=[5, 3, 2], c=[3, 3], d=[3, 3]) == result_a
assert big_foo([5, 3, 2], [1], [3, 3], [3, 3]) == result_a
def test_15_memoize_multiple_arg_kwarg_delete(self):
@self.memoizer.memoize()
def big_foo(a, b, c=[1, 1], d=[1, 1]):
return (sum(a) + sum(b) + sum(c) + sum(d) +
random.randrange(0, 100000))
result_a = big_foo([5, 3, 2], [1], c=[3, 3], d=[3, 3])
self.memoizer.delete_memoized(big_foo, [5, 3, 2], [1], [3, 3], [3, 3])
result_b = big_foo([5, 3, 2], [1], c=[3, 3], d=[3, 3])
assert result_a != result_b
self.memoizer.delete_memoized(big_foo, [5, 3, 2],
b=[1],
c=[3, 3],
d=[3, 3])
result_b = big_foo([5, 3, 2], [1], c=[3, 3], d=[3, 3])
assert result_a != result_b
self.memoizer.delete_memoized(big_foo, [5, 3, 2], [1],
c=[3, 3],
d=[3, 3])
result_a = big_foo([5, 3, 2], [1], c=[3, 3], d=[3, 3])
assert result_a != result_b
self.memoizer.delete_memoized(big_foo, [5, 3, 2],
b=[1],
c=[3, 3],
d=[3, 3])
result_a = big_foo([5, 3, 2], [1], c=[3, 3], d=[3, 3])
assert result_a != result_b
self.memoizer.delete_memoized(big_foo, [5, 3, 2], [1],
c=[3, 3],
d=[3, 3])
result_b = big_foo([5, 3, 2], [1], c=[3, 3], d=[3, 3])
assert result_a != result_b
self.memoizer.delete_memoized(big_foo, [5, 3, 2], [1], [3, 3], [3, 3])
result_a = big_foo([5, 3, 2], [1], c=[3, 3], d=[3, 3])
assert result_a != result_b
def test_16_memoize_kwargs_to_args(self):
def big_foo(a, b, c=None, d=None):
return sum(a) + sum(b) + random.randrange(0, 100000)
expected = (1, 2, 'foo', 'bar')
args, kwargs = self.memoizer._memoize_kwargs_to_args(
big_foo, 1, 2, 'foo', 'bar')
assert (args == expected)
args, kwargs = self.memoizer._memoize_kwargs_to_args(big_foo,
2,
'foo',
'bar',
a=1)
assert (args == expected)
args, kwargs = self.memoizer._memoize_kwargs_to_args(big_foo,
a=1,
b=2,
c='foo',
d='bar')
assert (args == expected)
args, kwargs = self.memoizer._memoize_kwargs_to_args(big_foo,
d='bar',
b=2,
a=1,
c='foo')
assert (args == expected)
args, kwargs = self.memoizer._memoize_kwargs_to_args(big_foo,
1,
2,
d='bar',
c='foo')
assert (args == expected)
def test_17_memoize_none_value(self):
self.memoizer = Memoizer()
@self.memoizer.memoize()
def foo():
return None
cache_key = foo.make_cache_key(foo.uncached)
assert (self.memoizer.get(cache_key) is
self.memoizer.default_cache_value)
result = foo()
assert result is None
assert self.memoizer.get(cache_key) is None
self.memoizer.delete_memoized(foo)
cache_key = foo.make_cache_key(foo.uncached)
assert (self.memoizer.get(cache_key) is
self.memoizer.default_cache_value)
def test_17_delete_memoized_instancemethod_with_mutable_param(self):
class Foo(object):
def __init__(self, id):
self.id = id
@self.memoizer.memoize(5)
def foo(self, bar_obj):
return random.randrange(0, 100000) + bar_obj.id
def __repr__(self):
return ('{}({})'.format(self.__class__.__name__, self.id))
class Bar(object):
def __init__(self, id):
self.id = id
def __repr__(self):
return ('{}({})'.format(self.__class__.__name__, self.id))
a = Foo(1)
b = Bar(1)
c = Bar(2)
result1 = a.foo(b)
result2 = a.foo(c)
assert (a.foo(b) == result1)
assert (a.foo(c) == result2)
self.memoizer.delete_memoized(a.foo, a, b)
assert (a.foo(b) != result1)
assert (a.foo(c) == result2)
def test_18_python3_uses_getfullargspec(self):
def foo():
return None
major_version = sys.version_info.major
argspec = _get_argspec(foo)
if major_version < 3:
assert argspec.__class__.__name__ is 'ArgSpec'
else:
assert argspec.__class__.__name__ is 'FullArgSpec'
def test_19_memoize_make_name(self):
@self.memoizer.memoize()
def f():
return None
without_make_name = f.make_cache_key(f.uncached)
@self.memoizer.memoize(make_name=lambda fname: 'make_name' + fname)
def f():
return None
with_make_name = f.make_cache_key(f.uncached)
assert without_make_name != with_make_name
def test_20_memoize_unless(self):
@self.memoizer.memoize(unless=lambda: True)
def f():
return random.randrange(0, 100000)
old_value = f()
new_value = f()
assert new_value != old_value
@patch('memoize.Memoizer.get', side_effect=Exception)
def test_21_memoize_get_backend_error(self, memoizer_get):
logging.basicConfig()
# disable logger (to avoid cluttering test output)
logging.disable(logging.ERROR)
memoizer = Memoizer()
@memoizer.memoize()
def f():
return random.randrange(0, 100000)
cache_key = f.make_cache_key(f.uncached)
old_value = f()
memoizer_get.assert_called_with(cache_key)
memoizer_get.reset_mock()
new_value = f()
memoizer_get.assert_called_with(cache_key)
assert new_value != old_value
# re-enable logger
logging.disable(logging.NOTSET)
@patch('memoize.Memoizer.set', side_effect=Exception)
def test_22_memoize_set_backend_error(self, memoizer_set):
logging.basicConfig()
# disable logger (to avoid cluttering test output)
logging.disable(logging.ERROR)
memoizer = Memoizer()
@memoizer.memoize()
def f():
return random.randrange(0, 100000)
cache_key = f.make_cache_key(f.uncached)
old_value = f()
memoizer_set.assert_called_with(cache_key,
old_value,
timeout=f.cache_timeout)
memoizer_set.reset_mock()
new_value = f()
memoizer_set.assert_called_with(cache_key,
new_value,
timeout=f.cache_timeout)
assert new_value != old_value
# re-enable logger
logging.disable(logging.NOTSET)
def test_23_delete_memoized_not_callable(self):
warning_raised = False
try:
self.memoizer.delete_memoized('function_name')
except DeprecationWarning:
warning_raised = True
assert warning_raised
@patch('memoize.Memoizer.delete', side_effect=Exception)
def test_24_delete_memoized_backend_error(self, memoizer_delete):
logging.basicConfig()
# disable logger (to avoid cluttering test output)
logging.disable(logging.ERROR)
memoizer = Memoizer()
@memoizer.memoize()
def f(*args, **kwargs):
return random.randrange(0, 100000)
cache_key = f.make_cache_key(f.uncached, 'a', 'b', c='c', d='d')
old_value = f('a', 'b', c='c', d='d')
memoizer.delete_memoized(f, 'a', 'b', c='c', d='d')
memoizer_delete.assert_called_with(cache_key)
new_value = f('a', 'b', c='c', d='d')
assert new_value == old_value
# re-enable logger
logging.disable(logging.NOTSET)
def test_25_delete_memoized_verhash_not_callable(self):
warning_raised = False
try:
self.memoizer.delete_memoized_verhash('function_name')
except DeprecationWarning:
warning_raised = True
assert warning_raised
@patch('memoize.Memoizer.delete', side_effect=Exception)
def test_26_delete_memoized_verhash_backend_error(self, memoizer_delete):
logging.basicConfig()
# disable logger (to avoid cluttering test output)
logging.disable(logging.ERROR)
memoizer = Memoizer()
@memoizer.memoize()
def f():
return random.randrange(0, 100000)
_fname, _ = function_namespace(f)
version_key = self.memoizer._memvname(_fname)
result = f()
assert f() == result
assert memoizer.get(version_key) is not memoizer.default_cache_value
memoizer.delete_memoized_verhash(f)
memoizer_delete.assert_called_with(version_key)
assert f() == result
assert memoizer.get(version_key) is not memoizer.default_cache_value
# re-enable logger
logging.disable(logging.NOTSET)
class MemoizeTestCase(SimpleTestCase):
def setUp(self):
self.memoizer = Memoizer()
def test_00_set(self):
self.memoizer.set('hi', 'hello')
assert self.memoizer.get('hi') == 'hello'
def test_01_add(self):
self.memoizer.add('hi', 'hello')
assert self.memoizer.get('hi') == 'hello'
self.memoizer.add('hi', 'foobar')
assert self.memoizer.get('hi') == 'hello'
def test_02_delete(self):
self.memoizer.set('hi', 'hello')
self.memoizer.delete('hi')
assert self.memoizer.get('hi') is None
def test_06_memoize(self):
@self.memoizer.memoize(5)
def big_foo(a, b):
return a+b+random.randrange(0, 100000)
result = big_foo(5, 2)
time.sleep(1)
assert big_foo(5, 2) == result
result2 = big_foo(5, 3)
assert result2 != result
time.sleep(6)
assert big_foo(5, 2) != result
time.sleep(1)
assert big_foo(5, 3) != result2
def test_06a_memoize(self):
@self.memoizer.memoize(50)
def big_foo(a, b):
return a+b+random.randrange(0, 100000)
result = big_foo(5, 2)
time.sleep(2)
assert big_foo(5, 2) == result
def test_07_delete_memoize(self):
@self.memoizer.memoize(5)
def big_foo(a, b):
return a+b+random.randrange(0, 100000)
result = big_foo(5, 2)
result2 = big_foo(5, 3)
time.sleep(1)
assert big_foo(5, 2) == result
assert big_foo(5, 2) == result
assert big_foo(5, 3) != result
assert big_foo(5, 3) == result2
self.memoizer.delete_memoized(big_foo)
assert big_foo(5, 2) != result
assert big_foo(5, 3) != result2
def test_07b_delete_memoized_verhash(self):
@self.memoizer.memoize(5)
def big_foo(a, b):
return a+b+random.randrange(0, 100000)
result = big_foo(5, 2)
result2 = big_foo(5, 3)
time.sleep(1)
assert big_foo(5, 2) == result
assert big_foo(5, 2) == result
assert big_foo(5, 3) != result
assert big_foo(5, 3) == result2
self.memoizer.delete_memoized_verhash(big_foo)
_fname, _fname_instance = function_namespace(big_foo)
version_key = self.memoizer._memvname(_fname)
assert self.memoizer.get(version_key) is None
assert big_foo(5, 2) != result
assert big_foo(5, 3) != result2
assert self.memoizer.get(version_key) is not None
def test_08_delete_memoize(self):
@self.memoizer.memoize()
def big_foo(a, b):
return a+b+random.randrange(0, 100000)
result_a = big_foo(5, 1)
result_b = big_foo(5, 2)
assert big_foo(5, 1) == result_a
assert big_foo(5, 2) == result_b
self.memoizer.delete_memoized(big_foo, 5, 2)
assert big_foo(5, 1) == result_a
assert big_foo(5, 2) != result_b
# Cleanup bigfoo 5,1 5,2 or it might conflict with
# following run if it also uses memecache
self.memoizer.delete_memoized(big_foo, 5, 2)
self.memoizer.delete_memoized(big_foo, 5, 1)
def test_09_args_memoize(self):
@self.memoizer.memoize()
def big_foo(a, b):
return sum(a)+sum(b)+random.randrange(0, 100000)
result_a = big_foo([5, 3, 2], [1])
result_b = big_foo([3, 3], [3, 1])
assert big_foo([5, 3, 2], [1]) == result_a
assert big_foo([3, 3], [3, 1]) == result_b
self.memoizer.delete_memoized(big_foo, [5, 3, 2], [1])
assert big_foo([5, 3, 2], [1]) != result_a
assert big_foo([3, 3], [3, 1]) == result_b
# Cleanup bigfoo 5,1 5,2 or it might conflict with
# following run if it also uses memecache
self.memoizer.delete_memoized(big_foo, [5, 3, 2], [1])
self.memoizer.delete_memoized(big_foo, [3, 3], [1])
def test_10_kwargs_memoize(self):
@self.memoizer.memoize()
def big_foo(a, b=None):
return a+sum(b.values())+random.randrange(0, 100000)
result_a = big_foo(1, dict(one=1, two=2))
result_b = big_foo(5, dict(three=3, four=4))
assert big_foo(1, dict(one=1, two=2)) == result_a
assert big_foo(5, dict(three=3, four=4)) == result_b
self.memoizer.delete_memoized(big_foo, 1, dict(one=1, two=2))
assert big_foo(1, dict(one=1, two=2)) != result_a
assert big_foo(5, dict(three=3, four=4)) == result_b
def test_10a_kwargonly_memoize(self):
@self.memoizer.memoize()
def big_foo(a=None):
if a is None:
a = 0
return a+random.random()
result_a = big_foo()
result_b = big_foo(5)
assert big_foo() == result_a
assert big_foo() < 1
assert big_foo(5) == result_b
assert big_foo(5) >= 5 and big_foo(5) < 6
def test_10a_arg_kwarg_memoize(self):
@self.memoizer.memoize()
def f(a, b, c=1):
return a+b+c+random.randrange(0, 100000)
assert f(1, 2) == f(1, 2, c=1)
assert f(1, 2) == f(1, 2, 1)
assert f(1, 2) == f(1, 2)
assert f(1, 2) != f(2, 1)
assert f(1, 2, 3) != f(1, 2)
with self.assertRaises(TypeError):
f(1)
def test_10ab_arg_kwarg_memoize(self):
@self.memoizer.memoize()
def f(a, **kwargs):
return (a, kwargs, random.randrange(0, 100000000))
f1 = f(1, b=2)
assert f1 == f(1, b=2)
assert f1 != f(1, b=4)
f2 = f(5, c=6)
assert f2 == f(5, c=6)
assert f2 != f(7, c=6)
self.memoizer.delete_memoized(f, 1, b=2)
assert f1 != f(1, b=2)
assert f2 == f(5, c=6)
assert f2 != f(7, c=6)
def test_10ac_arg_kwarg_memoize(self):
@self.memoizer.memoize()
def f(a, *args, **kwargs):
return (a, args, kwargs, random.randrange(0, 100000000))
f1 = f(1, 2, b=3)
assert f1 == f(1, 2, b=3)
assert f1 != f(1, 3, b=3)
f2 = f(5, 6, c=7)
assert f2 == f(5, 6, c=7)
assert f2 != f(7, 6, c=7)
self.memoizer.delete_memoized(f, 1, 2, b=3)
assert f1 != f(1, 2, b=3)
assert f2 == f(5, 6, c=7)
assert f2 != f(7, 6, c=7)
@self.memoizer.memoize()
def f(a, b=1, *args, **kwargs):
return (a, b, args, kwargs, random.randrange(0, 100000000))
f3 = f(1, 3, 4)
assert f3 == f(1, 3, 4)
assert f3 == f(*(1, 3, 4))
assert f3 != f(1, 3)
self.memoizer.delete_memoized(f, 1, 3, 4)
assert f3 != f(1, 3, 4)
def test_10b_classarg_memoize(self):
@self.memoizer.memoize()
def bar(a, *args):
return a.value + random.random() + sum(args)
class Adder(object):
def __init__(self, value, *args):
self.value = (value + sum(args))
adder = Adder(15)
adder2 = Adder(20)
adder3 = Adder(16, 5)
adder4 = Adder(21, 6)
w = bar(adder)
x = bar(adder2)
y = bar(adder3)
z = bar(adder4)
assert w != x
assert y != z
assert bar(adder) == w
assert bar(adder) != x
assert bar(adder3) == y
assert bar(adder3) != z
adder.value = 14
adder3.value = 15
assert bar(adder) == w
assert bar(adder) != x
assert bar(adder3) == y
assert bar(adder3) != z
assert bar(adder) != bar(adder2)
assert bar(adder3) != bar(adder4)
assert bar(adder2) == x
assert bar(adder4) == z
def test_10c_classfunc_memoize(self):
class Adder(object):
def __init__(self, initial):
self.initial = initial
@self.memoizer.memoize()
def add(self, b, *args):
return self.initial + b + sum(args)
adder1 = Adder(1)
adder2 = Adder(2)
x = adder1.add(3)
y = adder1.add(3, 4)
assert adder1.add(3) == x
assert adder1.add(4) != x
assert adder1.add(3, 4) == y
assert adder1.add(4, 4) != y
assert adder1.add(3) != adder2.add(3)
def test_10d_classfunc_memoize_delete(self):
class Adder(object):
def __init__(self, initial):
self.initial = initial
@self.memoizer.memoize()
def add(self, b, *args):
return self.initial + b + sum(args) + random.random()
adder1 = Adder(1)
adder2 = Adder(2)
a1 = adder1.add(3)
a2 = adder2.add(3)
b1 = adder1.add(3, 1)
b2 = adder2.add(3, 1)
assert a1 != a2
assert a1 != b1
assert a2 != b2
assert adder1.add(3) == a1
assert adder2.add(3) == a2
assert adder1.add(3, 1) == b1
assert adder2.add(3, 1) == b2
self.memoizer.delete_memoized(adder1.add)
a3 = adder1.add(3)
a4 = adder2.add(3)
b3 = adder1.add(3, 1)
b4 = adder2.add(3, 1)
self.assertNotEqual(a1, a3)
self.assertNotEqual(b1, b3)
assert a1 != a3
assert b1 != b3
self.assertEqual(a2, a4)
self.assertEqual(b2, b4)
self.memoizer.delete_memoized(Adder.add)
a5 = adder1.add(3)
a6 = adder2.add(3)
b5 = adder1.add(3, 1)
b6 = adder2.add(3, 1)
self.assertNotEqual(a5, a6)
self.assertNotEqual(b5, b6)
self.assertNotEqual(a3, a5)
self.assertNotEqual(b3, b5)
self.assertNotEqual(a4, a6)
self.assertNotEqual(b4, b6)
def test_10e_delete_memoize_classmethod(self):
class Mock(object):
@classmethod
@self.memoizer.memoize(5)
def big_foo(cls, a, b, *args):
return a+b+sum(args)+random.randrange(0, 100000)
result = Mock.big_foo(5, 2)
result2 = Mock.big_foo(5, 3)
result3 = Mock.big_foo(5, 2, 1)
result4 = Mock.big_foo(5, 3, 1)
time.sleep(1)
assert Mock.big_foo(5, 2) == result
assert Mock.big_foo(5, 2) == result
assert Mock.big_foo(5, 3) != result
assert Mock.big_foo(5, 3) == result2
assert Mock.big_foo(5, 2, 1) == result3
assert Mock.big_foo(5, 2, 1) == result3
assert Mock.big_foo(5, 3, 1) != result3
assert Mock.big_foo(5, 3, 1) == result4
self.memoizer.delete_memoized(Mock.big_foo)
assert Mock.big_foo(5, 2) != result
assert Mock.big_foo(5, 3) != result2
assert Mock.big_foo(5, 2, 1) != result3
assert Mock.big_foo(5, 3, 1) != result4
def test_14_memoized_multiple_arg_kwarg_calls(self):
@self.memoizer.memoize()
def big_foo(a, b, c=[1, 1], d=[1, 1]):
return sum(a)+sum(b)+sum(c)+sum(d)+random.randrange(0, 100000)
result_a = big_foo([5, 3, 2], [1], c=[3, 3], d=[3, 3])
assert big_foo([5, 3, 2], [1], d=[3, 3], c=[3, 3]) == result_a
assert big_foo(b=[1], a=[5, 3, 2], c=[3, 3], d=[3, 3]) == result_a
assert big_foo([5, 3, 2], [1], [3, 3], [3, 3]) == result_a
def test_15_memoize_multiple_arg_kwarg_delete(self):
@self.memoizer.memoize()
def big_foo(a, b, c=[1, 1], d=[1, 1]):
return sum(a)+sum(b)+sum(c)+sum(d)+random.randrange(0, 100000)
result_a = big_foo([5, 3, 2], [1], c=[3, 3], d=[3, 3])
self.memoizer.delete_memoized(big_foo, [5, 3, 2], [1], [3, 3], [3, 3])
result_b = big_foo([5, 3, 2], [1], c=[3, 3], d=[3, 3])
assert result_a != result_b
self.memoizer.delete_memoized(
big_foo, [5, 3, 2], b=[1], c=[3, 3], d=[3, 3]
)
result_b = big_foo([5, 3, 2], [1], c=[3, 3], d=[3, 3])
assert result_a != result_b
self.memoizer.delete_memoized(
big_foo, [5, 3, 2], [1], c=[3, 3], d=[3, 3]
)
result_a = big_foo([5, 3, 2], [1], c=[3, 3], d=[3, 3])
assert result_a != result_b
self.memoizer.delete_memoized(
big_foo, [5, 3, 2], b=[1], c=[3, 3], d=[3, 3]
)
result_a = big_foo([5, 3, 2], [1], c=[3, 3], d=[3, 3])
assert result_a != result_b
self.memoizer.delete_memoized(
big_foo, [5, 3, 2], [1], c=[3, 3], d=[3, 3]
)
result_b = big_foo([5, 3, 2], [1], c=[3, 3], d=[3, 3])
assert result_a != result_b
self.memoizer.delete_memoized(big_foo, [5, 3, 2], [1], [3, 3], [3, 3])
result_a = big_foo([5, 3, 2], [1], c=[3, 3], d=[3, 3])
assert result_a != result_b
def test_16_memoize_kwargs_to_args(self):
def big_foo(a, b, c=None, d=None):
return sum(a)+sum(b)+random.randrange(0, 100000)
expected = (1, 2, 'foo', 'bar')
args, kwargs = self.memoizer._memoize_kwargs_to_args(
big_foo, 1, 2, 'foo', 'bar')
assert (args == expected)
args, kwargs = self.memoizer._memoize_kwargs_to_args(
big_foo, 2, 'foo', 'bar', a=1)
assert (args == expected)
args, kwargs = self.memoizer._memoize_kwargs_to_args(
big_foo, a=1, b=2, c='foo', d='bar')
assert (args == expected)
args, kwargs = self.memoizer._memoize_kwargs_to_args(
big_foo, d='bar', b=2, a=1, c='foo')
assert (args == expected)
args, kwargs = self.memoizer._memoize_kwargs_to_args(
big_foo, 1, 2, d='bar', c='foo')
assert (args == expected)
class MemoizeTestCase(SimpleTestCase):
def setUp(self):
self.memoizer = Memoizer()
def test_00_set(self):
self.memoizer.set('hi', 'hello')
assert self.memoizer.get('hi') == 'hello'
def test_01_add(self):
self.memoizer.add('hi', 'hello')
assert self.memoizer.get('hi') == 'hello'
self.memoizer.add('hi', 'foobar')
assert self.memoizer.get('hi') == 'hello'
def test_02_delete(self):
self.memoizer.set('hi', 'hello')
self.memoizer.delete('hi')
assert self.memoizer.get('hi') is self.memoizer.default_cache_value
def test_06_memoize(self):
@self.memoizer.memoize(5)
def big_foo(a, b, func=lambda x: x):
func(a)
return a + b
test_func = MagicMock()
big_foo(5, 2, func=test_func)
# Cached call
big_foo(5, 2, func=test_func)
# We are expecting only 1 call here as second one should be cached
assert test_func.call_count == 1, "Count: {}".format(
test_func.call_count)
# different arguments so we should have additional call
big_foo(5, 3, func=test_func)
assert test_func.call_count == 2, "Count: {}".format(
test_func.call_count)
# Expired
now = datetime.datetime.utcfromtimestamp(time.time())
with freeze_time(now) as frozen_datetime:
frozen_datetime.tick(delta=datetime.timedelta(seconds=6))
# Cache should be expired so we have increasing amount of calls
big_foo(5, 2, func=test_func)
assert test_func.call_count == 3, "Count: {}".format(
test_func.call_count)
big_foo(5, 3, func=test_func)
assert test_func.call_count == 4, "Count: {}".format(
test_func.call_count)
def test_06a_memoize(self):
@self.memoizer.memoize(50)
def big_foo(a, b):
return a + b + random.randrange(0, 100000)
result = big_foo(5, 2)
assert big_foo(5, 2) == result
def test_07_delete_memoize(self):
@self.memoizer.memoize(5)
def big_foo(a, b):
return a + b + random.randrange(0, 100000)
result = big_foo(5, 2)
result2 = big_foo(5, 3)
assert big_foo(5, 2) == result
assert big_foo(5, 2) == result
assert big_foo(5, 3) != result
assert big_foo(5, 3) == result2
self.memoizer.delete_memoized(big_foo)
assert big_foo(5, 2) != result
assert big_foo(5, 3) != result2
def test_07b_delete_memoized_verhash(self):
@self.memoizer.memoize(5)
def big_foo(a, b):
return a + b + random.randrange(0, 100000)
result = big_foo(5, 2)
result2 = big_foo(5, 3)
assert big_foo(5, 2) == result
assert big_foo(5, 2) == result
assert big_foo(5, 3) != result
assert big_foo(5, 3) == result2
self.memoizer.delete_memoized_verhash(big_foo)
_fname, _fname_instance = function_namespace(big_foo)
version_key = self.memoizer._memvname(_fname)
assert (self.memoizer.get(version_key) is
self.memoizer.default_cache_value)
assert big_foo(5, 2) != result
assert big_foo(5, 3) != result2
assert (self.memoizer.get(version_key)
is not self.memoizer.default_cache_value)
def test_08_delete_memoize(self):
@self.memoizer.memoize()
def big_foo(a, b):
return a + b + random.randrange(0, 100000)
result_a = big_foo(5, 1)
result_b = big_foo(5, 2)
assert big_foo(5, 1) == result_a
assert big_foo(5, 2) == result_b
self.memoizer.delete_memoized(big_foo, 5, 2)
assert big_foo(5, 1) == result_a
assert big_foo(5, 2) != result_b
# Cleanup bigfoo 5,1 5,2 or it might conflict with
# following run if it also uses memecache
self.memoizer.delete_memoized(big_foo, 5, 2)
self.memoizer.delete_memoized(big_foo, 5, 1)
def test_09_args_memoize(self):
@self.memoizer.memoize()
def big_foo(a, b):
return sum(a) + sum(b) + random.randrange(0, 100000)
result_a = big_foo([5, 3, 2], [1])
result_b = big_foo([3, 3], [3, 1])
assert big_foo([5, 3, 2], [1]) == result_a
assert big_foo([3, 3], [3, 1]) == result_b
self.memoizer.delete_memoized(big_foo, [5, 3, 2], [1])
assert big_foo([5, 3, 2], [1]) != result_a
assert big_foo([3, 3], [3, 1]) == result_b
# Cleanup bigfoo 5,1 5,2 or it might conflict with
# following run if it also uses memecache
self.memoizer.delete_memoized(big_foo, [5, 3, 2], [1])
self.memoizer.delete_memoized(big_foo, [3, 3], [1])
def test_10_kwargs_memoize(self):
@self.memoizer.memoize()
def big_foo(a, b=None):
return a + sum(b.values()) + random.randrange(0, 100000)
result_a = big_foo(1, dict(one=1, two=2))
result_b = big_foo(5, dict(three=3, four=4))
assert big_foo(1, dict(one=1, two=2)) == result_a
assert big_foo(5, dict(three=3, four=4)) == result_b
self.memoizer.delete_memoized(big_foo, 1, dict(one=1, two=2))
assert big_foo(1, dict(one=1, two=2)) != result_a
assert big_foo(5, dict(three=3, four=4)) == result_b
def test_10a_kwargonly_memoize(self):
@self.memoizer.memoize()
def big_foo(a=None):
if a is None:
a = 0
return a + random.random()
result_a = big_foo()
result_b = big_foo(5)
assert big_foo() == result_a
assert big_foo() < 1
assert big_foo(5) == result_b
assert big_foo(5) >= 5 and big_foo(5) < 6
def test_10a_arg_kwarg_memoize(self):
@self.memoizer.memoize()
def f(a, b, c=1):
return a + b + c + random.randrange(0, 100000)
assert f(1, 2) == f(1, 2, c=1)
assert f(1, 2) == f(1, 2, 1)
assert f(1, 2) == f(1, 2)
assert f(1, 2) != f(2, 1)
assert f(1, 2, 3) != f(1, 2)
with self.assertRaises(TypeError):
f(1)
def test_10ab_arg_kwarg_memoize(self):
@self.memoizer.memoize()
def f(a, **kwargs):
return (a, kwargs, random.randrange(0, 100000000))
f1 = f(1, b=2)
assert f1 == f(1, b=2)
assert f1 != f(1, b=4)
f2 = f(5, c=6)
assert f2 == f(5, c=6)
assert f2 != f(7, c=6)
self.memoizer.delete_memoized(f, 1, b=2)
assert f1 != f(1, b=2)
assert f2 == f(5, c=6)
assert f2 != f(7, c=6)
def test_10ac_arg_kwarg_memoize(self):
@self.memoizer.memoize()
def f(a, *args, **kwargs):
return (a, args, kwargs, random.randrange(0, 100000000))
f1 = f(1, 2, b=3)
assert f1 == f(1, 2, b=3)
assert f1 != f(1, 3, b=3)
f2 = f(5, 6, c=7)
assert f2 == f(5, 6, c=7)
assert f2 != f(7, 6, c=7)
self.memoizer.delete_memoized(f, 1, 2, b=3)
assert f1 != f(1, 2, b=3)
assert f2 == f(5, 6, c=7)
assert f2 != f(7, 6, c=7)
@self.memoizer.memoize()
def f(a, b=1, *args, **kwargs):
return (a, b, args, kwargs, random.randrange(0, 100000000))
f3 = f(1, 3, 4)
assert f3 == f(1, 3, 4)
assert f3 == f(*(1, 3, 4))
assert f3 != f(1, 3)
self.memoizer.delete_memoized(f, 1, 3, 4)
assert f3 != f(1, 3, 4)
def test_10b_classarg_memoize(self):
@self.memoizer.memoize()
def bar(a, *args):
return a.value + random.random() + sum(args)
class Adder(object):
def __init__(self, value, *args):
self.value = (value + sum(args))
adder = Adder(15)
adder2 = Adder(20)
adder3 = Adder(16, 5)
adder4 = Adder(21, 6)
w = bar(adder)
x = bar(adder2)
y = bar(adder3)
z = bar(adder4)
assert w != x
assert y != z
assert bar(adder) == w
assert bar(adder) != x
assert bar(adder3) == y
assert bar(adder3) != z
adder.value = 14
adder3.value = 15
assert bar(adder) == w
assert bar(adder) != x
assert bar(adder3) == y
assert bar(adder3) != z
assert bar(adder) != bar(adder2)
assert bar(adder3) != bar(adder4)
assert bar(adder2) == x
assert bar(adder4) == z
def test_10c_classfunc_memoize(self):
class Adder(object):
def __init__(self, initial):
self.initial = initial
@self.memoizer.memoize()
def add(self, b, *args):
return self.initial + b + sum(args)
adder1 = Adder(1)
adder2 = Adder(2)
x = adder1.add(3)
y = adder1.add(3, 4)
assert adder1.add(3) == x
assert adder1.add(4) != x
assert adder1.add(3, 4) == y
assert adder1.add(4, 4) != y
assert adder1.add(3) != adder2.add(3)
def test_10d_classfunc_memoize_delete(self):
class Adder(object):
def __init__(self, initial):
self.initial = initial
@self.memoizer.memoize()
def add(self, b, *args):
return self.initial + b + sum(args) + random.random()
adder1 = Adder(1)
adder2 = Adder(2)
a1 = adder1.add(3)
a2 = adder2.add(3)
b1 = adder1.add(3, 1)
b2 = adder2.add(3, 1)
assert a1 != a2
assert a1 != b1
assert a2 != b2
assert adder1.add(3) == a1
assert adder2.add(3) == a2
assert adder1.add(3, 1) == b1
assert adder2.add(3, 1) == b2
self.memoizer.delete_memoized(adder1.add)
a3 = adder1.add(3)
a4 = adder2.add(3)
b3 = adder1.add(3, 1)
b4 = adder2.add(3, 1)
self.assertNotEqual(a1, a3)
self.assertNotEqual(b1, b3)
assert a1 != a3
assert b1 != b3
self.assertEqual(a2, a4)
self.assertEqual(b2, b4)
self.memoizer.delete_memoized(Adder.add)
a5 = adder1.add(3)
a6 = adder2.add(3)
b5 = adder1.add(3, 1)
b6 = adder2.add(3, 1)
self.assertNotEqual(a5, a6)
self.assertNotEqual(b5, b6)
self.assertNotEqual(a3, a5)
self.assertNotEqual(b3, b5)
self.assertNotEqual(a4, a6)
self.assertNotEqual(b4, b6)
def test_10e_delete_memoize_classmethod(self):
class Mock(object):
@classmethod
@self.memoizer.memoize(5)
def big_foo(cls, a, b, *args):
return a + b + sum(args) + random.randrange(0, 100000)
result = Mock.big_foo(5, 2)
result2 = Mock.big_foo(5, 3)
result3 = Mock.big_foo(5, 2, 1)
result4 = Mock.big_foo(5, 3, 1)
assert Mock.big_foo(5, 2) == result
assert Mock.big_foo(5, 2) == result
assert Mock.big_foo(5, 3) != result
assert Mock.big_foo(5, 3) == result2
assert Mock.big_foo(5, 2, 1) == result3
assert Mock.big_foo(5, 2, 1) == result3
assert Mock.big_foo(5, 3, 1) != result3
assert Mock.big_foo(5, 3, 1) == result4
self.memoizer.delete_memoized(Mock.big_foo)
assert Mock.big_foo(5, 2) != result
assert Mock.big_foo(5, 3) != result2
assert Mock.big_foo(5, 2, 1) != result3
assert Mock.big_foo(5, 3, 1) != result4
def test_14_memoized_multiple_arg_kwarg_calls(self):
@self.memoizer.memoize()
def big_foo(a, b, c=[1, 1], d=[1, 1]):
return (sum(a) + sum(b) + sum(c) + sum(d) +
random.randrange(0, 100000))
result_a = big_foo([5, 3, 2], [1], c=[3, 3], d=[3, 3])
assert big_foo([5, 3, 2], [1], d=[3, 3], c=[3, 3]) == result_a
assert big_foo(b=[1], a=[5, 3, 2], c=[3, 3], d=[3, 3]) == result_a
assert big_foo([5, 3, 2], [1], [3, 3], [3, 3]) == result_a
def test_15_memoize_multiple_arg_kwarg_delete(self):
@self.memoizer.memoize()
def big_foo(a, b, c=[1, 1], d=[1, 1]):
return (sum(a) + sum(b) + sum(c) + sum(d) +
random.randrange(0, 100000))
result_a = big_foo([5, 3, 2], [1], c=[3, 3], d=[3, 3])
self.memoizer.delete_memoized(big_foo, [5, 3, 2], [1], [3, 3], [3, 3])
result_b = big_foo([5, 3, 2], [1], c=[3, 3], d=[3, 3])
assert result_a != result_b
self.memoizer.delete_memoized(big_foo, [5, 3, 2],
b=[1],
c=[3, 3],
d=[3, 3])
result_b = big_foo([5, 3, 2], [1], c=[3, 3], d=[3, 3])
assert result_a != result_b
self.memoizer.delete_memoized(big_foo, [5, 3, 2], [1],
c=[3, 3],
d=[3, 3])
result_a = big_foo([5, 3, 2], [1], c=[3, 3], d=[3, 3])
assert result_a != result_b
self.memoizer.delete_memoized(big_foo, [5, 3, 2],
b=[1],
c=[3, 3],
d=[3, 3])
result_a = big_foo([5, 3, 2], [1], c=[3, 3], d=[3, 3])
assert result_a != result_b
self.memoizer.delete_memoized(big_foo, [5, 3, 2], [1],
c=[3, 3],
d=[3, 3])
result_b = big_foo([5, 3, 2], [1], c=[3, 3], d=[3, 3])
assert result_a != result_b
self.memoizer.delete_memoized(big_foo, [5, 3, 2], [1], [3, 3], [3, 3])
result_a = big_foo([5, 3, 2], [1], c=[3, 3], d=[3, 3])
assert result_a != result_b
def test_16_memoize_kwargs_to_args(self):
def big_foo(a, b, c=None, d=None):
return sum(a) + sum(b) + random.randrange(0, 100000)
expected = ('1', '2', "'foo'", "'bar'")
args, kwargs = self.memoizer._memoize_kwargs_to_args(
big_foo, 1, 2, 'foo', 'bar')
assert (args == expected)
args, kwargs = self.memoizer._memoize_kwargs_to_args(big_foo,
2,
'foo',
'bar',
a=1)
assert (args == expected)
args, kwargs = self.memoizer._memoize_kwargs_to_args(big_foo,
a=1,
b=2,
c='foo',
d='bar')
assert (args == expected)
args, kwargs = self.memoizer._memoize_kwargs_to_args(big_foo,
d='bar',
b=2,
a=1,
c='foo')
assert (args == expected)
args, kwargs = self.memoizer._memoize_kwargs_to_args(big_foo,
1,
2,
d='bar',
c='foo')
assert (args == expected)
def test_17_memoize_none_value(self):
self.memoizer = Memoizer()
@self.memoizer.memoize()
def foo():
return None
cache_key = foo.make_cache_key(foo.uncached)
assert (self.memoizer.get(cache_key) is
self.memoizer.default_cache_value)
result = foo()
assert result is None
assert self.memoizer.get(cache_key) is None
self.memoizer.delete_memoized(foo)
cache_key = foo.make_cache_key(foo.uncached)
assert (self.memoizer.get(cache_key) is
self.memoizer.default_cache_value)
def test_17_delete_memoized_instancemethod_with_mutable_param(self):
class Foo(object):
def __init__(self, id):
self.id = id
@self.memoizer.memoize(5)
def foo(self, bar_obj):
return random.randrange(0, 100000) + bar_obj.id
def __repr__(self):
return ('{}({})'.format(self.__class__.__name__, self.id))
class Bar(object):
def __init__(self, id):
self.id = id
def __repr__(self):
return ('{}({})'.format(self.__class__.__name__, self.id))
a = Foo(1)
b = Bar(1)
c = Bar(2)
result1 = a.foo(b)
result2 = a.foo(c)
assert (a.foo(b) == result1)
assert (a.foo(c) == result2)
self.memoizer.delete_memoized(a.foo, a, b)
assert (a.foo(b) != result1)
assert (a.foo(c) == result2)
def test_18_python3_uses_getfullargspec(self):
def foo():
return None
major_version = sys.version_info.major
argspec = _get_argspec(foo)
if major_version < 3:
assert argspec.__class__.__name__ is 'ArgSpec'
else:
assert argspec.__class__.__name__ is 'FullArgSpec'
def test_19_test_custom_repr_fn(self):
custom_memoizer = Memoizer(repr_fn=lambda x: 'static')
memoizer = Memoizer()
class Test(object):
def fn(self, arg=None):
return 1
cached_fn = memoizer.memoize(60)(fn)
cached_fn_custom_repr = custom_memoizer.memoize(60)(fn)
def __str__(self):
return 'str'
__unicode__ = __str__
def __repr__(self):
return 'repr'
fake_obj = Test()
fake_obj2 = Test()
def _get_keys(obj):
cache_key = obj.cached_fn.make_cache_key(obj.fn, obj, 123)
custom_cache_key = obj.cached_fn_custom_repr.make_cache_key(
obj.fn, obj, 123)
return cache_key, custom_cache_key
cache_key, custom_cache_key = _get_keys(fake_obj)
assert cache_key != custom_cache_key
cache_key2, custom_cache_key2 = _get_keys(fake_obj2)
assert cache_key == cache_key2
assert custom_cache_key == custom_cache_key2
custom_cache_key3 = fake_obj.cached_fn_custom_repr.make_cache_key(
fake_obj.fn, fake_obj, ['test'])
assert custom_cache_key == custom_cache_key3
import calendar
import datetime
import functools
from memoize import Memoizer
store = {}
memo = Memoizer(store)
def next_month(year, month):
"""
Return the year and month of the next month
NOTE: month is 1 based
"""
return add_months(year, month, 1)
def add_months(year, month, months):
"""
Return the year and month of #months after the given year and month
NOTE: month is 1 based
"""
month += months
while month > 12:
month -= 12
year += 1
return year, month
def record_runtime(func):
from urlparse import urljoin
import requests
from cloudfoundry.apps import CloudFoundryApp
from cloudfoundry.organizations import CloudFoundryOrg
from cloudfoundry.spaces import CloudFoundrySpace
from cloudfoundry.routes import CloudFoundryRoute
from cloudfoundry.domains import CloudFoundryDomain
from utils import create_bits_zip
from collections import OrderedDict
import logging
import time
from memoize import Memoizer
cache_store = {}
memo = Memoizer(cache_store)
max_cache_time = 10
class CloudFoundryException(Exception):
pass
class CloudFoundryAuthenticationException(CloudFoundryException):
pass
class CloudFoundryInterface(object):
def __init__(self, target, username=None, password=None, debug=False):
self._apps = None
