def test_24_delete_memoized_backend_error(self, memoizer_delete): logging.basicConfig() # disable logger (to avoid cluttering test output) logging.disable(logging.ERROR) memoizer = Memoizer() @memoizer.memoize() def f(*args, **kwargs): return random.randrange(0, 100000) cache_key = f.make_cache_key(f.uncached, 'a', 'b', c='c', d='d') old_value = f('a', 'b', c='c', d='d') memoizer.delete_memoized(f, 'a', 'b', c='c', d='d') memoizer_delete.assert_called_with(cache_key) new_value = f('a', 'b', c='c', d='d') assert new_value == old_value # re-enable logger logging.disable(logging.NOTSET)
def __init__( self, bridge_ip, logger, main_tbl_num, stats_table, next_table, scratch_table_num, session_rule_version_mapper, ): self._bridge_ip = bridge_ip self.logger = logger self.main_tbl_num = main_tbl_num self.stats_table = stats_table self.next_table = next_table self._scratch_tbl_num = scratch_table_num self._redirect_dict = RedirectDict() self._dns_cache = Memoizer({}) self._redirect_port = get_service_config_value( 'redirectd', 'http_port', 8080, ) self._session_rule_version_mapper = session_rule_version_mapper self._cwf_args_set = False self._mac_rewrite_scratch = None self._internal_ip_allocator = None self._arpd_controller_fut = None self._arp_contoller = None self._egress_table = None self._bridge_mac = None
def __init__(self, bridge_ip, logger, tbl_num, next_table): self._bridge_ip = bridge_ip self.logger = logger self.tbl_num = tbl_num self.next_table = next_table self._redirect_dict = RedirectDict() self._dns_cache = Memoizer({}) self._redirect_port = get_service_config_value( 'redirectd', 'http_port', 8080)
def __init__(self, bridge_ip, logger, main_tbl_num, next_table, scratch_table_num, session_rule_version_mapper): self._bridge_ip = bridge_ip self.logger = logger self.main_tbl_num = main_tbl_num self.next_table = next_table self._scratch_tbl_num = scratch_table_num self._redirect_dict = RedirectDict() self._dns_cache = Memoizer({}) self._redirect_port = get_service_config_value( 'redirectd', 'http_port', 8080) self._session_rule_version_mapper = session_rule_version_mapper
def test_32_update_memoized_backend_error(self, memoizer_set): settings.DEBUG = True logging.basicConfig() # disable logger (to avoid cluttering test output) logging.disable(logging.ERROR) memoizer = Memoizer() @memoizer.memoize() def f(): return random.randrange(0, 100000) exception_raised = False try: self.memoizer.update_memoized(f) except Exception: exception_raised = True assert exception_raised settings.DEBUG = False memoizer_set.reset_mock() # re-enable logger logging.disable(logging.NOTSET)
def test_22_memoize_set_backend_error(self, memoizer_set): logging.basicConfig() # disable logger (to avoid cluttering test output) logging.disable(logging.ERROR) memoizer = Memoizer() @memoizer.memoize() def f(): return random.randrange(0, 100000) cache_key = f.make_cache_key(f.uncached) old_value = f() memoizer_set.assert_called_with(cache_key, old_value, timeout=f.cache_timeout) memoizer_set.reset_mock() new_value = f() memoizer_set.assert_called_with(cache_key, new_value, timeout=f.cache_timeout) assert new_value != old_value # re-enable logger logging.disable(logging.NOTSET)
def main(): #%% data0 = { "x_train": pd.read_pickle(DATA_DIR + "/x_train.pkl"), "y_train": pd.read_pickle(DATA_DIR + "/y_train.pkl"), "x_test": pd.read_pickle(DATA_DIR + "/x_test.pkl"), "y_test": pd.read_pickle(DATA_DIR + "/y_test.pkl") } #%% param_grid_dic = OrderedDict([ ("learning_rate", [0.05, 0.10, 0.15, 0.20, 0.25, 0.30]), ("max_depth", [3, 4, 5, 6, 8, 10, 12, 15]), ("min_child_weight", [1, 3, 5, 7]), ("gamma", [0.0, 0.1, 0.2, 0.3, 0.4]), ("colsample_bytree", [0.3, 0.4, 0.5, 0.7]), ]) #%% train_fraction = 0.05 test_fraction = 0.16 data = subsample(data0, train_fraction, test_fraction) #%% memoization_path = DATA_DIR + "/" + "xgboost_memo%g" % train_fraction print("memoization_path= " + memoization_path) if not os.path.exists(memoization_path): os.mkdir(memoization_path) #%% fun = Memoizer(lambda param_dic: train_xgb(data, param_dic), memoization_path) #%% grid_search(param_grid_dic, fun)
def test_17_memoize_none_value(self): self.memoizer = Memoizer() @self.memoizer.memoize() def foo(): return None cache_key = foo.make_cache_key(foo.uncached) assert (self.memoizer.get(cache_key) is self.memoizer.default_cache_value) result = foo() assert result is None assert self.memoizer.get(cache_key) is None self.memoizer.delete_memoized(foo) cache_key = foo.make_cache_key(foo.uncached) assert (self.memoizer.get(cache_key) is self.memoizer.default_cache_value)
def test_26_delete_memoized_verhash_backend_error(self, memoizer_delete): logging.basicConfig() # disable logger (to avoid cluttering test output) logging.disable(logging.ERROR) memoizer = Memoizer() @memoizer.memoize() def f(): return random.randrange(0, 100000) _fname, _ = function_namespace(f) version_key = self.memoizer._memvname(_fname) result = f() assert f() == result assert memoizer.get(version_key) is not memoizer.default_cache_value memoizer.delete_memoized_verhash(f) memoizer_delete.assert_called_with(version_key) assert f() == result assert memoizer.get(version_key) is not memoizer.default_cache_value # re-enable logger logging.disable(logging.NOTSET)
def test_19_test_custom_repr_fn(self): custom_memoizer = Memoizer(repr_fn=lambda x: 'static') memoizer = Memoizer() class Test(object): def fn(self, arg=None): return 1 cached_fn = memoizer.memoize(60)(fn) cached_fn_custom_repr = custom_memoizer.memoize(60)(fn) def __str__(self): return 'str' __unicode__ = __str__ def __repr__(self): return 'repr' fake_obj = Test() fake_obj2 = Test() def _get_keys(obj): cache_key = obj.cached_fn.make_cache_key(obj.fn, obj, 123) custom_cache_key = obj.cached_fn_custom_repr.make_cache_key( obj.fn, obj, 123) return cache_key, custom_cache_key cache_key, custom_cache_key = _get_keys(fake_obj) assert cache_key != custom_cache_key cache_key2, custom_cache_key2 = _get_keys(fake_obj2) assert cache_key == cache_key2 assert custom_cache_key == custom_cache_key2 custom_cache_key3 = fake_obj.cached_fn_custom_repr.make_cache_key( fake_obj.fn, fake_obj, ['test']) assert custom_cache_key == custom_cache_key3
def test_17_memoize_none_value(self): self.memoizer = Memoizer() @self.memoizer.memoize() def foo(): return None cache_key = foo.make_cache_key(foo.uncached) assert ( self.memoizer.get(cache_key) is self.memoizer.default_cache_value) result = foo() assert result is None assert self.memoizer.get(cache_key) is None self.memoizer.delete_memoized(foo) cache_key = foo.make_cache_key(foo.uncached) assert ( self.memoizer.get(cache_key) is self.memoizer.default_cache_value)
def test_31_update_memoized_backend_error(self, memoizer_set): logging.basicConfig() # disable logger (to avoid cluttering test output) logging.disable(logging.ERROR) memoizer = Memoizer() @memoizer.memoize() def f(): return random.randrange(0, 100000) old_value = self.memoizer.update_memoized(f) fname, instance_fname = function_namespace(f, args=[]) version_key = self.memoizer._memvname(fname) version_val = self.memoizer.get(version_key) memoizer_set.assert_called_with( version_key, version_val, timeout=f.cache_timeout ) memoizer_set.reset_mock() new_value = self.memoizer.update_memoized(f) memoizer_set.assert_called_with( version_key, version_val, timeout=f.cache_timeout ) assert new_value != old_value # re-enable logger logging.disable(logging.NOTSET)
def setUp(self): self.store = {} self.records = [] self.memo = Memoizer(self.store, **self.memo_kwargs)
import json import os from os import path from pathlib import Path import gspread import pygsheets from memoize import Memoizer from oauth2client.service_account import ServiceAccountCredentials from pygsheets import Spreadsheet user_spreadsheet_store = {} memo = Memoizer(user_spreadsheet_store) class MissingGoogleKeyException(Exception): pass def get_credentials(): scope = ['https://spreadsheets.google.com/feeds', 'https://www.googleapis.com/auth/drive'] google_key = os.environ.get("PC_GOOGLE_KEY") if google_key: google_key_json_payload = json.loads(google_key) key_path = '/tmp/data.json' with open(key_path, 'w') as outfile: json.dump(google_key_json_payload, outfile) credentials = ServiceAccountCredentials.from_json_keyfile_name(key_path, scope) os.remove(key_path) return credentials
def make_auctions_app(global_conf, redis_url='redis://localhost:9002/1', redis_password='', redis_database='', sentinel_cluster_name='', sentinels='', external_couch_url='http://localhost:5000/auction', internal_couch_url='http://localhost:9000/', proxy_internal_couch_url='http://localhost:9000/', auctions_db='database', hash_secret_key='', timezone='Europe/Kiev', preferred_url_scheme='http', debug=False, auto_build=False, event_source_connection_limit=1000, limit_replications_progress=99, limit_replications_func='any'): """ [app:main] use = egg:openprocurement.auction#auctions_server redis_url = redis://:passwod@localhost:1111/0 external_couch_url = http://localhost:1111/auction internal_couch_url = http://localhost:9011/ auctions_db = auction timezone = Europe/Kiev """ auctions_server = components.queryUtility(IAuctionsServer) auctions_server.proxy_connection_pool = ConnectionPool(factory=Connection, max_size=20, backend='gevent') auctions_server.proxy_mappings = Memoizer({}) auctions_server.event_sources_pool = deque([]) auctions_server.config['PREFERRED_URL_SCHEME'] = preferred_url_scheme auctions_server.config['limit_replications_progress'] = float( limit_replications_progress) auctions_server.config['limit_replications_func'] = limit_replications_func auctions_server.config['REDIS'] = { 'redis': redis_url, 'redis_password': redis_password, 'redis_database': redis_database, 'sentinel_cluster_name': sentinel_cluster_name, 'sentinel': loads(sentinels) } auctions_server.config['event_source_connection_limit'] = int( event_source_connection_limit) auctions_server.config['EXT_COUCH_DB'] = urljoin(external_couch_url, auctions_db) auctions_server.add_url_rule('/' + auctions_db + '/<path:path>', 'couch_server_proxy', couch_server_proxy, methods=['GET']) auctions_server.add_url_rule('/' + auctions_db + '/', 'couch_server_proxy', couch_server_proxy, methods=['GET'], defaults={'path': ''}) auctions_server.add_url_rule('/' + auctions_db + '_secured/<path:path>', 'auth_couch_server_proxy', auth_couch_server_proxy, methods=['GET']) auctions_server.add_url_rule('/' + auctions_db + '_secured/', 'auth_couch_server_proxy', auth_couch_server_proxy, methods=['GET'], defaults={'path': ''}) auctions_server.config['INT_COUCH_URL'] = internal_couch_url auctions_server.config['PROXY_COUCH_URL'] = proxy_internal_couch_url auctions_server.config['COUCH_DB'] = auctions_db auctions_server.config['TIMEZONE'] = tz(timezone) auctions_server.couch_server = Server( auctions_server.config.get('INT_COUCH_URL'), session=Session(retry_delays=range(10))) if auctions_server.config['COUCH_DB'] not in auctions_server.couch_server: auctions_server.couch_server.create(auctions_server.config['COUCH_DB']) auctions_server.db = auctions_server.\ couch_server[auctions_server.config['COUCH_DB']] auctions_server.config['HASH_SECRET_KEY'] = hash_secret_key sync_design(auctions_server.db) for entry_point in iter_entry_points(PKG_NAMESPACE): plugin = entry_point.load() plugin(components) return auctions_server
from memoize import Memoizer import requests import utils.errors as errors from utils import config memoKeys = {} memo = Memoizer(memoKeys) @memo(max_age=3600) def validate_token(token): if not isinstance(token, str) or len(token) == 0: raise errors.InvalidAuth() headers = {'Authorization'.encode(): token.encode()} host = config.get_auth_server_url() port = config.get_auth_server_port() auth_url = ':'.join([host, str(port)]) response = requests.get('/'.join(['http:/', auth_url, 'v1/users/current']), headers=headers) if response.status_code != 200: raise errors.InvalidAuth() result = response.json() if not result: raise errors.InvalidAuth()
class MemoizeTestCase(SimpleTestCase): def setUp(self): self.memoizer = Memoizer() def test_00_set(self): self.memoizer.set('hi', 'hello') assert self.memoizer.get('hi') == 'hello' def test_01_add(self): self.memoizer.add('hi', 'hello') assert self.memoizer.get('hi') == 'hello' self.memoizer.add('hi', 'foobar') assert self.memoizer.get('hi') == 'hello' def test_02_delete(self): self.memoizer.set('hi', 'hello') self.memoizer.delete('hi') assert self.memoizer.get('hi') is self.memoizer.default_cache_value def test_06_memoize(self): @self.memoizer.memoize(5) def big_foo(a, b): return a + b + random.randrange(0, 100000) result = big_foo(5, 2) time.sleep(1) assert big_foo(5, 2) == result result2 = big_foo(5, 3) assert result2 != result time.sleep(6) assert big_foo(5, 2) != result time.sleep(1) assert big_foo(5, 3) != result2 def test_06a_memoize(self): @self.memoizer.memoize(50) def big_foo(a, b): return a + b + random.randrange(0, 100000) result = big_foo(5, 2) time.sleep(2) assert big_foo(5, 2) == result def test_07_delete_memoize(self): @self.memoizer.memoize(5) def big_foo(a, b): return a + b + random.randrange(0, 100000) result = big_foo(5, 2) result2 = big_foo(5, 3) time.sleep(1) assert big_foo(5, 2) == result assert big_foo(5, 2) == result assert big_foo(5, 3) != result assert big_foo(5, 3) == result2 self.memoizer.delete_memoized(big_foo) assert big_foo(5, 2) != result assert big_foo(5, 3) != result2 def test_07b_delete_memoized_verhash(self): @self.memoizer.memoize(5) def big_foo(a, b): return a + b + random.randrange(0, 100000) result = big_foo(5, 2) result2 = big_foo(5, 3) time.sleep(1) assert big_foo(5, 2) == result assert big_foo(5, 2) == result assert big_foo(5, 3) != result assert big_foo(5, 3) == result2 self.memoizer.delete_memoized_verhash(big_foo) _fname, _fname_instance = function_namespace(big_foo) version_key = self.memoizer._memvname(_fname) assert (self.memoizer.get(version_key) is self.memoizer.default_cache_value) assert big_foo(5, 2) != result assert big_foo(5, 3) != result2 assert (self.memoizer.get(version_key) is not self.memoizer.default_cache_value) def test_08_delete_memoize(self): @self.memoizer.memoize() def big_foo(a, b): return a + b + random.randrange(0, 100000) result_a = big_foo(5, 1) result_b = big_foo(5, 2) assert big_foo(5, 1) == result_a assert big_foo(5, 2) == result_b self.memoizer.delete_memoized(big_foo, 5, 2) assert big_foo(5, 1) == result_a assert big_foo(5, 2) != result_b # Cleanup bigfoo 5,1 5,2 or it might conflict with # following run if it also uses memecache self.memoizer.delete_memoized(big_foo, 5, 2) self.memoizer.delete_memoized(big_foo, 5, 1) def test_09_args_memoize(self): @self.memoizer.memoize() def big_foo(a, b): return sum(a) + sum(b) + random.randrange(0, 100000) result_a = big_foo([5, 3, 2], [1]) result_b = big_foo([3, 3], [3, 1]) assert big_foo([5, 3, 2], [1]) == result_a assert big_foo([3, 3], [3, 1]) == result_b self.memoizer.delete_memoized(big_foo, [5, 3, 2], [1]) assert big_foo([5, 3, 2], [1]) != result_a assert big_foo([3, 3], [3, 1]) == result_b # Cleanup bigfoo 5,1 5,2 or it might conflict with # following run if it also uses memecache self.memoizer.delete_memoized(big_foo, [5, 3, 2], [1]) self.memoizer.delete_memoized(big_foo, [3, 3], [1]) def test_10_kwargs_memoize(self): @self.memoizer.memoize() def big_foo(a, b=None): return a + sum(b.values()) + random.randrange(0, 100000) result_a = big_foo(1, dict(one=1, two=2)) result_b = big_foo(5, dict(three=3, four=4)) assert big_foo(1, dict(one=1, two=2)) == result_a assert big_foo(5, dict(three=3, four=4)) == result_b self.memoizer.delete_memoized(big_foo, 1, dict(one=1, two=2)) assert big_foo(1, dict(one=1, two=2)) != result_a assert big_foo(5, dict(three=3, four=4)) == result_b def test_10a_kwargonly_memoize(self): @self.memoizer.memoize() def big_foo(a=None): if a is None: a = 0 return a + random.random() result_a = big_foo() result_b = big_foo(5) assert big_foo() == result_a assert big_foo() < 1 assert big_foo(5) == result_b assert big_foo(5) >= 5 and big_foo(5) < 6 def test_10a_arg_kwarg_memoize(self): @self.memoizer.memoize() def f(a, b, c=1): return a + b + c + random.randrange(0, 100000) assert f(1, 2) == f(1, 2, c=1) assert f(1, 2) == f(1, 2, 1) assert f(1, 2) == f(1, 2) assert f(1, 2) != f(2, 1) assert f(1, 2, 3) != f(1, 2) with self.assertRaises(TypeError): f(1) def test_10ab_arg_kwarg_memoize(self): @self.memoizer.memoize() def f(a, **kwargs): return (a, kwargs, random.randrange(0, 100000000)) f1 = f(1, b=2) assert f1 == f(1, b=2) assert f1 != f(1, b=4) f2 = f(5, c=6) assert f2 == f(5, c=6) assert f2 != f(7, c=6) self.memoizer.delete_memoized(f, 1, b=2) assert f1 != f(1, b=2) assert f2 == f(5, c=6) assert f2 != f(7, c=6) def test_10ac_arg_kwarg_memoize(self): @self.memoizer.memoize() def f(a, *args, **kwargs): return (a, args, kwargs, random.randrange(0, 100000000)) f1 = f(1, 2, b=3) assert f1 == f(1, 2, b=3) assert f1 != f(1, 3, b=3) f2 = f(5, 6, c=7) assert f2 == f(5, 6, c=7) assert f2 != f(7, 6, c=7) self.memoizer.delete_memoized(f, 1, 2, b=3) assert f1 != f(1, 2, b=3) assert f2 == f(5, 6, c=7) assert f2 != f(7, 6, c=7) @self.memoizer.memoize() def f(a, b=1, *args, **kwargs): return (a, b, args, kwargs, random.randrange(0, 100000000)) f3 = f(1, 3, 4) assert f3 == f(1, 3, 4) assert f3 == f(*(1, 3, 4)) assert f3 != f(1, 3) self.memoizer.delete_memoized(f, 1, 3, 4) assert f3 != f(1, 3, 4) def test_10b_classarg_memoize(self): @self.memoizer.memoize() def bar(a, *args): return a.value + random.random() + sum(args) class Adder(object): def __init__(self, value, *args): self.value = (value + sum(args)) adder = Adder(15) adder2 = Adder(20) adder3 = Adder(16, 5) adder4 = Adder(21, 6) w = bar(adder) x = bar(adder2) y = bar(adder3) z = bar(adder4) assert w != x assert y != z assert bar(adder) == w assert bar(adder) != x assert bar(adder3) == y assert bar(adder3) != z adder.value = 14 adder3.value = 15 assert bar(adder) == w assert bar(adder) != x assert bar(adder3) == y assert bar(adder3) != z assert bar(adder) != bar(adder2) assert bar(adder3) != bar(adder4) assert bar(adder2) == x assert bar(adder4) == z def test_10c_classfunc_memoize(self): class Adder(object): def __init__(self, initial): self.initial = initial @self.memoizer.memoize() def add(self, b, *args): return self.initial + b + sum(args) adder1 = Adder(1) adder2 = Adder(2) x = adder1.add(3) y = adder1.add(3, 4) assert adder1.add(3) == x assert adder1.add(4) != x assert adder1.add(3, 4) == y assert adder1.add(4, 4) != y assert adder1.add(3) != adder2.add(3) def test_10d_classfunc_memoize_delete(self): class Adder(object): def __init__(self, initial): self.initial = initial @self.memoizer.memoize() def add(self, b, *args): return self.initial + b + sum(args) + random.random() adder1 = Adder(1) adder2 = Adder(2) a1 = adder1.add(3) a2 = adder2.add(3) b1 = adder1.add(3, 1) b2 = adder2.add(3, 1) assert a1 != a2 assert a1 != b1 assert a2 != b2 assert adder1.add(3) == a1 assert adder2.add(3) == a2 assert adder1.add(3, 1) == b1 assert adder2.add(3, 1) == b2 self.memoizer.delete_memoized(adder1.add) a3 = adder1.add(3) a4 = adder2.add(3) b3 = adder1.add(3, 1) b4 = adder2.add(3, 1) self.assertNotEqual(a1, a3) self.assertNotEqual(b1, b3) assert a1 != a3 assert b1 != b3 self.assertEqual(a2, a4) self.assertEqual(b2, b4) self.memoizer.delete_memoized(Adder.add) a5 = adder1.add(3) a6 = adder2.add(3) b5 = adder1.add(3, 1) b6 = adder2.add(3, 1) self.assertNotEqual(a5, a6) self.assertNotEqual(b5, b6) self.assertNotEqual(a3, a5) self.assertNotEqual(b3, b5) self.assertNotEqual(a4, a6) self.assertNotEqual(b4, b6) def test_10e_delete_memoize_classmethod(self): class Mock(object): @classmethod @self.memoizer.memoize(5) def big_foo(cls, a, b, *args): return a + b + sum(args) + random.randrange(0, 100000) result = Mock.big_foo(5, 2) result2 = Mock.big_foo(5, 3) result3 = Mock.big_foo(5, 2, 1) result4 = Mock.big_foo(5, 3, 1) time.sleep(1) assert Mock.big_foo(5, 2) == result assert Mock.big_foo(5, 2) == result assert Mock.big_foo(5, 3) != result assert Mock.big_foo(5, 3) == result2 assert Mock.big_foo(5, 2, 1) == result3 assert Mock.big_foo(5, 2, 1) == result3 assert Mock.big_foo(5, 3, 1) != result3 assert Mock.big_foo(5, 3, 1) == result4 self.memoizer.delete_memoized(Mock.big_foo) assert Mock.big_foo(5, 2) != result assert Mock.big_foo(5, 3) != result2 assert Mock.big_foo(5, 2, 1) != result3 assert Mock.big_foo(5, 3, 1) != result4 def test_14_memoized_multiple_arg_kwarg_calls(self): @self.memoizer.memoize() def big_foo(a, b, c=[1, 1], d=[1, 1]): return sum(a) + sum(b) + sum(c) + sum(d) + random.randrange( 0, 100000) result_a = big_foo([5, 3, 2], [1], c=[3, 3], d=[3, 3]) assert big_foo([5, 3, 2], [1], d=[3, 3], c=[3, 3]) == result_a assert big_foo(b=[1], a=[5, 3, 2], c=[3, 3], d=[3, 3]) == result_a assert big_foo([5, 3, 2], [1], [3, 3], [3, 3]) == result_a def test_15_memoize_multiple_arg_kwarg_delete(self): @self.memoizer.memoize() def big_foo(a, b, c=[1, 1], d=[1, 1]): return sum(a) + sum(b) + sum(c) + sum(d) + random.randrange( 0, 100000) result_a = big_foo([5, 3, 2], [1], c=[3, 3], d=[3, 3]) self.memoizer.delete_memoized(big_foo, [5, 3, 2], [1], [3, 3], [3, 3]) result_b = big_foo([5, 3, 2], [1], c=[3, 3], d=[3, 3]) assert result_a != result_b self.memoizer.delete_memoized(big_foo, [5, 3, 2], b=[1], c=[3, 3], d=[3, 3]) result_b = big_foo([5, 3, 2], [1], c=[3, 3], d=[3, 3]) assert result_a != result_b self.memoizer.delete_memoized(big_foo, [5, 3, 2], [1], c=[3, 3], d=[3, 3]) result_a = big_foo([5, 3, 2], [1], c=[3, 3], d=[3, 3]) assert result_a != result_b self.memoizer.delete_memoized(big_foo, [5, 3, 2], b=[1], c=[3, 3], d=[3, 3]) result_a = big_foo([5, 3, 2], [1], c=[3, 3], d=[3, 3]) assert result_a != result_b self.memoizer.delete_memoized(big_foo, [5, 3, 2], [1], c=[3, 3], d=[3, 3]) result_b = big_foo([5, 3, 2], [1], c=[3, 3], d=[3, 3]) assert result_a != result_b self.memoizer.delete_memoized(big_foo, [5, 3, 2], [1], [3, 3], [3, 3]) result_a = big_foo([5, 3, 2], [1], c=[3, 3], d=[3, 3]) assert result_a != result_b def test_16_memoize_kwargs_to_args(self): def big_foo(a, b, c=None, d=None): return sum(a) + sum(b) + random.randrange(0, 100000) expected = (1, 2, 'foo', 'bar') args, kwargs = self.memoizer._memoize_kwargs_to_args( big_foo, 1, 2, 'foo', 'bar') assert (args == expected) args, kwargs = self.memoizer._memoize_kwargs_to_args(big_foo, 2, 'foo', 'bar', a=1) assert (args == expected) args, kwargs = self.memoizer._memoize_kwargs_to_args(big_foo, a=1, b=2, c='foo', d='bar') assert (args == expected) args, kwargs = self.memoizer._memoize_kwargs_to_args(big_foo, d='bar', b=2, a=1, c='foo') assert (args == expected) args, kwargs = self.memoizer._memoize_kwargs_to_args(big_foo, 1, 2, d='bar', c='foo') assert (args == expected) def test_17_memoize_none_value(self): self.memoizer = Memoizer() @self.memoizer.memoize() def foo(): return None cache_key = foo.make_cache_key(foo.uncached) assert (self.memoizer.get(cache_key) is self.memoizer.default_cache_value) result = foo() assert result is None assert self.memoizer.get(cache_key) is None self.memoizer.delete_memoized(foo) cache_key = foo.make_cache_key(foo.uncached) assert (self.memoizer.get(cache_key) is self.memoizer.default_cache_value) def test_17_delete_memoized_instancemethod_with_mutable_param(self): class Foo(object): def __init__(self, id): self.id = id @self.memoizer.memoize(5) def foo(self, bar_obj): return random.randrange(0, 100000) + bar_obj.id def __repr__(self): return ('{}({})'.format(self.__class__.__name__, self.id)) class Bar(object): def __init__(self, id): self.id = id def __repr__(self): return ('{}({})'.format(self.__class__.__name__, self.id)) a = Foo(1) b = Bar(1) c = Bar(2) result1 = a.foo(b) result2 = a.foo(c) time.sleep(1) assert (a.foo(b) == result1) assert (a.foo(c) == result2) self.memoizer.delete_memoized(a.foo, a, b) assert (a.foo(b) != result1) assert (a.foo(c) == result2)
class MemoizeTestCase(TestCase): def setUp(self): self.memoizer = Memoizer() def test_00_set(self): self.memoizer.set('hi', 'hello') assert self.memoizer.get('hi') == 'hello' def test_01_add(self): self.memoizer.add('hi', 'hello') assert self.memoizer.get('hi') == 'hello' self.memoizer.add('hi', 'foobar') assert self.memoizer.get('hi') == 'hello' def test_02_delete(self): self.memoizer.set('hi', 'hello') self.memoizer.delete('hi') assert self.memoizer.get('hi') is None def test_06_memoize(self): @self.memoizer.memoize(5) def big_foo(a, b): return a + b + random.randrange(0, 100000) result = big_foo(5, 2) time.sleep(1) assert big_foo(5, 2) == result result2 = big_foo(5, 3) assert result2 != result time.sleep(6) assert big_foo(5, 2) != result time.sleep(1) assert big_foo(5, 3) != result2 def test_06a_memoize(self): @self.memoizer.memoize(50) def big_foo(a, b): return a + b + random.randrange(0, 100000) result = big_foo(5, 2) time.sleep(2) assert big_foo(5, 2) == result def test_07_delete_memoize(self): @self.memoizer.memoize(5) def big_foo(a, b): return a + b + random.randrange(0, 100000) result = big_foo(5, 2) result2 = big_foo(5, 3) time.sleep(1) assert big_foo(5, 2) == result assert big_foo(5, 2) == result assert big_foo(5, 3) != result assert big_foo(5, 3) == result2 self.memoizer.delete_memoized(big_foo) assert big_foo(5, 2) != result assert big_foo(5, 3) != result2 def test_07b_delete_memoized_verhash(self): @self.memoizer.memoize(5) def big_foo(a, b): return a + b + random.randrange(0, 100000) result = big_foo(5, 2) result2 = big_foo(5, 3) time.sleep(1) assert big_foo(5, 2) == result assert big_foo(5, 2) == result assert big_foo(5, 3) != result assert big_foo(5, 3) == result2 self.memoizer.delete_memoized_verhash(big_foo) _fname, _fname_instance = function_namespace(big_foo) version_key = self.memoizer._memvname(_fname) assert self.memoizer.get(version_key) is None assert big_foo(5, 2) != result assert big_foo(5, 3) != result2 assert self.memoizer.get(version_key) is not None def test_08_delete_memoize(self): @self.memoizer.memoize() def big_foo(a, b): return a + b + random.randrange(0, 100000) result_a = big_foo(5, 1) result_b = big_foo(5, 2) assert big_foo(5, 1) == result_a assert big_foo(5, 2) == result_b self.memoizer.delete_memoized(big_foo, 5, 2) assert big_foo(5, 1) == result_a assert big_foo(5, 2) != result_b ## Cleanup bigfoo 5,1 5,2 or it might conflict with ## following run if it also uses memecache self.memoizer.delete_memoized(big_foo, 5, 2) self.memoizer.delete_memoized(big_foo, 5, 1) def test_09_args_memoize(self): @self.memoizer.memoize() def big_foo(a, b): return sum(a) + sum(b) + random.randrange(0, 100000) result_a = big_foo([5, 3, 2], [1]) result_b = big_foo([3, 3], [3, 1]) assert big_foo([5, 3, 2], [1]) == result_a assert big_foo([3, 3], [3, 1]) == result_b self.memoizer.delete_memoized(big_foo, [5, 3, 2], [1]) assert big_foo([5, 3, 2], [1]) != result_a assert big_foo([3, 3], [3, 1]) == result_b ## Cleanup bigfoo 5,1 5,2 or it might conflict with ## following run if it also uses memecache self.memoizer.delete_memoized(big_foo, [5, 3, 2], [1]) self.memoizer.delete_memoized(big_foo, [3, 3], [1]) def test_10_kwargs_memoize(self): @self.memoizer.memoize() def big_foo(a, b=None): return a + sum(b.values()) + random.randrange(0, 100000) result_a = big_foo(1, dict(one=1, two=2)) result_b = big_foo(5, dict(three=3, four=4)) assert big_foo(1, dict(one=1, two=2)) == result_a assert big_foo(5, dict(three=3, four=4)) == result_b self.memoizer.delete_memoized(big_foo, 1, dict(one=1, two=2)) assert big_foo(1, dict(one=1, two=2)) != result_a assert big_foo(5, dict(three=3, four=4)) == result_b def test_10a_kwargonly_memoize(self): @self.memoizer.memoize() def big_foo(a=None): if a is None: a = 0 return a + random.random() result_a = big_foo() result_b = big_foo(5) assert big_foo() == result_a assert big_foo() < 1 assert big_foo(5) == result_b assert big_foo(5) >= 5 and big_foo(5) < 6 def test_10a_arg_kwarg_memoize(self): @self.memoizer.memoize() def f(a, b, c=1): return a + b + c + random.randrange(0, 100000) assert f(1, 2) == f(1, 2, c=1) assert f(1, 2) == f(1, 2, 1) assert f(1, 2) == f(1, 2) assert f(1, 2, 3) != f(1, 2) with self.assertRaises(TypeError): f(1) def test_10b_classarg_memoize(self): @self.memoizer.memoize() def bar(a): return a.value + random.random() class Adder(object): def __init__(self, value): self.value = value adder = Adder(15) adder2 = Adder(20) y = bar(adder) z = bar(adder2) assert y != z assert bar(adder) == y assert bar(adder) != z adder.value = 14 assert bar(adder) == y assert bar(adder) != z assert bar(adder) != bar(adder2) assert bar(adder2) == z def test_10c_classfunc_memoize(self): class Adder(object): def __init__(self, initial): self.initial = initial @self.memoizer.memoize() def add(self, b): return self.initial + b adder1 = Adder(1) adder2 = Adder(2) x = adder1.add(3) assert adder1.add(3) == x assert adder1.add(4) != x assert adder1.add(3) != adder2.add(3) def test_10d_classfunc_memoize_delete(self): class Adder(object): def __init__(self, initial): self.initial = initial @self.memoizer.memoize() def add(self, b): return self.initial + b + random.random() adder1 = Adder(1) adder2 = Adder(2) a1 = adder1.add(3) a2 = adder2.add(3) assert a1 != a2 assert adder1.add(3) == a1 assert adder2.add(3) == a2 self.memoizer.delete_memoized(adder1.add) a3 = adder1.add(3) a4 = adder2.add(3) self.assertNotEqual(a1, a3) assert a1 != a3 self.assertEqual(a2, a4) self.memoizer.delete_memoized(Adder.add) a5 = adder1.add(3) a6 = adder2.add(3) self.assertNotEqual(a5, a6) self.assertNotEqual(a3, a5) self.assertNotEqual(a4, a6) def test_10e_delete_memoize_classmethod(self): class Mock(object): @classmethod @self.memoizer.memoize(5) def big_foo(cls, a, b): return a + b + random.randrange(0, 100000) result = Mock.big_foo(5, 2) result2 = Mock.big_foo(5, 3) time.sleep(1) assert Mock.big_foo(5, 2) == result assert Mock.big_foo(5, 2) == result assert Mock.big_foo(5, 3) != result assert Mock.big_foo(5, 3) == result2 self.memoizer.delete_memoized(Mock.big_foo) assert Mock.big_foo(5, 2) != result assert Mock.big_foo(5, 3) != result2 def test_14_memoized_multiple_arg_kwarg_calls(self): @self.memoizer.memoize() def big_foo(a, b, c=[1, 1], d=[1, 1]): return sum(a) + sum(b) + sum(c) + sum(d) + random.randrange( 0, 100000) result_a = big_foo([5, 3, 2], [1], c=[3, 3], d=[3, 3]) assert big_foo([5, 3, 2], [1], d=[3, 3], c=[3, 3]) == result_a assert big_foo(b=[1], a=[5, 3, 2], c=[3, 3], d=[3, 3]) == result_a assert big_foo([5, 3, 2], [1], [3, 3], [3, 3]) == result_a def test_15_memoize_multiple_arg_kwarg_delete(self): @self.memoizer.memoize() def big_foo(a, b, c=[1, 1], d=[1, 1]): return sum(a) + sum(b) + sum(c) + sum(d) + random.randrange( 0, 100000) result_a = big_foo([5, 3, 2], [1], c=[3, 3], d=[3, 3]) self.memoizer.delete_memoized(big_foo, [5, 3, 2], [1], [3, 3], [3, 3]) result_b = big_foo([5, 3, 2], [1], c=[3, 3], d=[3, 3]) assert result_a != result_b self.memoizer.delete_memoized(big_foo, [5, 3, 2], b=[1], c=[3, 3], d=[3, 3]) result_b = big_foo([5, 3, 2], [1], c=[3, 3], d=[3, 3]) assert result_a != result_b self.memoizer.delete_memoized(big_foo, [5, 3, 2], [1], c=[3, 3], d=[3, 3]) result_a = big_foo([5, 3, 2], [1], c=[3, 3], d=[3, 3]) assert result_a != result_b self.memoizer.delete_memoized(big_foo, [5, 3, 2], b=[1], c=[3, 3], d=[3, 3]) result_a = big_foo([5, 3, 2], [1], c=[3, 3], d=[3, 3]) assert result_a != result_b self.memoizer.delete_memoized(big_foo, [5, 3, 2], [1], c=[3, 3], d=[3, 3]) result_b = big_foo([5, 3, 2], [1], c=[3, 3], d=[3, 3]) assert result_a != result_b self.memoizer.delete_memoized(big_foo, [5, 3, 2], [1], [3, 3], [3, 3]) result_a = big_foo([5, 3, 2], [1], c=[3, 3], d=[3, 3]) assert result_a != result_b def test_16_memoize_kwargs_to_args(self): def big_foo(a, b, c=None, d=None): return sum(a) + sum(b) + random.randrange(0, 100000) expected = (1, 2, 'foo', 'bar') args, kwargs = self.memoizer._memoize_kwargs_to_args( big_foo, 1, 2, 'foo', 'bar') assert (args == expected) args, kwargs = self.memoizer._memoize_kwargs_to_args(big_foo, 2, 'foo', 'bar', a=1) assert (args == expected) args, kwargs = self.memoizer._memoize_kwargs_to_args(big_foo, a=1, b=2, c='foo', d='bar') assert (args == expected) args, kwargs = self.memoizer._memoize_kwargs_to_args(big_foo, d='bar', b=2, a=1, c='foo') assert (args == expected) args, kwargs = self.memoizer._memoize_kwargs_to_args(big_foo, 1, 2, d='bar', c='foo') assert (args == expected)
def setUp(self): self.memoizer = Memoizer()
from memoize import Memoizer PYPI_URL_PATTERN = 'https://pypi.python.org/pypi/{package}/json' CHANNEL_URL_PATTERN = 'https://conda.anaconda.org/{channel}/{platform}/repodata.json' REDIS_CONN = redis.StrictRedis(host=os.environ.get('REDIS_HOST', '127.0.0.1'), port=os.environ.get('REDIS_PORT', '6379'), password=os.environ.get('REDIS_PASSWORD', '¯\_(ツ)_/¯')) CHANNELS = ['conda-forge', 'anaconda', 'c3i_test'] PKG_INFO = {} memo = Memoizer({}) # https://stackoverflow.com/a/34366589/1005215 def get_pypi_version(package, url_pattern=PYPI_URL_PATTERN): """Return version of package on pypi.python.org using json.""" req = requests.get(url_pattern.format(package=package)) version = parse('0') if req.status_code == requests.codes.ok: j = json.loads(req.text.encode(req.encoding)) if 'releases' in j: versions = [parse(s) for s in j['releases']] filtered = [v for v in versions if not v.is_prerelease] if len(filtered) == 0: return max(versions) else:
class RedirectionManager: """ RedirectionManager The redirection manager handles subscribers who have redirection enabled, it adds the flows into ovs for redirecting user to the redirection server """ DNS_TIMEOUT_SECS = 15 REDIRECT_NOT_PROCESSED = REG_ZERO_VAL REDIRECT_PROCESSED = 0x1 RedirectRequest = namedtuple( 'RedirectRequest', ['imsi', 'ip_addr', 'rule', 'rule_num', 'rule_version', 'priority'], ) def __init__(self, bridge_ip, logger, main_tbl_num, next_table, scratch_table_num, session_rule_version_mapper): self._bridge_ip = bridge_ip self.logger = logger self.main_tbl_num = main_tbl_num self.next_table = next_table self._scratch_tbl_num = scratch_table_num self._redirect_dict = RedirectDict() self._dns_cache = Memoizer({}) self._redirect_port = get_service_config_value('redirectd', 'http_port', 8080) self._session_rule_version_mapper = session_rule_version_mapper self._cwf_args_set = False self._mac_rewrite_scratch = None self._internal_ip_allocator = None self._arpd_controller_fut = None self._arp_contoller = None self._egress_table = None self._bridge_mac = None def set_cwf_args(self, internal_ip_allocator, arp, mac_rewrite, bridge_name, egress_table): self._mac_rewrite_scratch = mac_rewrite self._internal_ip_allocator = internal_ip_allocator self._arpd_controller_fut = arp self._arp_contoller = None self._egress_table = egress_table def get_virtual_iface_mac(iface): virt_ifaddresses = netifaces.ifaddresses(iface) return virt_ifaddresses[netifaces.AF_LINK][0]['addr'] self._bridge_mac = get_virtual_iface_mac(bridge_name) self._cwf_args_set = True return self def setup_lte_redirect(self, datapath, loop, redirect_request): """ Depending on redirection server address type install redirection rules """ imsi = redirect_request.imsi ip_addr = redirect_request.ip_addr rule = redirect_request.rule rule_num = redirect_request.rule_num rule_version = redirect_request.rule_version priority = redirect_request.priority # TODO IMPORTANT check that redirectd service is running, as its a # dynamic service its not on by default. Will save you some sanity :) # TODO figure out what to do with SIP_URI if rule.redirect.address_type == rule.redirect.SIP_URI: raise RedirectException("SIP_URIs redirection isn't setup") if rule.redirect.address_type == rule.redirect.IPv6: raise RedirectException("No ipv6 support, so no ipv6 redirect") self._save_redirect_entry(ip_addr, rule.redirect) self._install_redirect_flows(datapath, loop, imsi, ip_addr, rule, rule_num, rule_version, priority) return def _install_redirect_flows(self, datapath, loop, imsi, ip_addr, rule, rule_num, rule_version, priority): """ Add flows to forward traffic to the redirection server. 1) Intercept tcp traffic to the web to the redirection server, which completes the tcp handshake. This is done by adding an OVS flow with a learn action (flow catches inbound tcp packets, while learn action creates another flow that sends packets back from server) 2) Add flows to allow UDP traffic so DNS queries can go through. Finally add flows with a higher priority that allow traffic to and from the address provided in redirect rule. """ if rule.redirect.address_type == rule.redirect.URL: self._install_url_bypass_flows(datapath, loop, imsi, rule, rule_num, rule_version, priority, ue_ip=ip_addr) elif rule.redirect.address_type == rule.redirect.IPv4: self._install_ipv4_bypass_flows(datapath, imsi, rule, rule_num, rule_version, priority, [rule.redirect.server_address], ue_ip=ip_addr) self._install_dns_flows(datapath, imsi, rule, rule_num, rule_version, priority) self._install_server_flows(datapath, imsi, ip_addr, rule, rule_num, rule_version, priority) def _install_scratch_table_flows(self, datapath, imsi, rule, rule_num, rule_version, priority): """ The flow action for subscribers that need to be redirected does 2 things * Forward requests from subscriber to the internal http server * Instantiate a flow that matches response packets from the server and sends them back to subscriber Match: incoming tcp traffic with port 80, direction out Action: 1) Set reg2 to rule_num 2) Set ip dst to server ip 3) Output to table 20 4) Apply LearnAction: LearnAction(adds new flow for every pkt flow that hits this rule) 1) Match ip packets 2) Match tcp protocol 3) Match packets from LOCAL port 4) Match ip src = server ip 5) Match ip dst = current flow ip src 6) Match tcp src = current flow tcp dst 7) Match tcp dst = current flow tcp src 8) Load ip src = current flow ip dst 9) Output through gtp0 """ parser = datapath.ofproto_parser match_http = MagmaMatch(eth_type=ether_types.ETH_TYPE_IP, ip_proto=IPPROTO_TCP, tcp_dst=80, imsi=encode_imsi(imsi), direction=Direction.OUT) of_note = parser.NXActionNote(list(rule.id.encode())) actions = [ parser.NXActionLearn( table_id=self.main_tbl_num, priority=priority, cookie=rule_num, specs=[ parser.NXFlowSpecMatch(src=ether_types.ETH_TYPE_IP, dst=('eth_type_nxm', 0), n_bits=16), parser.NXFlowSpecMatch(src=IPPROTO_TCP, dst=('ip_proto_nxm', 0), n_bits=8), parser.NXFlowSpecMatch(src=Direction.IN, dst=(DIRECTION_REG, 0), n_bits=32), parser.NXFlowSpecMatch(src=int( ipaddress.IPv4Address(self._bridge_ip)), dst=('ipv4_src_nxm', 0), n_bits=32), parser.NXFlowSpecMatch(src=('ipv4_src_nxm', 0), dst=('ipv4_dst_nxm', 0), n_bits=32), parser.NXFlowSpecMatch(src=('tcp_src_nxm', 0), dst=('tcp_dst_nxm', 0), n_bits=16), parser.NXFlowSpecMatch(src=self._redirect_port, dst=('tcp_src_nxm', 0), n_bits=16), parser.NXFlowSpecMatch(src=encode_imsi(imsi), dst=(IMSI_REG, 0), n_bits=64), parser.NXFlowSpecLoad(src=('ipv4_dst_nxm', 0), dst=('ipv4_src_nxm', 0), n_bits=32), parser.NXFlowSpecLoad(src=80, dst=('tcp_src_nxm', 0), n_bits=16), # Learn doesn't support resubmit to table, so send directly parser.NXFlowSpecOutput(src=('in_port', 0), dst="", n_bits=16), ]), parser.NXActionRegLoad2(dst=SCRATCH_REGS[0], value=self.REDIRECT_PROCESSED), parser.OFPActionSetField(ipv4_dst=self._bridge_ip), parser.OFPActionSetField(tcp_dst=self._redirect_port), of_note, ] actions += self._load_rule_actions(parser, rule_num, rule_version) flows.add_resubmit_current_service_flow( datapath, self._scratch_tbl_num, match_http, actions, priority=priority, cookie=rule_num, hard_timeout=rule.hard_timeout, resubmit_table=self.main_tbl_num) match = MagmaMatch(imsi=encode_imsi(imsi)) action = [] flows.add_drop_flow(datapath, self._scratch_tbl_num, match, action, priority=flows.MINIMUM_PRIORITY + 1, cookie=rule_num) def _install_not_processed_flows(self, datapath, imsi, ip_addr, rule, rule_num, priority): """ Redirect all traffic to the scratch table to only allow redirected http traffic to go through, the rest will be dropped. reg0 is used as a boolean to know whether the drop rule was processed. """ parser = datapath.ofproto_parser of_note = parser.NXActionNote(list(rule.id.encode())) match = MagmaMatch(imsi=encode_imsi(imsi), direction=Direction.OUT, reg0=self.REDIRECT_NOT_PROCESSED, eth_type=ether_types.ETH_TYPE_IP, ipv4_src=ip_addr) action = [of_note] flows.add_resubmit_current_service_flow( datapath, self.main_tbl_num, match, action, priority=priority, cookie=rule_num, hard_timeout=rule.hard_timeout, resubmit_table=self._scratch_tbl_num) match = MagmaMatch(imsi=encode_imsi(imsi), direction=Direction.OUT, reg0=self.REDIRECT_PROCESSED, eth_type=ether_types.ETH_TYPE_IP, ipv4_src=ip_addr) action = [of_note] flows.add_resubmit_next_service_flow(datapath, self.main_tbl_num, match, action, priority=priority, cookie=rule_num, hard_timeout=rule.hard_timeout, resubmit_table=self.next_table) def setup_cwf_redirect(self, datapath, loop, redirect_request): """ Add flows to forward traffic to the redirection server for cwf networks 1) Intercept tcp traffic to the web to the redirection server, which completes the tcp handshake. Also overwrite UE src ip to match the subnet of the redirection server. This is done by assigning an internal IP per each subscriber. Add an OVS flow with a learn action (flow catches inbound tcp http packets, while learn action creates another flow that rewrites packet back to send to ue) 2) Add flows to allow UDP traffic so DNS queries can go through. Add flows with a higher priority that allow traffic to and from the address provided in redirect rule. TODO we might want to track stats for these rules and report to sessiond """ if not self._cwf_args_set: raise RedirectException("Can't install cwf redirection, missing" "cwf specific args, call set_cwf_args()") imsi = redirect_request.imsi rule = redirect_request.rule rule_num = redirect_request.rule_num rule_version = redirect_request.rule_version priority = redirect_request.priority if rule.redirect.address_type == rule.redirect.URL: self._install_url_bypass_flows(datapath, loop, imsi, rule, rule_num, rule_version, priority) elif rule.redirect.address_type == rule.redirect.IPv4: self._install_ipv4_bypass_flows(datapath, imsi, rule, rule_num, rule_version, priority, [rule.redirect.server_address]) parser = datapath.ofproto_parser # TODO use subscriber ip_addr to generate internal IP and release # internal IP when subscriber disconnects or redirection flow is removed internal_ip = self._internal_ip_allocator.next_ip() self._save_redirect_entry(internal_ip, rule.redirect) #TODO check if we actually need this, dns might already be allowed self._install_dns_flows(datapath, imsi, rule, rule_num, rule_version, priority) match_tcp_80 = MagmaMatch(imsi=encode_imsi(imsi), eth_type=ether_types.ETH_TYPE_IP, ip_proto=IPPROTO_TCP, direction=Direction.OUT, tcp_dst=80) match_tcp_8008 = MagmaMatch(imsi=encode_imsi(imsi), eth_type=ether_types.ETH_TYPE_IP, ip_proto=IPPROTO_TCP, direction=Direction.OUT, tcp_dst=8080) match_tcp_8080 = MagmaMatch(imsi=encode_imsi(imsi), eth_type=ether_types.ETH_TYPE_IP, ip_proto=IPPROTO_TCP, direction=Direction.OUT, tcp_dst=8008) actions = [ parser.NXActionLearn( table_id=self._mac_rewrite_scratch, priority=flows.UE_FLOW_PRIORITY, cookie=rule_num, specs=[ parser.NXFlowSpecMatch(src=ether_types.ETH_TYPE_IP, dst=('eth_type_nxm', 0), n_bits=16), parser.NXFlowSpecMatch(src=IPPROTO_TCP, dst=('ip_proto_nxm', 0), n_bits=8), parser.NXFlowSpecMatch(src=int( ipaddress.IPv4Address(self._bridge_ip)), dst=('ipv4_src_nxm', 0), n_bits=32), parser.NXFlowSpecMatch(src=int(internal_ip), dst=('ipv4_dst_nxm', 0), n_bits=32), parser.NXFlowSpecMatch(src=('tcp_src_nxm', 0), dst=('tcp_dst_nxm', 0), n_bits=16), parser.NXFlowSpecMatch(src=self._redirect_port, dst=('tcp_src_nxm', 0), n_bits=16), parser.NXFlowSpecLoad(src=('eth_src_nxm', 0), dst=('eth_dst_nxm', 0), n_bits=48), parser.NXFlowSpecLoad(src=encode_imsi(imsi), dst=(IMSI_REG, 0), n_bits=64), parser.NXFlowSpecLoad(src=('ipv4_src_nxm', 0), dst=('ipv4_dst_nxm', 0), n_bits=32), parser.NXFlowSpecLoad(src=('ipv4_dst_nxm', 0), dst=('ipv4_src_nxm', 0), n_bits=32), parser.NXFlowSpecLoad(src=('tcp_dst_nxm', 0), dst=('tcp_src_nxm', 0), n_bits=16), ]), parser.OFPActionSetField(ipv4_src=str(internal_ip)), parser.OFPActionSetField(ipv4_dst=self._bridge_ip), parser.OFPActionSetField(eth_dst=self._bridge_mac), parser.OFPActionSetField(tcp_dst=self._redirect_port), ] flows.add_output_flow(datapath, self.main_tbl_num, match_tcp_80, actions, priority=flows.UE_FLOW_PRIORITY, cookie=rule_num, output_port=OFPP_LOCAL) flows.add_output_flow(datapath, self.main_tbl_num, match_tcp_8008, actions, priority=flows.UE_FLOW_PRIORITY, cookie=rule_num, output_port=OFPP_LOCAL) flows.add_output_flow(datapath, self.main_tbl_num, match_tcp_8080, actions, priority=flows.UE_FLOW_PRIORITY, cookie=rule_num, output_port=OFPP_LOCAL) # Add flows for vlan traffic too (we need to pop vlan for flask server) # In ryu vlan_vid=(0x1000, 0x1000) matches all vlans match_tcp_80_vlan = MagmaMatch(imsi=encode_imsi(imsi), eth_type=ether_types.ETH_TYPE_IP, ip_proto=IPPROTO_TCP, direction=Direction.OUT, tcp_dst=80, vlan_vid=(0x1000, 0x1000)) match_tcp_8008_vlan = MagmaMatch(imsi=encode_imsi(imsi), eth_type=ether_types.ETH_TYPE_IP, ip_proto=IPPROTO_TCP, direction=Direction.OUT, tcp_dst=8080, vlan_vid=(0x1000, 0x1000)) match_tcp_8080_vlan = MagmaMatch(imsi=encode_imsi(imsi), eth_type=ether_types.ETH_TYPE_IP, ip_proto=IPPROTO_TCP, direction=Direction.OUT, tcp_dst=8008, vlan_vid=(0x1000, 0x1000)) actions.append(parser.OFPActionPopVlan()) flows.add_output_flow(datapath, self.main_tbl_num, match_tcp_80_vlan, actions, priority=flows.UE_FLOW_PRIORITY + 1, cookie=rule_num, output_port=OFPP_LOCAL) flows.add_output_flow(datapath, self.main_tbl_num, match_tcp_8008_vlan, actions, priority=flows.UE_FLOW_PRIORITY + 1, cookie=rule_num, output_port=OFPP_LOCAL) flows.add_output_flow(datapath, self.main_tbl_num, match_tcp_8080_vlan, actions, priority=flows.UE_FLOW_PRIORITY + 1, cookie=rule_num, output_port=OFPP_LOCAL) # TODO cleanup, make this a default rule in the ue_mac table ue_tbl = 0 ue_next_tbl = 1 # Allows traffic back from the flask server match = MagmaMatch(in_port=OFPP_LOCAL) actions = [ parser.NXActionResubmitTable(table_id=self._mac_rewrite_scratch) ] flows.add_resubmit_next_service_flow(datapath, ue_tbl, match, actions=actions, priority=flows.DEFAULT_PRIORITY, resubmit_table=ue_next_tbl) match = MagmaMatch(imsi=encode_imsi(imsi), eth_type=ether_types.ETH_TYPE_IP, ip_proto=IPPROTO_TCP, direction=Direction.IN, in_port=OFPP_LOCAL) flows.add_resubmit_next_service_flow(datapath, self.main_tbl_num, match, [], priority=flows.DEFAULT_PRIORITY, cookie=rule_num, resubmit_table=self._egress_table) # Mac doesn't matter as we rewrite it anwyays mac_addr = '01:02:03:04:05:06' if self._arp_contoller or self._arpd_controller_fut.done(): if not self._arp_contoller: self._arp_contoller = self._arpd_controller_fut.result() self._arp_contoller.set_incoming_arp_flows(datapath, internal_ip, mac_addr) # Drop all other traffic that doesn't match match = MagmaMatch(imsi=encode_imsi(imsi)) flows.add_drop_flow(datapath, self.main_tbl_num, match, [], priority=flows.MINIMUM_PRIORITY + 1, cookie=rule_num) def _install_server_flows(self, datapath, imsi, ip_addr, rule, rule_num, rule_version, priority): """ Install the redirect flows to redirect all HTTP traffic to the captive portal and to drop all of the rest. """ self._install_scratch_table_flows(datapath, imsi, rule, rule_num, rule_version, priority) self._install_not_processed_flows(datapath, imsi, ip_addr, rule, rule_num, priority) def _install_url_bypass_flows(self, datapath, loop, imsi, rule, rule_num, rule_version, priority, ue_ip=None): """ Resolve DNS queries to get the ip address of redirect url, this is done to allow traffic to safely pass through as we want subscribers to have full access to the url they are redirected to. First check cache for redirect url, if not in cache submit a DNS query """ redirect_addr_host = urlsplit(rule.redirect.server_address).netloc cached_ips = self._dns_cache.get(redirect_addr_host) if cached_ips is not None: self.logger.debug( "DNS cache hit for {}, entry expires in {} sec".format( redirect_addr_host, self._dns_cache.ttl(redirect_addr_host))) self._install_ipv4_bypass_flows(datapath, imsi, rule, rule_num, rule_version, priority, cached_ips, ue_ip) return resolver = aiodns.DNSResolver(timeout=self.DNS_TIMEOUT_SECS, loop=loop) query = resolver.query(redirect_addr_host, 'A') def add_flows(dns_resolve_future): """ Callback for when DNS query is resolved, adds the bypass flows """ try: ips = [entry.host for entry in dns_resolve_future.result()] ttl = min(entry.ttl for entry in dns_resolve_future.result()) except aiodns.error.DNSError as err: self.logger.error("Error: ip lookup for {}: {}".format( redirect_addr_host, err)) return self._dns_cache.get(redirect_addr_host, lambda: ips, max_age=ttl) self._install_ipv4_bypass_flows(datapath, imsi, rule, rule_num, rule_version, priority, ips, ue_ip) asyncio.ensure_future(query, loop=loop).add_done_callback(add_flows) def _install_ipv4_bypass_flows(self, datapath, imsi, rule, rule_num, rule_version, priority, ips, ue_ip=None): """ Installs flows for traffic that is allowed to pass through for subscriber who has redirection enabled. Allow access to all passed ips. Allow UDP traffic(for DNS queries), traffic to/from redirection address """ parser = datapath.ofproto_parser of_note = parser.NXActionNote(list(rule.id.encode())) actions = [ of_note, ] actions += self._load_rule_actions(parser, rule_num, rule_version) matches = [] uplink_ip_match = {} downlink_ip_match = {} if ue_ip != None: uplink_ip_match['ipv4_src'] = ue_ip downlink_ip_match['ipv4_dst'] = ue_ip for ip in ips: matches.append( MagmaMatch(eth_type=ether_types.ETH_TYPE_IP, direction=Direction.OUT, ipv4_dst=ip, imsi=encode_imsi(imsi), **uplink_ip_match)) matches.append( MagmaMatch(eth_type=ether_types.ETH_TYPE_IP, direction=Direction.IN, ipv4_src=ip, imsi=encode_imsi(imsi), **downlink_ip_match)) for match in matches: flows.add_resubmit_next_service_flow( datapath, self.main_tbl_num, match, actions, priority=priority + 1, cookie=rule_num, hard_timeout=rule.hard_timeout, resubmit_table=self.next_table) def _install_dns_flows(self, datapath, imsi, rule, rule_num, rule_version, priority): """ Installs flows that allow DNS queries to path through. """ parser = datapath.ofproto_parser of_note = parser.NXActionNote(list(rule.id.encode())) actions = [ of_note, ] actions += self._load_rule_actions(parser, rule_num, rule_version) matches = [] # Install UDP flows for DNS matches.append( MagmaMatch(eth_type=ether_types.ETH_TYPE_IP, ip_proto=IPPROTO_UDP, udp_src=53, direction=Direction.IN, imsi=encode_imsi(imsi))) matches.append( MagmaMatch(eth_type=ether_types.ETH_TYPE_IP, ip_proto=IPPROTO_UDP, udp_dst=53, direction=Direction.OUT, imsi=encode_imsi(imsi))) # Install TCP flows for DNS matches.append( MagmaMatch(eth_type=ether_types.ETH_TYPE_IP, ip_proto=IPPROTO_TCP, tcp_src=53, direction=Direction.IN, imsi=encode_imsi(imsi))) matches.append( MagmaMatch(eth_type=ether_types.ETH_TYPE_IP, ip_proto=IPPROTO_TCP, tcp_dst=53, direction=Direction.OUT, imsi=encode_imsi(imsi))) for match in matches: flows.add_resubmit_next_service_flow( datapath, self.main_tbl_num, match, actions, priority=priority, cookie=rule_num, hard_timeout=rule.hard_timeout, resubmit_table=self.next_table) def _save_redirect_entry(self, ip_addr, redirect_info): """ Saves the redirect entry in Redis. Throws: RedirectException: on error """ try: # Verify if ip_addr is in the correct format, and also # normalize the variants of the address into a single format ip_str = str(ipaddress.ip_address(ip_addr)) except ValueError as exp: raise RedirectException(exp) try: self._redirect_dict[ip_str] = redirect_info except RedisError as exp: raise RedirectException(exp) self.logger.info("Saved redirect rule for %s in Redis" % ip_str) def deactivate_flow_for_rule(self, datapath, imsi, rule_num): """ Deactivate a specific rule using the flow cookie for a subscriber """ cookie, mask = (rule_num, flows.OVS_COOKIE_MATCH_ALL) match = MagmaMatch(imsi=encode_imsi(imsi)) flows.delete_flow(datapath, self._scratch_tbl_num, match, cookie=cookie, cookie_mask=mask) def deactivate_flows_for_subscriber(self, datapath, imsi): """ Deactivate all rules for a subscriber """ flows.delete_flow(datapath, self._scratch_tbl_num, MagmaMatch(imsi=encode_imsi(imsi))) def _load_rule_actions(self, parser, rule_num, rule_version): return [ parser.NXActionRegLoad2(dst='reg2', value=rule_num), parser.NXActionRegLoad2(dst=RULE_VERSION_REG, value=rule_version), ]
def make_auctions_app(global_conf, redis_url='redis://localhost:7777/0', external_couch_url='http://localhost:5000/auction', internal_couch_url='http://localhost:9000/', proxy_internal_couch_url='http://localhost:9000/', auctions_db='auctions', hash_secret_key='', timezone='Europe/Kiev', preferred_url_scheme='http', debug=False, auto_build=False, event_source_connection_limit=1000): """ [app:main] use = egg:openprocurement.auction#auctions_server redis_url = redis://:passwod@localhost:1111/0 external_couch_url = http://localhost:1111/auction internal_couch_url = http://localhost:9011/ auctions_db = auction timezone = Europe/Kiev """ auctions_server.proxy_connection_pool = ConnectionPool(factory=Connection, max_size=20, backend="gevent") auctions_server.proxy_mappings = Memoizer({}) auctions_server.event_sources_pool = deque([]) auctions_server.config['PREFERRED_URL_SCHEME'] = preferred_url_scheme auctions_server.config['REDIS_URL'] = redis_url auctions_server.config['event_source_connection_limit'] = int( event_source_connection_limit) auctions_server.config['EXT_COUCH_DB'] = urljoin(external_couch_url, auctions_db) auctions_server.add_url_rule('/' + auctions_db + '/<path:path>', 'couch_server_proxy', couch_server_proxy, methods=['GET']) auctions_server.add_url_rule('/' + auctions_db + '/', 'couch_server_proxy', couch_server_proxy, methods=['GET'], defaults={'path': ''}) auctions_server.add_url_rule('/' + auctions_db + '_secured/<path:path>', 'auth_couch_server_proxy', auth_couch_server_proxy, methods=['GET']) auctions_server.add_url_rule('/' + auctions_db + '_secured/', 'auth_couch_server_proxy', auth_couch_server_proxy, methods=['GET'], defaults={'path': ''}) auctions_server.config['INT_COUCH_URL'] = internal_couch_url auctions_server.config['PROXY_COUCH_URL'] = proxy_internal_couch_url auctions_server.config['COUCH_DB'] = auctions_db auctions_server.config['TIMEZONE'] = tz(timezone) auctions_server.redis = Redis(auctions_server) auctions_server.couch_server = Server( auctions_server.config.get('INT_COUCH_URL'), session=Session(retry_delays=range(10))) if auctions_server.config['COUCH_DB'] not in auctions_server.couch_server: auctions_server.couch_server.create(auctions_server.config['COUCH_DB']) auctions_server.db = auctions_server.couch_server[ auctions_server.config['COUCH_DB']] auctions_server.config['HASH_SECRET_KEY'] = hash_secret_key sync_design(auctions_server.db) auctions_server.config['ASSETS_DEBUG'] = True if debug else False assets.auto_build = True if auto_build else False return auctions_server
def mock_auctions_server(request, mocker): params = getattr(request, 'param', {}) server_config_redis = params.get('server_config_redis', DEFAULT) connection_limit = params.get('connection_limit', DEFAULT) get_mapping = params.get('get_mapping', DEFAULT) proxy_path = params.get('proxy_path', DEFAULT) event_sources_pool = params.get('event_sources_pool', DEFAULT) proxy_connection_pool = params.get('proxy_connection_pool', DEFAULT) stream_proxy = params.get('stream_proxy', DEFAULT) db = params.get('db', DEFAULT) request_headers = params.get('request_headers', []) request_url = params.get('request_url', DEFAULT) redirect_url = params.get('redirect_url', DEFAULT) abort = params.get('abort', DEFAULT) class AuctionsServerAttributesContainer(object): logger = NotImplemented proxy_mappings = NotImplemented config = NotImplemented event_sources_pool = NotImplemented proxy_connection_pool = NotImplemented get_mapping = NotImplemented db = NotImplemented request_headers = NotImplemented class Request(object): headers = NotImplemented environ = NotImplemented url = NotImplemented class Config(object): __getitem__ = NotImplemented def config_getitem(item): if item == 'REDIS': return server_config_redis elif item == 'event_source_connection_limit': return connection_limit else: raise KeyError mock_path_info = MagicMock() def environ_setitem(item, value): if item == 'PATH_INFO': mock_path_info(value) return value else: raise KeyError mocker.patch.object(auctions_server_module, 'get_mapping', get_mapping) patch_pysse = mocker.patch.object(auctions_server_module, 'PySse', spec_set=PySse) patch_add_message = patch_pysse.return_value.add_message patch_request = mocker.patch.object(auctions_server_module, 'request', spec_set=Request) patch_request.environ.__setitem__.side_effect = environ_setitem patch_request.headers = request_headers patch_request.url = request_url patch_redirect = mocker.patch.object(auctions_server_module, 'redirect', return_value=redirect_url) patch_abort = mocker.patch.object(auctions_server_module, 'abort', return_value=abort) patch_StreamProxy = \ mocker.patch.object(auctions_server_module, 'StreamProxy', return_value=stream_proxy) auctions_server = NonCallableMock( spec_set=AuctionsServerAttributesContainer) logger = MagicMock(spec_set=frontend.logger) proxy_mappings = MagicMock(spec_set=Memoizer({})) proxy_mappings.get.return_value = proxy_path config = MagicMock(spec_set=Config) config.__getitem__.side_effect = config_getitem auctions_server.logger = logger auctions_server.proxy_mappings = proxy_mappings auctions_server.config = config auctions_server.event_sources_pool = event_sources_pool auctions_server.proxy_connection_pool = proxy_connection_pool auctions_server.db = db mocker.patch.object(auctions_server_module, 'auctions_server', auctions_server) return { 'server': auctions_server, 'proxy_mappings': proxy_mappings, 'mock_path_info': mock_path_info, 'patch_StreamProxy': patch_StreamProxy, 'patch_redirect': patch_redirect, 'patch_abort': patch_abort, 'patch_PySse': patch_pysse, 'patch_add_message': patch_add_message }
class ep_risk: #example folio_dict: {’JNJ’:10**6, ’AMGN’:10**6, ’UNH’:10**6, ’MYL’:10**6, ’A’: 10**6, ’XLV’: -(5*10**6)} def __init__(self, folio_dict): self.port = folio_dict memo = Memoizer({}) #caches the data pulls for up to 60 seconds @memo(max_age=60) def get_data(self, end_date="01/15/15", window_length=700): last_date = datetime.datetime.strptime(end_date, "%x") first_date = last_date - datetime.timedelta(days=window_length) s = first_date.__str__() l = last_date.__str__() #Create List of Dataframes for all stocks df_s = [self.prices(tick, s, l) for tick in self.port.keys()] #Merge that List of Dataframes df_final = df_s[0] for i in range(1, len(df_s)): df_final = DataFrame.join(df_final, df_s[i]) return df_final def prices(self, ticker, start_string, end_string): #holds key:Date Val:Adj Close data_hash = dict() #Create dictionary from yahoo data tick_test = ystockquote.get_historical_prices(ticker, start_string, end_string) #fill data_hash for date in tick_test.keys(): data_hash[datetime.date(int(date[:4]), int(date[5:7]), int( date[8:]))] = float(tick_test[date]['Adj Close']) #make datahash into dataframe called df df = DataFrame.from_dict(data_hash, orient='index', dtype=float) #rename first column df.columns = [ticker] #sort in date order df = df.sort(ascending=True, inplace=False) return df def data_stats(self, n_days=2, end_date="01/15/15", window_length=700): price_data = self.get_data(end_date=end_date, window_length=window_length) #calculate n-day returns returns = price_data.pct_change(periods=n_days, fill_method='pad', limit=None, freq=None) #apply log transformatinon returns = 1 + returns returns = returns.applymap(m.log) #calculate means and volatilities means = returns.mean(axis=0) vols = returns.std(axis=0) #calc correlations correlation_matrix = returns.corr() #returns lambda_plus for PCA approaches lambda_plus = (1 + m.sqrt( float(len(self.port.keys())) / max([ returns.ix[:, i].count() for i in range(0, len(self.port.keys())) ])))**2 return correlation_matrix, means, vols, lambda_plus def pca_risk(self, alpha, n_days=2, end_date="01/15/15", window_length=700): correlation_matrix, means, vols, lambda_plus = self.data_stats( n_days=n_days, end_date=end_date, window_length=window_length) #return eigenvectors and values (eigenvectors are columns) e_vals, e_vecs = np.linalg.eigh(correlation_matrix) #sort (largest to smallest) indices = e_vals.argsort()[::-1] sorted_evals = [e_vals[i] for i in indices] sorted_evecs = e_vecs[:, indices] significant_num = len( [value for value in sorted_evals if value > lambda_plus]) cleaned = self.clean_correlation_matrix(significant_num, sorted_evals, sorted_evecs) cleaned_df = DataFrame(data=cleaned, index=correlation_matrix.columns.tolist(), columns=correlation_matrix.columns.tolist(), dtype=float, copy=True) var, es = self.mc_risk(alpha, cleaned_df, means, vols) return var, es def unclean_risk(self, alpha, n_days=2, end_date="01/15/15", window_length=700): correlation_matrix, means, vols, lambda_plus = self.data_stats( n_days=n_days, end_date=end_date, window_length=window_length) var, es = self.mc_risk(alpha, correlation_matrix, means, vols) return var, es def historical_risk(self, alpha, n_days=2, end_date="01/15/15", window_length=700): price_data = self.get_data(end_date=end_date, window_length=window_length) returns = price_data.pct_change(periods=n_days, fill_method='pad', limit=None, freq=None) return self.port_rets(returns, alpha) def mc_risk(self, alpha, corr_df, means, vols): #generate 50,000 samples from T dist joint_scenarios = self.multivariatet_sampler( np.zeros(len(self.port.keys())), corr_df, 4, 50000) #scale these scenarios by multiplying by scaling vol then shifting by mean joint_scenarios = m.sqrt(1 / 2.) * np.dot( joint_scenarios, np.diag(vols)) + np.tile(means, (50000, 1)) #calculate portfolio returns var_alpha, es_alpha = self.port_rets(joint_scenarios, alpha) return var_alpha, es_alpha def port_rets(self, scenarios, alpha): port_vec = np.array(self.port.values()) sim_returns = np.dot(scenarios, port_vec) var_alpha = np.percentile(sim_returns, alpha) es_alpha = np.mean([v for v in sim_returns.tolist() if v < var_alpha]) return var_alpha, es_alpha def clean_correlation_matrix(self, significant_num, lambdas, vecs): dimension = len(lambdas) matrix = np.zeros((dimension, dimension)) for i in range(0, dimension): for j in range(0, dimension): ro = 0 for k in range(0, significant_num): ro += lambdas[k] * vecs[:, k][i] * vecs[:, k][j] if (i == j): eii_sq = 1 for k in range(0, significant_num): eii_sq = eii_sq - lambdas[k] * vecs[:, k][i] * vecs[:, k][i] ro = ro + eii_sq matrix[i, j] = ro return matrix #Known Function to generate samples from multivariate t in python (Kenny Chowdhary - PhD) def multivariatet_sampler(self, mu, Sigma, N, M): ''' Output: Produce M samples of d-dimensional multivariate t distribution Input: mu = mean (d dimensional numpy array or scalar) Sigma = scale matrix (dxd numpy array) N = degrees of freedom M = # of samples to produce ''' d = len(Sigma) g = np.tile(np.random.gamma(N / 2., 2. / N, M), (d, 1)).T Z = np.random.multivariate_normal(np.zeros(d), Sigma, M) return mu + Z / np.sqrt(g)
class RedirectionManager: """ RedirectionManager The redirection manager handles subscribers who have redirection enabled, it adds the flows into ovs for redirecting user to the redirection server """ DNS_TIMEOUT_SECS = 15 REDIRECT_NOT_PROCESSED = REG_ZERO_VAL REDIRECT_PROCESSED = 0x1 RedirectRequest = namedtuple( 'RedirectRequest', ['imsi', 'ip_addr', 'rule', 'rule_num', 'priority'], ) def __init__(self, bridge_ip, logger, main_tbl_num, next_table, scratch_table_num, session_rule_version_mapper): self._bridge_ip = bridge_ip self.logger = logger self.main_tbl_num = main_tbl_num self.next_table = next_table self._scratch_tbl_num = scratch_table_num self._redirect_dict = RedirectDict() self._dns_cache = Memoizer({}) self._redirect_port = get_service_config_value( 'redirectd', 'http_port', 8080) self._session_rule_version_mapper = session_rule_version_mapper def handle_redirection(self, datapath, loop, redirect_request): """ Depending on redirection server address type install redirection rules """ imsi = redirect_request.imsi ip_addr = redirect_request.ip_addr rule = redirect_request.rule rule_num = redirect_request.rule_num priority = redirect_request.priority # TODO IMPORTANT check that redirectd service is running, as its a # dynamic service its not on by default. Will save you some sanity :) # TODO figure out what to do with SIP_URI if rule.redirect.address_type == rule.redirect.SIP_URI: raise RedirectException("SIP_URIs redirection isn't setup") if rule.redirect.address_type == rule.redirect.IPv6: raise RedirectException("No ipv6 support, so no ipv6 redirect") self._save_redirect_entry(ip_addr, rule.redirect) self._install_redirect_flows(datapath, loop, imsi, rule, rule_num, priority) return def _install_redirect_flows(self, datapath, loop, imsi, rule, rule_num, priority): """ Add flows to forward traffic to the redirection server. 1) Intercept tcp traffic to the web to the redirection server, which completes the tcp handshake. This is done by adding an OVS flow with a learn action (flow catches inbound tcp packets, while learn action creates another flow that sends packets back from server) 2) Add flows to allow UDP traffic so DNS queries can go through. Finally add flows with a higher priority that allow traffic to and from the address provided in redirect rule. """ if rule.redirect.address_type == rule.redirect.URL: self._install_url_bypass_flows(datapath, loop, imsi, rule, rule_num, priority) elif rule.redirect.address_type == rule.redirect.IPv4: self._install_ipv4_bypass_flows(datapath, imsi, rule, rule_num, priority, [rule.redirect.server_address]) self._install_dns_flows(datapath, imsi, rule, rule_num, priority) self._install_server_flows(datapath, imsi, rule, rule_num, priority) def _install_scratch_table_flows(self, datapath, imsi, rule, rule_num, priority): """ The flow action for subscribers that need to be redirected does 2 things * Forward requests from subscriber to the internal http server * Instantiate a flow that matches response packets from the server and sends them back to subscriber Match: incoming tcp traffic with port 80, direction out Action: 1) Set reg2 to rule_num 2) Set ip dst to server ip 3) Output to table 20 4) Apply LearnAction: LearnAction(adds new flow for every pkt flow that hits this rule) 1) Match ip packets 2) Match tcp protocol 3) Match packets from LOCAL port 4) Match ip src = server ip 5) Match ip dst = current flow ip src 6) Match tcp src = current flow tcp dst 7) Match tcp dst = current flow tcp src 8) Load ip src = current flow ip dst 9) Output through gtp0 """ parser = datapath.ofproto_parser match_http = MagmaMatch( eth_type=ether_types.ETH_TYPE_IP, ip_proto=IPPROTO_TCP, tcp_dst=80, imsi=encode_imsi(imsi), direction=Direction.OUT) of_note = parser.NXActionNote(list(rule.id.encode())) actions = [ parser.NXActionLearn( table_id=self.main_tbl_num, priority=priority, cookie=rule_num, specs=[ parser.NXFlowSpecMatch( src=ether_types.ETH_TYPE_IP, dst=('eth_type_nxm', 0), n_bits=16 ), parser.NXFlowSpecMatch( src=IPPROTO_TCP, dst=('ip_proto_nxm', 0), n_bits=8 ), parser.NXFlowSpecMatch( src=Direction.IN, dst=(DIRECTION_REG, 0), n_bits=32 ), parser.NXFlowSpecMatch( src=int(ipaddress.IPv4Address(self._bridge_ip)), dst=('ipv4_src_nxm', 0), n_bits=32 ), parser.NXFlowSpecMatch( src=('ipv4_src_nxm', 0), dst=('ipv4_dst_nxm', 0), n_bits=32 ), parser.NXFlowSpecMatch( src=('tcp_src_nxm', 0), dst=('tcp_dst_nxm', 0), n_bits=16 ), parser.NXFlowSpecMatch( src=self._redirect_port, dst=('tcp_src_nxm', 0), n_bits=16 ), parser.NXFlowSpecMatch( src=encode_imsi(imsi), dst=(IMSI_REG, 0), n_bits=64 ), parser.NXFlowSpecLoad( src=('ipv4_dst_nxm', 0), dst=('ipv4_src_nxm', 0), n_bits=32 ), parser.NXFlowSpecLoad( src=80, dst=('tcp_src_nxm', 0), n_bits=16 ), # Learn doesn't support resubmit to table, so send directly parser.NXFlowSpecOutput( src=('in_port', 0), dst="", n_bits=16 ), ] ), parser.NXActionRegLoad2(dst=SCRATCH_REGS[0], value=self.REDIRECT_PROCESSED), parser.OFPActionSetField(ipv4_dst=self._bridge_ip), parser.OFPActionSetField(tcp_dst=self._redirect_port), of_note, ] actions += self._load_rule_actions(parser, rule_num, imsi, rule.id) flows.add_resubmit_current_service_flow( datapath, self._scratch_tbl_num, match_http, actions, priority=priority, cookie=rule_num, hard_timeout=rule.hard_timeout, resubmit_table=self.main_tbl_num) match = MagmaMatch(imsi=encode_imsi(imsi)) action = [] flows.add_drop_flow(datapath, self._scratch_tbl_num, match, action, priority=flows.MINIMUM_PRIORITY + 1, cookie=rule_num) def _install_not_processed_flows(self, datapath, imsi, rule, rule_num, priority): """ Redirect all traffic to the scratch table to only allow redirected http traffic to go through, the rest will be dropped. reg0 is used as a boolean to know whether the drop rule was processed. """ parser = datapath.ofproto_parser of_note = parser.NXActionNote(list(rule.id.encode())) match = MagmaMatch(imsi=encode_imsi(imsi), direction=Direction.OUT, reg0=self.REDIRECT_NOT_PROCESSED, eth_type=ether_types.ETH_TYPE_IP) action = [of_note] flows.add_resubmit_current_service_flow( datapath, self.main_tbl_num, match, action, priority=priority, cookie=rule_num, hard_timeout=rule.hard_timeout, resubmit_table=self._scratch_tbl_num) match = MagmaMatch(imsi=encode_imsi(imsi), direction=Direction.OUT, reg0=self.REDIRECT_PROCESSED) action = [of_note] flows.add_resubmit_next_service_flow( datapath, self.main_tbl_num, match, action, priority=priority, cookie=rule_num, hard_timeout=rule.hard_timeout, resubmit_table=self.next_table) def _install_server_flows(self, datapath, imsi, rule, rule_num, priority): """ Install the redirect flows to redirect all HTTP traffic to the captive portal and to drop all of the rest. """ self._install_scratch_table_flows(datapath, imsi, rule, rule_num, priority) self._install_not_processed_flows(datapath, imsi, rule, rule_num, priority) def _install_url_bypass_flows(self, datapath, loop, imsi, rule, rule_num, priority): """ Resolve DNS queries to get the ip address of redirect url, this is done to allow traffic to safely pass through as we want subscribers to have full access to the url they are redirected to. First check cache for redirect url, if not in cache submit a DNS query """ redirect_addr_host = urlsplit(rule.redirect.server_address).netloc cached_ips = self._dns_cache.get(redirect_addr_host) if cached_ips is not None: self.logger.debug( "DNS cache hit for {}, entry expires in {} sec".format( redirect_addr_host, self._dns_cache.ttl(redirect_addr_host) ) ) self._install_ipv4_bypass_flows(datapath, imsi, rule, rule_num, priority, cached_ips) return resolver = aiodns.DNSResolver(timeout=self.DNS_TIMEOUT_SECS, loop=loop) query = resolver.query(redirect_addr_host, 'A') def add_flows(dns_resolve_future): """ Callback for when DNS query is resolved, adds the bypass flows """ try: ips = [entry.host for entry in dns_resolve_future.result()] ttl = min(entry.ttl for entry in dns_resolve_future.result()) except aiodns.error.DNSError as err: self.logger.error("Error: ip lookup for {}: {}".format( redirect_addr_host, err)) return self._dns_cache.get(redirect_addr_host, lambda: ips, max_age=ttl) self._install_ipv4_bypass_flows(datapath, imsi, rule, rule_num, priority, ips) asyncio.ensure_future(query, loop=loop).add_done_callback(add_flows) def _install_ipv4_bypass_flows(self, datapath, imsi, rule, rule_num, priority, ips): """ Installs flows for traffic that is allowed to pass through for subscriber who has redirection enabled. Allow access to all passed ips. Allow UDP traffic(for DNS queries), traffic to/from redirection address """ parser = datapath.ofproto_parser of_note = parser.NXActionNote(list(rule.id.encode())) actions = [ of_note, ] actions += self._load_rule_actions(parser, rule_num, imsi, rule.id) matches = [] for ip in ips: matches.append(MagmaMatch( eth_type=ether_types.ETH_TYPE_IP, direction=Direction.OUT, ipv4_dst=ip, imsi=encode_imsi(imsi) )) matches.append(MagmaMatch( eth_type=ether_types.ETH_TYPE_IP, direction=Direction.IN, ipv4_src=ip, imsi=encode_imsi(imsi) )) for match in matches: flows.add_resubmit_next_service_flow( datapath, self.main_tbl_num, match, actions, priority=priority + 1, cookie=rule_num, hard_timeout=rule.hard_timeout, resubmit_table=self.next_table) def _install_dns_flows(self, datapath, imsi, rule, rule_num, priority): """ Installs flows that allow DNS queries to path through. """ parser = datapath.ofproto_parser of_note = parser.NXActionNote(list(rule.id.encode())) actions = [ of_note, ] actions += self._load_rule_actions(parser, rule_num, imsi, rule.id) matches = [] # Install UDP flows for DNS matches.append(MagmaMatch(eth_type=ether_types.ETH_TYPE_IP, ip_proto=IPPROTO_UDP, udp_src=53, direction=Direction.IN, imsi=encode_imsi(imsi))) matches.append(MagmaMatch(eth_type=ether_types.ETH_TYPE_IP, ip_proto=IPPROTO_UDP, udp_dst=53, direction=Direction.OUT, imsi=encode_imsi(imsi))) # Install TCP flows for DNS matches.append(MagmaMatch(eth_type=ether_types.ETH_TYPE_IP, ip_proto=IPPROTO_TCP, tcp_src=53, direction=Direction.IN, imsi=encode_imsi(imsi))) matches.append(MagmaMatch(eth_type=ether_types.ETH_TYPE_IP, ip_proto=IPPROTO_TCP, tcp_dst=53, direction=Direction.OUT, imsi=encode_imsi(imsi))) for match in matches: flows.add_resubmit_next_service_flow( datapath, self.main_tbl_num, match, actions, priority=priority, cookie=rule_num, hard_timeout=rule.hard_timeout, resubmit_table=self.next_table) def _save_redirect_entry(self, ip_addr, redirect_info): """ Saves the redirect entry in Redis. Throws: RedirectException: on error """ try: # Verify if ip_addr is in the correct format, and also # normalize the variants of the address into a single format ip_str = str(ipaddress.ip_address(ip_addr)) except ValueError as exp: raise RedirectException(exp) try: self._redirect_dict[ip_str] = redirect_info except RedisError as exp: raise RedirectException(exp) self.logger.info("Saved redirect rule for %s in Redis" % ip_str) def deactivate_flow_for_rule(self, datapath, imsi, rule_num): """ Deactivate a specific rule using the flow cookie for a subscriber """ cookie, mask = (rule_num, flows.OVS_COOKIE_MATCH_ALL) match = MagmaMatch(imsi=encode_imsi(imsi)) flows.delete_flow(datapath, self._scratch_tbl_num, match, cookie=cookie, cookie_mask=mask) def deactivate_flows_for_subscriber(self, datapath, imsi): """ Deactivate all rules for a subscriber """ flows.delete_flow(datapath, self._scratch_tbl_num, MagmaMatch(imsi=encode_imsi(imsi))) def _load_rule_actions(self, parser, rule_num, imsi, rule_id): version = self._session_rule_version_mapper.get_version(imsi, rule_id) return [ parser.NXActionRegLoad2(dst='reg2', value=rule_num), parser.NXActionRegLoad2(dst=RULE_VERSION_REG, value=version), ]
from memoize import Memoizer memoized = Memoizer() @memoized def fib(n): if n <= 1: return n return fib(n - 1) + fib(n - 2) @memoized def choose(n, k): if k < 0 or k > n: return 0 if k == 0 or k == n: return 1 return choose(n - 1, k - 1) + choose(n - 1, k)
class MemoizeTestCase(SimpleTestCase): def setUp(self): self.memoizer = Memoizer() def test_00_set(self): self.memoizer.set('hi', 'hello') assert self.memoizer.get('hi') == 'hello' def test_01_add(self): self.memoizer.add('hi', 'hello') assert self.memoizer.get('hi') == 'hello' self.memoizer.add('hi', 'foobar') assert self.memoizer.get('hi') == 'hello' def test_02_delete(self): self.memoizer.set('hi', 'hello') self.memoizer.delete('hi') assert self.memoizer.get('hi') is self.memoizer.default_cache_value def test_03_delete_many(self): self.memoizer.set('hi', 'hello') self.memoizer.set('bye', 'goodbye') self.memoizer.delete_many('hi', 'bye') assert self.memoizer.get('hi') is self.memoizer.default_cache_value assert self.memoizer.get('bye') is self.memoizer.default_cache_value def test_04_clear(self): self.memoizer.set('hi', 'hello') self.memoizer.set('bye', 'goodbye') self.memoizer.clear() assert self.memoizer.get('hi') is self.memoizer.default_cache_value assert self.memoizer.get('bye') is self.memoizer.default_cache_value def test_06_memoize(self): @self.memoizer.memoize(5) def big_foo(a, b, func=lambda x: x): func(a) return a + b test_func = MagicMock() big_foo(5, 2, func=test_func) # Cached call big_foo(5, 2, func=test_func) # We are expecting only 1 call here as second one should be cached assert test_func.call_count == 1, "Count: {}".format( test_func.call_count) # different arguments so we should have additional call big_foo(5, 3, func=test_func) assert test_func.call_count == 2, "Count: {}".format( test_func.call_count) # Expired now = datetime.datetime.utcfromtimestamp(time.time()) with freeze_time(now) as frozen_datetime: frozen_datetime.tick(delta=datetime.timedelta(seconds=6)) # Cache should be expired so we have increasing amount of calls big_foo(5, 2, func=test_func) assert test_func.call_count == 3, "Count: {}".format( test_func.call_count) big_foo(5, 3, func=test_func) assert test_func.call_count == 4, "Count: {}".format( test_func.call_count) def test_06a_memoize(self): @self.memoizer.memoize(50) def big_foo(a, b): return a + b + random.randrange(0, 100000) result = big_foo(5, 2) assert big_foo(5, 2) == result def test_07_delete_memoize(self): @self.memoizer.memoize(5) def big_foo(a, b): return a + b + random.randrange(0, 100000) result = big_foo(5, 2) result2 = big_foo(5, 3) assert big_foo(5, 2) == result assert big_foo(5, 2) == result assert big_foo(5, 3) != result assert big_foo(5, 3) == result2 self.memoizer.delete_memoized(big_foo) assert big_foo(5, 2) != result assert big_foo(5, 3) != result2 def test_07b_delete_memoized_verhash(self): @self.memoizer.memoize(5) def big_foo(a, b): return a + b + random.randrange(0, 100000) result = big_foo(5, 2) result2 = big_foo(5, 3) assert big_foo(5, 2) == result assert big_foo(5, 2) == result assert big_foo(5, 3) != result assert big_foo(5, 3) == result2 self.memoizer.delete_memoized_verhash(big_foo) _fname, _fname_instance = function_namespace(big_foo) version_key = self.memoizer._memvname(_fname) assert (self.memoizer.get(version_key) is self.memoizer.default_cache_value) assert big_foo(5, 2) != result assert big_foo(5, 3) != result2 assert (self.memoizer.get(version_key) is not self.memoizer.default_cache_value) def test_08_delete_memoize(self): @self.memoizer.memoize() def big_foo(a, b): return a + b + random.randrange(0, 100000) result_a = big_foo(5, 1) result_b = big_foo(5, 2) assert big_foo(5, 1) == result_a assert big_foo(5, 2) == result_b self.memoizer.delete_memoized(big_foo, 5, 2) assert big_foo(5, 1) == result_a assert big_foo(5, 2) != result_b # Cleanup bigfoo 5,1 5,2 or it might conflict with # following run if it also uses memcache self.memoizer.delete_memoized(big_foo, 5, 2) self.memoizer.delete_memoized(big_foo, 5, 1) def test_09_args_memoize(self): @self.memoizer.memoize() def big_foo(a, b): return sum(a) + sum(b) + random.randrange(0, 100000) result_a = big_foo([5, 3, 2], [1]) result_b = big_foo([3, 3], [3, 1]) assert big_foo([5, 3, 2], [1]) == result_a assert big_foo([3, 3], [3, 1]) == result_b self.memoizer.delete_memoized(big_foo, [5, 3, 2], [1]) assert big_foo([5, 3, 2], [1]) != result_a assert big_foo([3, 3], [3, 1]) == result_b # Cleanup bigfoo 5,1 5,2 or it might conflict with # following run if it also uses memcache self.memoizer.delete_memoized(big_foo, [5, 3, 2], [1]) self.memoizer.delete_memoized(big_foo, [3, 3], [1]) def test_10_kwargs_memoize(self): @self.memoizer.memoize() def big_foo(a, b=None): return a + sum(b.values()) + random.randrange(0, 100000) result_a = big_foo(1, dict(one=1, two=2)) result_b = big_foo(5, dict(three=3, four=4)) assert big_foo(1, dict(one=1, two=2)) == result_a assert big_foo(5, dict(three=3, four=4)) == result_b self.memoizer.delete_memoized(big_foo, 1, dict(one=1, two=2)) assert big_foo(1, dict(one=1, two=2)) != result_a assert big_foo(5, dict(three=3, four=4)) == result_b def test_10a_kwargonly_memoize(self): @self.memoizer.memoize() def big_foo(a=None): if a is None: a = 0 return a + random.random() result_a = big_foo() result_b = big_foo(5) assert big_foo() == result_a assert big_foo() < 1 assert big_foo(5) == result_b assert big_foo(5) >= 5 and big_foo(5) < 6 def test_10a_arg_kwarg_memoize(self): @self.memoizer.memoize() def f(a, b, c=1): return a + b + c + random.randrange(0, 100000) assert f(1, 2) == f(1, 2, c=1) assert f(1, 2) == f(1, 2, 1) assert f(1, 2) == f(1, 2) assert f(1, 2) != f(2, 1) assert f(1, 2, 3) != f(1, 2) with self.assertRaises(TypeError): f(1) def test_10ab_arg_kwarg_memoize(self): @self.memoizer.memoize() def f(a, **kwargs): return (a, kwargs, random.randrange(0, 100000000)) f1 = f(1, b=2) assert f1 == f(1, b=2) assert f1 != f(1, b=4) f2 = f(5, c=6) assert f2 == f(5, c=6) assert f2 != f(7, c=6) self.memoizer.delete_memoized(f, 1, b=2) assert f1 != f(1, b=2) assert f2 == f(5, c=6) assert f2 != f(7, c=6) def test_10ac_arg_kwarg_memoize(self): @self.memoizer.memoize() def f(a, *args, **kwargs): return (a, args, kwargs, random.randrange(0, 100000000)) f1 = f(1, 2, b=3) assert f1 == f(1, 2, b=3) assert f1 != f(1, 3, b=3) f2 = f(5, 6, c=7) assert f2 == f(5, 6, c=7) assert f2 != f(7, 6, c=7) self.memoizer.delete_memoized(f, 1, 2, b=3) assert f1 != f(1, 2, b=3) assert f2 == f(5, 6, c=7) assert f2 != f(7, 6, c=7) @self.memoizer.memoize() def f(a, b=1, *args, **kwargs): return (a, b, args, kwargs, random.randrange(0, 100000000)) f3 = f(1, 3, 4) assert f3 == f(1, 3, 4) assert f3 == f(*(1, 3, 4)) assert f3 != f(1, 3) self.memoizer.delete_memoized(f, 1, 3, 4) assert f3 != f(1, 3, 4) def test_10b_classarg_memoize(self): @self.memoizer.memoize() def bar(a, *args): return a.value + random.random() + sum(args) class Adder(object): def __init__(self, value, *args): self.value = (value + sum(args)) adder = Adder(15) adder2 = Adder(20) adder3 = Adder(16, 5) adder4 = Adder(21, 6) w = bar(adder) x = bar(adder2) y = bar(adder3) z = bar(adder4) assert w != x assert y != z assert bar(adder) == w assert bar(adder) != x assert bar(adder3) == y assert bar(adder3) != z adder.value = 14 adder3.value = 15 assert bar(adder) == w assert bar(adder) != x assert bar(adder3) == y assert bar(adder3) != z assert bar(adder) != bar(adder2) assert bar(adder3) != bar(adder4) assert bar(adder2) == x assert bar(adder4) == z def test_10c_classfunc_memoize(self): class Adder(object): def __init__(self, initial): self.initial = initial @self.memoizer.memoize() def add(self, b, *args): return self.initial + b + sum(args) adder1 = Adder(1) adder2 = Adder(2) x = adder1.add(3) y = adder1.add(3, 4) assert adder1.add(3) == x assert adder1.add(4) != x assert adder1.add(3, 4) == y assert adder1.add(4, 4) != y assert adder1.add(3) != adder2.add(3) def test_10d_classfunc_memoize_delete(self): class Adder(object): def __init__(self, initial): self.initial = initial @self.memoizer.memoize() def add(self, b, *args): return self.initial + b + sum(args) + random.random() adder1 = Adder(1) adder2 = Adder(2) a1 = adder1.add(3) a2 = adder2.add(3) b1 = adder1.add(3, 1) b2 = adder2.add(3, 1) assert a1 != a2 assert a1 != b1 assert a2 != b2 assert adder1.add(3) == a1 assert adder2.add(3) == a2 assert adder1.add(3, 1) == b1 assert adder2.add(3, 1) == b2 self.memoizer.delete_memoized(adder1.add) a3 = adder1.add(3) a4 = adder2.add(3) b3 = adder1.add(3, 1) b4 = adder2.add(3, 1) self.assertNotEqual(a1, a3) self.assertNotEqual(b1, b3) assert a1 != a3 assert b1 != b3 self.assertEqual(a2, a4) self.assertEqual(b2, b4) self.memoizer.delete_memoized(Adder.add) a5 = adder1.add(3) a6 = adder2.add(3) b5 = adder1.add(3, 1) b6 = adder2.add(3, 1) self.assertNotEqual(a5, a6) self.assertNotEqual(b5, b6) self.assertNotEqual(a3, a5) self.assertNotEqual(b3, b5) self.assertNotEqual(a4, a6) self.assertNotEqual(b4, b6) def test_10e_delete_memoize_classmethod(self): class Mock(object): @classmethod @self.memoizer.memoize(5) def big_foo(cls, a, b, *args): return a + b + sum(args) + random.randrange(0, 100000) result = Mock.big_foo(5, 2) result2 = Mock.big_foo(5, 3) result3 = Mock.big_foo(5, 2, 1) result4 = Mock.big_foo(5, 3, 1) assert Mock.big_foo(5, 2) == result assert Mock.big_foo(5, 2) == result assert Mock.big_foo(5, 3) != result assert Mock.big_foo(5, 3) == result2 assert Mock.big_foo(5, 2, 1) == result3 assert Mock.big_foo(5, 2, 1) == result3 assert Mock.big_foo(5, 3, 1) != result3 assert Mock.big_foo(5, 3, 1) == result4 self.memoizer.delete_memoized(Mock.big_foo) assert Mock.big_foo(5, 2) != result assert Mock.big_foo(5, 3) != result2 assert Mock.big_foo(5, 2, 1) != result3 assert Mock.big_foo(5, 3, 1) != result4 def test_14_memoized_multiple_arg_kwarg_calls(self): @self.memoizer.memoize() def big_foo(a, b, c=[1, 1], d=[1, 1]): return (sum(a) + sum(b) + sum(c) + sum(d) + random.randrange(0, 100000)) result_a = big_foo([5, 3, 2], [1], c=[3, 3], d=[3, 3]) assert big_foo([5, 3, 2], [1], d=[3, 3], c=[3, 3]) == result_a assert big_foo(b=[1], a=[5, 3, 2], c=[3, 3], d=[3, 3]) == result_a assert big_foo([5, 3, 2], [1], [3, 3], [3, 3]) == result_a def test_15_memoize_multiple_arg_kwarg_delete(self): @self.memoizer.memoize() def big_foo(a, b, c=[1, 1], d=[1, 1]): return (sum(a) + sum(b) + sum(c) + sum(d) + random.randrange(0, 100000)) result_a = big_foo([5, 3, 2], [1], c=[3, 3], d=[3, 3]) self.memoizer.delete_memoized(big_foo, [5, 3, 2], [1], [3, 3], [3, 3]) result_b = big_foo([5, 3, 2], [1], c=[3, 3], d=[3, 3]) assert result_a != result_b self.memoizer.delete_memoized(big_foo, [5, 3, 2], b=[1], c=[3, 3], d=[3, 3]) result_b = big_foo([5, 3, 2], [1], c=[3, 3], d=[3, 3]) assert result_a != result_b self.memoizer.delete_memoized(big_foo, [5, 3, 2], [1], c=[3, 3], d=[3, 3]) result_a = big_foo([5, 3, 2], [1], c=[3, 3], d=[3, 3]) assert result_a != result_b self.memoizer.delete_memoized(big_foo, [5, 3, 2], b=[1], c=[3, 3], d=[3, 3]) result_a = big_foo([5, 3, 2], [1], c=[3, 3], d=[3, 3]) assert result_a != result_b self.memoizer.delete_memoized(big_foo, [5, 3, 2], [1], c=[3, 3], d=[3, 3]) result_b = big_foo([5, 3, 2], [1], c=[3, 3], d=[3, 3]) assert result_a != result_b self.memoizer.delete_memoized(big_foo, [5, 3, 2], [1], [3, 3], [3, 3]) result_a = big_foo([5, 3, 2], [1], c=[3, 3], d=[3, 3]) assert result_a != result_b def test_16_memoize_kwargs_to_args(self): def big_foo(a, b, c=None, d=None): return sum(a) + sum(b) + random.randrange(0, 100000) expected = (1, 2, 'foo', 'bar') args, kwargs = self.memoizer._memoize_kwargs_to_args( big_foo, 1, 2, 'foo', 'bar') assert (args == expected) args, kwargs = self.memoizer._memoize_kwargs_to_args(big_foo, 2, 'foo', 'bar', a=1) assert (args == expected) args, kwargs = self.memoizer._memoize_kwargs_to_args(big_foo, a=1, b=2, c='foo', d='bar') assert (args == expected) args, kwargs = self.memoizer._memoize_kwargs_to_args(big_foo, d='bar', b=2, a=1, c='foo') assert (args == expected) args, kwargs = self.memoizer._memoize_kwargs_to_args(big_foo, 1, 2, d='bar', c='foo') assert (args == expected) def test_17_memoize_none_value(self): self.memoizer = Memoizer() @self.memoizer.memoize() def foo(): return None cache_key = foo.make_cache_key(foo.uncached) assert (self.memoizer.get(cache_key) is self.memoizer.default_cache_value) result = foo() assert result is None assert self.memoizer.get(cache_key) is None self.memoizer.delete_memoized(foo) cache_key = foo.make_cache_key(foo.uncached) assert (self.memoizer.get(cache_key) is self.memoizer.default_cache_value) def test_17_delete_memoized_instancemethod_with_mutable_param(self): class Foo(object): def __init__(self, id): self.id = id @self.memoizer.memoize(5) def foo(self, bar_obj): return random.randrange(0, 100000) + bar_obj.id def __repr__(self): return ('{}({})'.format(self.__class__.__name__, self.id)) class Bar(object): def __init__(self, id): self.id = id def __repr__(self): return ('{}({})'.format(self.__class__.__name__, self.id)) a = Foo(1) b = Bar(1) c = Bar(2) result1 = a.foo(b) result2 = a.foo(c) assert (a.foo(b) == result1) assert (a.foo(c) == result2) self.memoizer.delete_memoized(a.foo, a, b) assert (a.foo(b) != result1) assert (a.foo(c) == result2) def test_18_python3_uses_getfullargspec(self): def foo(): return None major_version = sys.version_info.major argspec = _get_argspec(foo) if major_version < 3: assert argspec.__class__.__name__ is 'ArgSpec' else: assert argspec.__class__.__name__ is 'FullArgSpec' def test_19_memoize_make_name(self): @self.memoizer.memoize() def f(): return None without_make_name = f.make_cache_key(f.uncached) @self.memoizer.memoize(make_name=lambda fname: 'make_name' + fname) def f(): return None with_make_name = f.make_cache_key(f.uncached) assert without_make_name != with_make_name def test_20_memoize_unless(self): @self.memoizer.memoize(unless=lambda: True) def f(): return random.randrange(0, 100000) old_value = f() new_value = f() assert new_value != old_value @patch('memoize.Memoizer.get', side_effect=Exception) def test_21_memoize_get_backend_error(self, memoizer_get): logging.basicConfig() # disable logger (to avoid cluttering test output) logging.disable(logging.ERROR) memoizer = Memoizer() @memoizer.memoize() def f(): return random.randrange(0, 100000) cache_key = f.make_cache_key(f.uncached) old_value = f() memoizer_get.assert_called_with(cache_key) memoizer_get.reset_mock() new_value = f() memoizer_get.assert_called_with(cache_key) assert new_value != old_value # re-enable logger logging.disable(logging.NOTSET) @patch('memoize.Memoizer.set', side_effect=Exception) def test_22_memoize_set_backend_error(self, memoizer_set): logging.basicConfig() # disable logger (to avoid cluttering test output) logging.disable(logging.ERROR) memoizer = Memoizer() @memoizer.memoize() def f(): return random.randrange(0, 100000) cache_key = f.make_cache_key(f.uncached) old_value = f() memoizer_set.assert_called_with(cache_key, old_value, timeout=f.cache_timeout) memoizer_set.reset_mock() new_value = f() memoizer_set.assert_called_with(cache_key, new_value, timeout=f.cache_timeout) assert new_value != old_value # re-enable logger logging.disable(logging.NOTSET) def test_23_delete_memoized_not_callable(self): warning_raised = False try: self.memoizer.delete_memoized('function_name') except DeprecationWarning: warning_raised = True assert warning_raised @patch('memoize.Memoizer.delete', side_effect=Exception) def test_24_delete_memoized_backend_error(self, memoizer_delete): logging.basicConfig() # disable logger (to avoid cluttering test output) logging.disable(logging.ERROR) memoizer = Memoizer() @memoizer.memoize() def f(*args, **kwargs): return random.randrange(0, 100000) cache_key = f.make_cache_key(f.uncached, 'a', 'b', c='c', d='d') old_value = f('a', 'b', c='c', d='d') memoizer.delete_memoized(f, 'a', 'b', c='c', d='d') memoizer_delete.assert_called_with(cache_key) new_value = f('a', 'b', c='c', d='d') assert new_value == old_value # re-enable logger logging.disable(logging.NOTSET) def test_25_delete_memoized_verhash_not_callable(self): warning_raised = False try: self.memoizer.delete_memoized_verhash('function_name') except DeprecationWarning: warning_raised = True assert warning_raised @patch('memoize.Memoizer.delete', side_effect=Exception) def test_26_delete_memoized_verhash_backend_error(self, memoizer_delete): logging.basicConfig() # disable logger (to avoid cluttering test output) logging.disable(logging.ERROR) memoizer = Memoizer() @memoizer.memoize() def f(): return random.randrange(0, 100000) _fname, _ = function_namespace(f) version_key = self.memoizer._memvname(_fname) result = f() assert f() == result assert memoizer.get(version_key) is not memoizer.default_cache_value memoizer.delete_memoized_verhash(f) memoizer_delete.assert_called_with(version_key) assert f() == result assert memoizer.get(version_key) is not memoizer.default_cache_value # re-enable logger logging.disable(logging.NOTSET)
class MemoizeTestCase(SimpleTestCase): def setUp(self): self.memoizer = Memoizer() def test_00_set(self): self.memoizer.set('hi', 'hello') assert self.memoizer.get('hi') == 'hello' def test_01_add(self): self.memoizer.add('hi', 'hello') assert self.memoizer.get('hi') == 'hello' self.memoizer.add('hi', 'foobar') assert self.memoizer.get('hi') == 'hello' def test_02_delete(self): self.memoizer.set('hi', 'hello') self.memoizer.delete('hi') assert self.memoizer.get('hi') is None def test_06_memoize(self): @self.memoizer.memoize(5) def big_foo(a, b): return a+b+random.randrange(0, 100000) result = big_foo(5, 2) time.sleep(1) assert big_foo(5, 2) == result result2 = big_foo(5, 3) assert result2 != result time.sleep(6) assert big_foo(5, 2) != result time.sleep(1) assert big_foo(5, 3) != result2 def test_06a_memoize(self): @self.memoizer.memoize(50) def big_foo(a, b): return a+b+random.randrange(0, 100000) result = big_foo(5, 2) time.sleep(2) assert big_foo(5, 2) == result def test_07_delete_memoize(self): @self.memoizer.memoize(5) def big_foo(a, b): return a+b+random.randrange(0, 100000) result = big_foo(5, 2) result2 = big_foo(5, 3) time.sleep(1) assert big_foo(5, 2) == result assert big_foo(5, 2) == result assert big_foo(5, 3) != result assert big_foo(5, 3) == result2 self.memoizer.delete_memoized(big_foo) assert big_foo(5, 2) != result assert big_foo(5, 3) != result2 def test_07b_delete_memoized_verhash(self): @self.memoizer.memoize(5) def big_foo(a, b): return a+b+random.randrange(0, 100000) result = big_foo(5, 2) result2 = big_foo(5, 3) time.sleep(1) assert big_foo(5, 2) == result assert big_foo(5, 2) == result assert big_foo(5, 3) != result assert big_foo(5, 3) == result2 self.memoizer.delete_memoized_verhash(big_foo) _fname, _fname_instance = function_namespace(big_foo) version_key = self.memoizer._memvname(_fname) assert self.memoizer.get(version_key) is None assert big_foo(5, 2) != result assert big_foo(5, 3) != result2 assert self.memoizer.get(version_key) is not None def test_08_delete_memoize(self): @self.memoizer.memoize() def big_foo(a, b): return a+b+random.randrange(0, 100000) result_a = big_foo(5, 1) result_b = big_foo(5, 2) assert big_foo(5, 1) == result_a assert big_foo(5, 2) == result_b self.memoizer.delete_memoized(big_foo, 5, 2) assert big_foo(5, 1) == result_a assert big_foo(5, 2) != result_b # Cleanup bigfoo 5,1 5,2 or it might conflict with # following run if it also uses memecache self.memoizer.delete_memoized(big_foo, 5, 2) self.memoizer.delete_memoized(big_foo, 5, 1) def test_09_args_memoize(self): @self.memoizer.memoize() def big_foo(a, b): return sum(a)+sum(b)+random.randrange(0, 100000) result_a = big_foo([5, 3, 2], [1]) result_b = big_foo([3, 3], [3, 1]) assert big_foo([5, 3, 2], [1]) == result_a assert big_foo([3, 3], [3, 1]) == result_b self.memoizer.delete_memoized(big_foo, [5, 3, 2], [1]) assert big_foo([5, 3, 2], [1]) != result_a assert big_foo([3, 3], [3, 1]) == result_b # Cleanup bigfoo 5,1 5,2 or it might conflict with # following run if it also uses memecache self.memoizer.delete_memoized(big_foo, [5, 3, 2], [1]) self.memoizer.delete_memoized(big_foo, [3, 3], [1]) def test_10_kwargs_memoize(self): @self.memoizer.memoize() def big_foo(a, b=None): return a+sum(b.values())+random.randrange(0, 100000) result_a = big_foo(1, dict(one=1, two=2)) result_b = big_foo(5, dict(three=3, four=4)) assert big_foo(1, dict(one=1, two=2)) == result_a assert big_foo(5, dict(three=3, four=4)) == result_b self.memoizer.delete_memoized(big_foo, 1, dict(one=1, two=2)) assert big_foo(1, dict(one=1, two=2)) != result_a assert big_foo(5, dict(three=3, four=4)) == result_b def test_10a_kwargonly_memoize(self): @self.memoizer.memoize() def big_foo(a=None): if a is None: a = 0 return a+random.random() result_a = big_foo() result_b = big_foo(5) assert big_foo() == result_a assert big_foo() < 1 assert big_foo(5) == result_b assert big_foo(5) >= 5 and big_foo(5) < 6 def test_10a_arg_kwarg_memoize(self): @self.memoizer.memoize() def f(a, b, c=1): return a+b+c+random.randrange(0, 100000) assert f(1, 2) == f(1, 2, c=1) assert f(1, 2) == f(1, 2, 1) assert f(1, 2) == f(1, 2) assert f(1, 2) != f(2, 1) assert f(1, 2, 3) != f(1, 2) with self.assertRaises(TypeError): f(1) def test_10ab_arg_kwarg_memoize(self): @self.memoizer.memoize() def f(a, **kwargs): return (a, kwargs, random.randrange(0, 100000000)) f1 = f(1, b=2) assert f1 == f(1, b=2) assert f1 != f(1, b=4) f2 = f(5, c=6) assert f2 == f(5, c=6) assert f2 != f(7, c=6) self.memoizer.delete_memoized(f, 1, b=2) assert f1 != f(1, b=2) assert f2 == f(5, c=6) assert f2 != f(7, c=6) def test_10ac_arg_kwarg_memoize(self): @self.memoizer.memoize() def f(a, *args, **kwargs): return (a, args, kwargs, random.randrange(0, 100000000)) f1 = f(1, 2, b=3) assert f1 == f(1, 2, b=3) assert f1 != f(1, 3, b=3) f2 = f(5, 6, c=7) assert f2 == f(5, 6, c=7) assert f2 != f(7, 6, c=7) self.memoizer.delete_memoized(f, 1, 2, b=3) assert f1 != f(1, 2, b=3) assert f2 == f(5, 6, c=7) assert f2 != f(7, 6, c=7) @self.memoizer.memoize() def f(a, b=1, *args, **kwargs): return (a, b, args, kwargs, random.randrange(0, 100000000)) f3 = f(1, 3, 4) assert f3 == f(1, 3, 4) assert f3 == f(*(1, 3, 4)) assert f3 != f(1, 3) self.memoizer.delete_memoized(f, 1, 3, 4) assert f3 != f(1, 3, 4) def test_10b_classarg_memoize(self): @self.memoizer.memoize() def bar(a, *args): return a.value + random.random() + sum(args) class Adder(object): def __init__(self, value, *args): self.value = (value + sum(args)) adder = Adder(15) adder2 = Adder(20) adder3 = Adder(16, 5) adder4 = Adder(21, 6) w = bar(adder) x = bar(adder2) y = bar(adder3) z = bar(adder4) assert w != x assert y != z assert bar(adder) == w assert bar(adder) != x assert bar(adder3) == y assert bar(adder3) != z adder.value = 14 adder3.value = 15 assert bar(adder) == w assert bar(adder) != x assert bar(adder3) == y assert bar(adder3) != z assert bar(adder) != bar(adder2) assert bar(adder3) != bar(adder4) assert bar(adder2) == x assert bar(adder4) == z def test_10c_classfunc_memoize(self): class Adder(object): def __init__(self, initial): self.initial = initial @self.memoizer.memoize() def add(self, b, *args): return self.initial + b + sum(args) adder1 = Adder(1) adder2 = Adder(2) x = adder1.add(3) y = adder1.add(3, 4) assert adder1.add(3) == x assert adder1.add(4) != x assert adder1.add(3, 4) == y assert adder1.add(4, 4) != y assert adder1.add(3) != adder2.add(3) def test_10d_classfunc_memoize_delete(self): class Adder(object): def __init__(self, initial): self.initial = initial @self.memoizer.memoize() def add(self, b, *args): return self.initial + b + sum(args) + random.random() adder1 = Adder(1) adder2 = Adder(2) a1 = adder1.add(3) a2 = adder2.add(3) b1 = adder1.add(3, 1) b2 = adder2.add(3, 1) assert a1 != a2 assert a1 != b1 assert a2 != b2 assert adder1.add(3) == a1 assert adder2.add(3) == a2 assert adder1.add(3, 1) == b1 assert adder2.add(3, 1) == b2 self.memoizer.delete_memoized(adder1.add) a3 = adder1.add(3) a4 = adder2.add(3) b3 = adder1.add(3, 1) b4 = adder2.add(3, 1) self.assertNotEqual(a1, a3) self.assertNotEqual(b1, b3) assert a1 != a3 assert b1 != b3 self.assertEqual(a2, a4) self.assertEqual(b2, b4) self.memoizer.delete_memoized(Adder.add) a5 = adder1.add(3) a6 = adder2.add(3) b5 = adder1.add(3, 1) b6 = adder2.add(3, 1) self.assertNotEqual(a5, a6) self.assertNotEqual(b5, b6) self.assertNotEqual(a3, a5) self.assertNotEqual(b3, b5) self.assertNotEqual(a4, a6) self.assertNotEqual(b4, b6) def test_10e_delete_memoize_classmethod(self): class Mock(object): @classmethod @self.memoizer.memoize(5) def big_foo(cls, a, b, *args): return a+b+sum(args)+random.randrange(0, 100000) result = Mock.big_foo(5, 2) result2 = Mock.big_foo(5, 3) result3 = Mock.big_foo(5, 2, 1) result4 = Mock.big_foo(5, 3, 1) time.sleep(1) assert Mock.big_foo(5, 2) == result assert Mock.big_foo(5, 2) == result assert Mock.big_foo(5, 3) != result assert Mock.big_foo(5, 3) == result2 assert Mock.big_foo(5, 2, 1) == result3 assert Mock.big_foo(5, 2, 1) == result3 assert Mock.big_foo(5, 3, 1) != result3 assert Mock.big_foo(5, 3, 1) == result4 self.memoizer.delete_memoized(Mock.big_foo) assert Mock.big_foo(5, 2) != result assert Mock.big_foo(5, 3) != result2 assert Mock.big_foo(5, 2, 1) != result3 assert Mock.big_foo(5, 3, 1) != result4 def test_14_memoized_multiple_arg_kwarg_calls(self): @self.memoizer.memoize() def big_foo(a, b, c=[1, 1], d=[1, 1]): return sum(a)+sum(b)+sum(c)+sum(d)+random.randrange(0, 100000) result_a = big_foo([5, 3, 2], [1], c=[3, 3], d=[3, 3]) assert big_foo([5, 3, 2], [1], d=[3, 3], c=[3, 3]) == result_a assert big_foo(b=[1], a=[5, 3, 2], c=[3, 3], d=[3, 3]) == result_a assert big_foo([5, 3, 2], [1], [3, 3], [3, 3]) == result_a def test_15_memoize_multiple_arg_kwarg_delete(self): @self.memoizer.memoize() def big_foo(a, b, c=[1, 1], d=[1, 1]): return sum(a)+sum(b)+sum(c)+sum(d)+random.randrange(0, 100000) result_a = big_foo([5, 3, 2], [1], c=[3, 3], d=[3, 3]) self.memoizer.delete_memoized(big_foo, [5, 3, 2], [1], [3, 3], [3, 3]) result_b = big_foo([5, 3, 2], [1], c=[3, 3], d=[3, 3]) assert result_a != result_b self.memoizer.delete_memoized( big_foo, [5, 3, 2], b=[1], c=[3, 3], d=[3, 3] ) result_b = big_foo([5, 3, 2], [1], c=[3, 3], d=[3, 3]) assert result_a != result_b self.memoizer.delete_memoized( big_foo, [5, 3, 2], [1], c=[3, 3], d=[3, 3] ) result_a = big_foo([5, 3, 2], [1], c=[3, 3], d=[3, 3]) assert result_a != result_b self.memoizer.delete_memoized( big_foo, [5, 3, 2], b=[1], c=[3, 3], d=[3, 3] ) result_a = big_foo([5, 3, 2], [1], c=[3, 3], d=[3, 3]) assert result_a != result_b self.memoizer.delete_memoized( big_foo, [5, 3, 2], [1], c=[3, 3], d=[3, 3] ) result_b = big_foo([5, 3, 2], [1], c=[3, 3], d=[3, 3]) assert result_a != result_b self.memoizer.delete_memoized(big_foo, [5, 3, 2], [1], [3, 3], [3, 3]) result_a = big_foo([5, 3, 2], [1], c=[3, 3], d=[3, 3]) assert result_a != result_b def test_16_memoize_kwargs_to_args(self): def big_foo(a, b, c=None, d=None): return sum(a)+sum(b)+random.randrange(0, 100000) expected = (1, 2, 'foo', 'bar') args, kwargs = self.memoizer._memoize_kwargs_to_args( big_foo, 1, 2, 'foo', 'bar') assert (args == expected) args, kwargs = self.memoizer._memoize_kwargs_to_args( big_foo, 2, 'foo', 'bar', a=1) assert (args == expected) args, kwargs = self.memoizer._memoize_kwargs_to_args( big_foo, a=1, b=2, c='foo', d='bar') assert (args == expected) args, kwargs = self.memoizer._memoize_kwargs_to_args( big_foo, d='bar', b=2, a=1, c='foo') assert (args == expected) args, kwargs = self.memoizer._memoize_kwargs_to_args( big_foo, 1, 2, d='bar', c='foo') assert (args == expected)
class MemoizeTestCase(SimpleTestCase): def setUp(self): self.memoizer = Memoizer() def test_00_set(self): self.memoizer.set('hi', 'hello') assert self.memoizer.get('hi') == 'hello' def test_01_add(self): self.memoizer.add('hi', 'hello') assert self.memoizer.get('hi') == 'hello' self.memoizer.add('hi', 'foobar') assert self.memoizer.get('hi') == 'hello' def test_02_delete(self): self.memoizer.set('hi', 'hello') self.memoizer.delete('hi') assert self.memoizer.get('hi') is self.memoizer.default_cache_value def test_06_memoize(self): @self.memoizer.memoize(5) def big_foo(a, b, func=lambda x: x): func(a) return a + b test_func = MagicMock() big_foo(5, 2, func=test_func) # Cached call big_foo(5, 2, func=test_func) # We are expecting only 1 call here as second one should be cached assert test_func.call_count == 1, "Count: {}".format( test_func.call_count) # different arguments so we should have additional call big_foo(5, 3, func=test_func) assert test_func.call_count == 2, "Count: {}".format( test_func.call_count) # Expired now = datetime.datetime.utcfromtimestamp(time.time()) with freeze_time(now) as frozen_datetime: frozen_datetime.tick(delta=datetime.timedelta(seconds=6)) # Cache should be expired so we have increasing amount of calls big_foo(5, 2, func=test_func) assert test_func.call_count == 3, "Count: {}".format( test_func.call_count) big_foo(5, 3, func=test_func) assert test_func.call_count == 4, "Count: {}".format( test_func.call_count) def test_06a_memoize(self): @self.memoizer.memoize(50) def big_foo(a, b): return a + b + random.randrange(0, 100000) result = big_foo(5, 2) assert big_foo(5, 2) == result def test_07_delete_memoize(self): @self.memoizer.memoize(5) def big_foo(a, b): return a + b + random.randrange(0, 100000) result = big_foo(5, 2) result2 = big_foo(5, 3) assert big_foo(5, 2) == result assert big_foo(5, 2) == result assert big_foo(5, 3) != result assert big_foo(5, 3) == result2 self.memoizer.delete_memoized(big_foo) assert big_foo(5, 2) != result assert big_foo(5, 3) != result2 def test_07b_delete_memoized_verhash(self): @self.memoizer.memoize(5) def big_foo(a, b): return a + b + random.randrange(0, 100000) result = big_foo(5, 2) result2 = big_foo(5, 3) assert big_foo(5, 2) == result assert big_foo(5, 2) == result assert big_foo(5, 3) != result assert big_foo(5, 3) == result2 self.memoizer.delete_memoized_verhash(big_foo) _fname, _fname_instance = function_namespace(big_foo) version_key = self.memoizer._memvname(_fname) assert (self.memoizer.get(version_key) is self.memoizer.default_cache_value) assert big_foo(5, 2) != result assert big_foo(5, 3) != result2 assert (self.memoizer.get(version_key) is not self.memoizer.default_cache_value) def test_08_delete_memoize(self): @self.memoizer.memoize() def big_foo(a, b): return a + b + random.randrange(0, 100000) result_a = big_foo(5, 1) result_b = big_foo(5, 2) assert big_foo(5, 1) == result_a assert big_foo(5, 2) == result_b self.memoizer.delete_memoized(big_foo, 5, 2) assert big_foo(5, 1) == result_a assert big_foo(5, 2) != result_b # Cleanup bigfoo 5,1 5,2 or it might conflict with # following run if it also uses memecache self.memoizer.delete_memoized(big_foo, 5, 2) self.memoizer.delete_memoized(big_foo, 5, 1) def test_09_args_memoize(self): @self.memoizer.memoize() def big_foo(a, b): return sum(a) + sum(b) + random.randrange(0, 100000) result_a = big_foo([5, 3, 2], [1]) result_b = big_foo([3, 3], [3, 1]) assert big_foo([5, 3, 2], [1]) == result_a assert big_foo([3, 3], [3, 1]) == result_b self.memoizer.delete_memoized(big_foo, [5, 3, 2], [1]) assert big_foo([5, 3, 2], [1]) != result_a assert big_foo([3, 3], [3, 1]) == result_b # Cleanup bigfoo 5,1 5,2 or it might conflict with # following run if it also uses memecache self.memoizer.delete_memoized(big_foo, [5, 3, 2], [1]) self.memoizer.delete_memoized(big_foo, [3, 3], [1]) def test_10_kwargs_memoize(self): @self.memoizer.memoize() def big_foo(a, b=None): return a + sum(b.values()) + random.randrange(0, 100000) result_a = big_foo(1, dict(one=1, two=2)) result_b = big_foo(5, dict(three=3, four=4)) assert big_foo(1, dict(one=1, two=2)) == result_a assert big_foo(5, dict(three=3, four=4)) == result_b self.memoizer.delete_memoized(big_foo, 1, dict(one=1, two=2)) assert big_foo(1, dict(one=1, two=2)) != result_a assert big_foo(5, dict(three=3, four=4)) == result_b def test_10a_kwargonly_memoize(self): @self.memoizer.memoize() def big_foo(a=None): if a is None: a = 0 return a + random.random() result_a = big_foo() result_b = big_foo(5) assert big_foo() == result_a assert big_foo() < 1 assert big_foo(5) == result_b assert big_foo(5) >= 5 and big_foo(5) < 6 def test_10a_arg_kwarg_memoize(self): @self.memoizer.memoize() def f(a, b, c=1): return a + b + c + random.randrange(0, 100000) assert f(1, 2) == f(1, 2, c=1) assert f(1, 2) == f(1, 2, 1) assert f(1, 2) == f(1, 2) assert f(1, 2) != f(2, 1) assert f(1, 2, 3) != f(1, 2) with self.assertRaises(TypeError): f(1) def test_10ab_arg_kwarg_memoize(self): @self.memoizer.memoize() def f(a, **kwargs): return (a, kwargs, random.randrange(0, 100000000)) f1 = f(1, b=2) assert f1 == f(1, b=2) assert f1 != f(1, b=4) f2 = f(5, c=6) assert f2 == f(5, c=6) assert f2 != f(7, c=6) self.memoizer.delete_memoized(f, 1, b=2) assert f1 != f(1, b=2) assert f2 == f(5, c=6) assert f2 != f(7, c=6) def test_10ac_arg_kwarg_memoize(self): @self.memoizer.memoize() def f(a, *args, **kwargs): return (a, args, kwargs, random.randrange(0, 100000000)) f1 = f(1, 2, b=3) assert f1 == f(1, 2, b=3) assert f1 != f(1, 3, b=3) f2 = f(5, 6, c=7) assert f2 == f(5, 6, c=7) assert f2 != f(7, 6, c=7) self.memoizer.delete_memoized(f, 1, 2, b=3) assert f1 != f(1, 2, b=3) assert f2 == f(5, 6, c=7) assert f2 != f(7, 6, c=7) @self.memoizer.memoize() def f(a, b=1, *args, **kwargs): return (a, b, args, kwargs, random.randrange(0, 100000000)) f3 = f(1, 3, 4) assert f3 == f(1, 3, 4) assert f3 == f(*(1, 3, 4)) assert f3 != f(1, 3) self.memoizer.delete_memoized(f, 1, 3, 4) assert f3 != f(1, 3, 4) def test_10b_classarg_memoize(self): @self.memoizer.memoize() def bar(a, *args): return a.value + random.random() + sum(args) class Adder(object): def __init__(self, value, *args): self.value = (value + sum(args)) adder = Adder(15) adder2 = Adder(20) adder3 = Adder(16, 5) adder4 = Adder(21, 6) w = bar(adder) x = bar(adder2) y = bar(adder3) z = bar(adder4) assert w != x assert y != z assert bar(adder) == w assert bar(adder) != x assert bar(adder3) == y assert bar(adder3) != z adder.value = 14 adder3.value = 15 assert bar(adder) == w assert bar(adder) != x assert bar(adder3) == y assert bar(adder3) != z assert bar(adder) != bar(adder2) assert bar(adder3) != bar(adder4) assert bar(adder2) == x assert bar(adder4) == z def test_10c_classfunc_memoize(self): class Adder(object): def __init__(self, initial): self.initial = initial @self.memoizer.memoize() def add(self, b, *args): return self.initial + b + sum(args) adder1 = Adder(1) adder2 = Adder(2) x = adder1.add(3) y = adder1.add(3, 4) assert adder1.add(3) == x assert adder1.add(4) != x assert adder1.add(3, 4) == y assert adder1.add(4, 4) != y assert adder1.add(3) != adder2.add(3) def test_10d_classfunc_memoize_delete(self): class Adder(object): def __init__(self, initial): self.initial = initial @self.memoizer.memoize() def add(self, b, *args): return self.initial + b + sum(args) + random.random() adder1 = Adder(1) adder2 = Adder(2) a1 = adder1.add(3) a2 = adder2.add(3) b1 = adder1.add(3, 1) b2 = adder2.add(3, 1) assert a1 != a2 assert a1 != b1 assert a2 != b2 assert adder1.add(3) == a1 assert adder2.add(3) == a2 assert adder1.add(3, 1) == b1 assert adder2.add(3, 1) == b2 self.memoizer.delete_memoized(adder1.add) a3 = adder1.add(3) a4 = adder2.add(3) b3 = adder1.add(3, 1) b4 = adder2.add(3, 1) self.assertNotEqual(a1, a3) self.assertNotEqual(b1, b3) assert a1 != a3 assert b1 != b3 self.assertEqual(a2, a4) self.assertEqual(b2, b4) self.memoizer.delete_memoized(Adder.add) a5 = adder1.add(3) a6 = adder2.add(3) b5 = adder1.add(3, 1) b6 = adder2.add(3, 1) self.assertNotEqual(a5, a6) self.assertNotEqual(b5, b6) self.assertNotEqual(a3, a5) self.assertNotEqual(b3, b5) self.assertNotEqual(a4, a6) self.assertNotEqual(b4, b6) def test_10e_delete_memoize_classmethod(self): class Mock(object): @classmethod @self.memoizer.memoize(5) def big_foo(cls, a, b, *args): return a + b + sum(args) + random.randrange(0, 100000) result = Mock.big_foo(5, 2) result2 = Mock.big_foo(5, 3) result3 = Mock.big_foo(5, 2, 1) result4 = Mock.big_foo(5, 3, 1) assert Mock.big_foo(5, 2) == result assert Mock.big_foo(5, 2) == result assert Mock.big_foo(5, 3) != result assert Mock.big_foo(5, 3) == result2 assert Mock.big_foo(5, 2, 1) == result3 assert Mock.big_foo(5, 2, 1) == result3 assert Mock.big_foo(5, 3, 1) != result3 assert Mock.big_foo(5, 3, 1) == result4 self.memoizer.delete_memoized(Mock.big_foo) assert Mock.big_foo(5, 2) != result assert Mock.big_foo(5, 3) != result2 assert Mock.big_foo(5, 2, 1) != result3 assert Mock.big_foo(5, 3, 1) != result4 def test_14_memoized_multiple_arg_kwarg_calls(self): @self.memoizer.memoize() def big_foo(a, b, c=[1, 1], d=[1, 1]): return (sum(a) + sum(b) + sum(c) + sum(d) + random.randrange(0, 100000)) result_a = big_foo([5, 3, 2], [1], c=[3, 3], d=[3, 3]) assert big_foo([5, 3, 2], [1], d=[3, 3], c=[3, 3]) == result_a assert big_foo(b=[1], a=[5, 3, 2], c=[3, 3], d=[3, 3]) == result_a assert big_foo([5, 3, 2], [1], [3, 3], [3, 3]) == result_a def test_15_memoize_multiple_arg_kwarg_delete(self): @self.memoizer.memoize() def big_foo(a, b, c=[1, 1], d=[1, 1]): return (sum(a) + sum(b) + sum(c) + sum(d) + random.randrange(0, 100000)) result_a = big_foo([5, 3, 2], [1], c=[3, 3], d=[3, 3]) self.memoizer.delete_memoized(big_foo, [5, 3, 2], [1], [3, 3], [3, 3]) result_b = big_foo([5, 3, 2], [1], c=[3, 3], d=[3, 3]) assert result_a != result_b self.memoizer.delete_memoized(big_foo, [5, 3, 2], b=[1], c=[3, 3], d=[3, 3]) result_b = big_foo([5, 3, 2], [1], c=[3, 3], d=[3, 3]) assert result_a != result_b self.memoizer.delete_memoized(big_foo, [5, 3, 2], [1], c=[3, 3], d=[3, 3]) result_a = big_foo([5, 3, 2], [1], c=[3, 3], d=[3, 3]) assert result_a != result_b self.memoizer.delete_memoized(big_foo, [5, 3, 2], b=[1], c=[3, 3], d=[3, 3]) result_a = big_foo([5, 3, 2], [1], c=[3, 3], d=[3, 3]) assert result_a != result_b self.memoizer.delete_memoized(big_foo, [5, 3, 2], [1], c=[3, 3], d=[3, 3]) result_b = big_foo([5, 3, 2], [1], c=[3, 3], d=[3, 3]) assert result_a != result_b self.memoizer.delete_memoized(big_foo, [5, 3, 2], [1], [3, 3], [3, 3]) result_a = big_foo([5, 3, 2], [1], c=[3, 3], d=[3, 3]) assert result_a != result_b def test_16_memoize_kwargs_to_args(self): def big_foo(a, b, c=None, d=None): return sum(a) + sum(b) + random.randrange(0, 100000) expected = ('1', '2', "'foo'", "'bar'") args, kwargs = self.memoizer._memoize_kwargs_to_args( big_foo, 1, 2, 'foo', 'bar') assert (args == expected) args, kwargs = self.memoizer._memoize_kwargs_to_args(big_foo, 2, 'foo', 'bar', a=1) assert (args == expected) args, kwargs = self.memoizer._memoize_kwargs_to_args(big_foo, a=1, b=2, c='foo', d='bar') assert (args == expected) args, kwargs = self.memoizer._memoize_kwargs_to_args(big_foo, d='bar', b=2, a=1, c='foo') assert (args == expected) args, kwargs = self.memoizer._memoize_kwargs_to_args(big_foo, 1, 2, d='bar', c='foo') assert (args == expected) def test_17_memoize_none_value(self): self.memoizer = Memoizer() @self.memoizer.memoize() def foo(): return None cache_key = foo.make_cache_key(foo.uncached) assert (self.memoizer.get(cache_key) is self.memoizer.default_cache_value) result = foo() assert result is None assert self.memoizer.get(cache_key) is None self.memoizer.delete_memoized(foo) cache_key = foo.make_cache_key(foo.uncached) assert (self.memoizer.get(cache_key) is self.memoizer.default_cache_value) def test_17_delete_memoized_instancemethod_with_mutable_param(self): class Foo(object): def __init__(self, id): self.id = id @self.memoizer.memoize(5) def foo(self, bar_obj): return random.randrange(0, 100000) + bar_obj.id def __repr__(self): return ('{}({})'.format(self.__class__.__name__, self.id)) class Bar(object): def __init__(self, id): self.id = id def __repr__(self): return ('{}({})'.format(self.__class__.__name__, self.id)) a = Foo(1) b = Bar(1) c = Bar(2) result1 = a.foo(b) result2 = a.foo(c) assert (a.foo(b) == result1) assert (a.foo(c) == result2) self.memoizer.delete_memoized(a.foo, a, b) assert (a.foo(b) != result1) assert (a.foo(c) == result2) def test_18_python3_uses_getfullargspec(self): def foo(): return None major_version = sys.version_info.major argspec = _get_argspec(foo) if major_version < 3: assert argspec.__class__.__name__ is 'ArgSpec' else: assert argspec.__class__.__name__ is 'FullArgSpec' def test_19_test_custom_repr_fn(self): custom_memoizer = Memoizer(repr_fn=lambda x: 'static') memoizer = Memoizer() class Test(object): def fn(self, arg=None): return 1 cached_fn = memoizer.memoize(60)(fn) cached_fn_custom_repr = custom_memoizer.memoize(60)(fn) def __str__(self): return 'str' __unicode__ = __str__ def __repr__(self): return 'repr' fake_obj = Test() fake_obj2 = Test() def _get_keys(obj): cache_key = obj.cached_fn.make_cache_key(obj.fn, obj, 123) custom_cache_key = obj.cached_fn_custom_repr.make_cache_key( obj.fn, obj, 123) return cache_key, custom_cache_key cache_key, custom_cache_key = _get_keys(fake_obj) assert cache_key != custom_cache_key cache_key2, custom_cache_key2 = _get_keys(fake_obj2) assert cache_key == cache_key2 assert custom_cache_key == custom_cache_key2 custom_cache_key3 = fake_obj.cached_fn_custom_repr.make_cache_key( fake_obj.fn, fake_obj, ['test']) assert custom_cache_key == custom_cache_key3
import calendar import datetime import functools from memoize import Memoizer store = {} memo = Memoizer(store) def next_month(year, month): """ Return the year and month of the next month NOTE: month is 1 based """ return add_months(year, month, 1) def add_months(year, month, months): """ Return the year and month of #months after the given year and month NOTE: month is 1 based """ month += months while month > 12: month -= 12 year += 1 return year, month def record_runtime(func):
from urlparse import urljoin import requests from cloudfoundry.apps import CloudFoundryApp from cloudfoundry.organizations import CloudFoundryOrg from cloudfoundry.spaces import CloudFoundrySpace from cloudfoundry.routes import CloudFoundryRoute from cloudfoundry.domains import CloudFoundryDomain from utils import create_bits_zip from collections import OrderedDict import logging import time from memoize import Memoizer cache_store = {} memo = Memoizer(cache_store) max_cache_time = 10 class CloudFoundryException(Exception): pass class CloudFoundryAuthenticationException(CloudFoundryException): pass class CloudFoundryInterface(object): def __init__(self, target, username=None, password=None, debug=False): self._apps = None