def __init__(self, path=None, symmetry=None, hydrogens=False, pdbfix=False, vdw_radii=None, **kwargs): self._kwargs = kwargs.copy() GeneProvider.__init__(self, **kwargs) self._kwargs = kwargs self.path = path self.symmetry = symmetry self.hydrogens = hydrogens self.pdbfix = pdbfix self.vdw_radii = vdw_radii try: self.catalog = self._CATALOG[self.name] except KeyError: self.catalog = self._CATALOG[self.name] = tuple( self._compile_catalog()) self._compounds_cache = self._cache.setdefault( self.name + '_compounds', LRU(300)) self._atomlookup_cache = self._cache.setdefault( self.name + '_atomlookup', LRU(300)) self._residuelookup_cache = self._cache.setdefault( self.name + '_residuelookup', LRU(300)) self.allele = random.choice(self.catalog) # An optimization for similarity methods: xform coords are # cached here after all genes have expressed. See Individual.express. self._expressed_coordinates = None
def __init__(self, cache_location=None, pickle_cache=False, memory_cache_size=None): if memory_cache_size: self._cache = LRU(max_size=memory_cache_size) else: self._cache = {} self.cache_location = cache_location self.pickle_cache = pickle_cache
def __init__(self, max_size: int, ttl_millis: int, from_remote): ''' :param max_size: max_size of the cache,using LRU :param ttl_millis: max live time for the objects :param from_remote: function to load data if the key not exists ''' self.cache = LRU(max_size=max_size) self.ttl = ttl_millis self.from_remote = from_remote self._lock = RLock()
def __init__(self, name, max_size=128, from_remote=None): ''' :param max_size: 最大容量 :param from_remote: 查询数据函数,返回值必须是一个dict :param ttl: 有效时间,单位:秒 ''' self.local_cache = LRU(max_size=max_size) self.from_remote = from_remote self.empty = {} self.name = name
def test_lru(): lru = LRU(max_size=1) lru['hi'] = 0 lru['bye'] = 1 assert len(lru) == 1 lru['bye'] assert lru.get('hi') is None del lru['bye'] assert 'bye' not in lru assert len(lru) == 0 assert not lru
class CacheModule: def __init__(self, max_size: int, ttl_millis: int, from_remote): ''' :param max_size: max_size of the cache,using LRU :param ttl_millis: max live time for the objects :param from_remote: function to load data if the key not exists ''' self.cache = LRU(max_size=max_size) self.ttl = ttl_millis self.from_remote = from_remote self._lock = RLock() def multi_get(self, keys) -> dict: result, missing = {}, [] now = int(round(time.time() * 1000)) for key in keys: value: dict = self.cache.get(key) # key未命中 if not value: missing.append(key) # key超时 elif now < value['max_time']: missing.append(key) # key存在且不是默认值 elif value['data']: result[key] = value['data'] if len(missing) > 0: loads_d: dict = self.from_remote(missing) dft = { 'max_time': int(round(time.time() * 1000)) + self.ttl, 'data': {} } for key in missing: loads_d.setdefault(key, dft) self.multi_set(loads_d) return result def multi_set(self, data: dict): with self._lock: for k, v in data.items(): self.cache.setdefault(k, v) def remove(self, keys): with self._lock: for key in keys: self.cache.pop(key)
def __init__(self, residues=None, library='Dunbrack', avoid_replacement=False, mutations=[], ligation=False, hydrogens=False, **kwargs): GeneProvider.__init__(self, **kwargs) self._kwargs = kwargs self._residues = residues self.library = library self.mutations = mutations self.ligation = ligation self.hydrogens = hydrogens self.avoid_replacement = avoid_replacement self.allele = [] # set caches try: self.residues = self._cache[self.name + '_residues'] except KeyError: self.residues = self._cache[self.name + '_residues'] = OrderedDict() try: self.rotamers = self._cache[self.name + '_rotamers'] except KeyError: cache_size = len(residues) * (1 + 0.5 * len(mutations)) self.rotamers = self._cache[self.name + '_rotamers'] = LRU(int(cache_size)) if self.ligation: self.random_number = random.random() else: self.random_number = None # Avoid unnecessary calls to expensive get_rotamers if residue is known # to not have any rotamers self._residues_without_rotamers = ['ALA', 'GLY']
def __setstate__(self, state): if not '_cache' in state: if state.get('memory_cache_size'): state['_cache'] = LRU(max_size=memory_cache_size) else: state['_cache'] = {} self.__dict__ = dict(state)
def __init__(self, stimuli, fixations, max_fixations_in_cache=500*1000*1000): self.stimuli = stimuli self.fixations = fixations cache_size = int(max_fixations_in_cache / len(self.fixations.x)) self.cache = LRU(cache_size) self.nonfixations_for_image = cached(self.cache)(self._nonfixations_for_image) self.widths = np.asarray([s[1] for s in stimuli.sizes]).astype(float) self.heights = np.asarray([s[0] for s in stimuli.sizes]).astype(float)
class CacheModule: def __init__(self, name, max_size=128, from_remote=None): ''' :param max_size: 最大容量 :param from_remote: 查询数据函数,返回值必须是一个dict :param ttl: 有效时间,单位:秒 ''' self.local_cache = LRU(max_size=max_size) self.from_remote = from_remote self.empty = {} self.name = name def get_all(self, keys: list) -> dict: result, missing_keys = {}, [] for key in keys: val = self.local_cache.get(key) if val is not None: if val != self.empty: result[key] = val else: missing_keys.append(key) # 缓存未命中 if len(missing_keys) > 0: missing_dict = self.reload(missing_keys) result = {**result, **missing_dict} return result def remove(self, keys: list): for key in keys: self.local_cache.__delitem__(key) def statistics(self): hitRateAsString = '%.3f' % ( self.local_cache.hit_count / (self.local_cache.hit_count + self.local_cache.miss_count)) return { "name": self.name, "hit_count": self.local_cache.hit_count, "miss_count": self.local_cache.miss_count, "size": len(self.local_cache.keys()), "hitRateAsString": hitRateAsString + "%", } def reload(self, keys: list) -> dict: d = {} missing_vals = self.from_remote(keys) # 不存在的值,设置为空 if missing_vals: for key, val in missing_vals.items(): self.local_cache.setdefault(key, val) d[key] = val for missing_key in keys: self.local_cache.setdefault(missing_key, self.empty) return d
def test_cachedmethod(): class Car(object): def __init__(self, cache=None): self.h_cache = LRI() if cache is None else cache self.door_count = 0 self.hook_count = 0 self.hand_count = 0 @cachedmethod('h_cache') def hand(self, *a, **kw): self.hand_count += 1 @cachedmethod(lambda obj: obj.h_cache) def hook(self, *a, **kw): self.hook_count += 1 @cachedmethod('h_cache', scoped=False) def door(self, *a, **kw): self.door_count += 1 car = Car() # attribute name-style assert car.hand_count == 0 car.hand('h', a='nd') assert car.hand_count == 1 car.hand('h', a='nd') assert car.hand_count == 1 # callable-style assert car.hook_count == 0 car.hook() assert car.hook_count == 1 car.hook() assert car.hook_count == 1 # Ensure that non-selfish caches share the cache nicely lru = LRU() car_one = Car(cache=lru) assert car_one.door_count == 0 car_one.door('bob') assert car_one.door_count == 1 car_one.door('bob') assert car_one.door_count == 1 car_two = Car(cache=lru) assert car_two.door_count == 0 car_two.door('bob') assert car_two.door_count == 0 # try unbound for kicks Car.door(Car(), 'bob') # always check the repr print(repr(car_two.door)) print(repr(Car.door)) return
def __init__(self, username, password, uri_base, dry_run=False): self.log = logging.getLogger('confl-copier') self._dry_run = dry_run self._client = ConfluenceAPIDryRunProxy(username=username, password=password, uri_base=uri_base, dry_run=dry_run) self._cache = LRU()
def test_lru_with_dupes_2(): "From Issue #55, h/t github.com/mt" SIZE = 3 lru = LRU(max_size=SIZE) keys = ['A', 'A', 'B', 'A', 'C', 'B', 'D', 'E'] for i, k in enumerate(keys): lru[k] = 'HIT' assert _test_linkage(lru._anchor, SIZE + 1), 'linked list invalid' return
def __init__(self, parent_model, stimuli, resized_predictions_cache_size=5000, compute_size=(500, 500), **kwargs): super(ShuffledBaselineModel, self).__init__(**kwargs) self.parent_model = parent_model self.stimuli = stimuli self.compute_size = compute_size self.resized_predictions_cache = LRU( max_size=resized_predictions_cache_size, on_miss=self._cache_miss)
def test_cached_dec(): lru = LRU() inner_func = CountingCallable() func = cached(lru)(inner_func) assert inner_func.call_count == 0 func() assert inner_func.call_count == 1 func() assert inner_func.call_count == 1 func('man door hand hook car door') assert inner_func.call_count == 2 return
def create_devices(netbox_api, devices, role_id, site_id, threads=10): device_types_mapper = NetboxMapper(netbox_api, "dcim", "device-types") platforms_mapper = NetboxMapper(netbox_api, "dcim", "platforms") caches = { "device_types": LRU(on_miss=lambda slug: next(device_types_mapper.get(slug=slug))), "platforms": LRU(on_miss=lambda slug: next(platforms_mapper.get(slug=slug))) } device_mapper = NetboxMapper(netbox_api, "dcim", "devices") with ThreadPoolExecutor(max_workers=threads) as executor: futures = [] for name, props in devices.items(): future = executor.submit(_thread_push_device, device_mapper, caches, name, props, role_id, site_id) futures.append(future) try: [future.result() in concurrent.futures.as_completed(futures)] except requests.exceptions.HTTPError as e: print(e.response.data) raise
def test_callable_cached_dec(): lru = LRU() get_lru = lambda: lru inner_func = CountingCallable() func = cached(get_lru)(inner_func) assert inner_func.call_count == 0 func() assert inner_func.call_count == 1 func() assert inner_func.call_count == 1 lru.clear() func() assert inner_func.call_count == 2 func() assert inner_func.call_count == 2 print(repr(func)) return
def test_unscoped_cached_dec(): lru = LRU() inner_func = CountingCallable() func = cached(lru)(inner_func) other_inner_func = CountingCallable() other_func = cached(lru)(other_inner_func) assert inner_func.call_count == 0 func('a') assert inner_func.call_count == 1 func('a') other_func('a') assert other_inner_func.call_count == 0 return
def __init__(self, parent_model, stimuli, resized_predictions_cache_size=5000, compute_size=(500, 500), library='torch', **kwargs): super(ShuffledBaselineModel, self).__init__(**kwargs) self.parent_model = parent_model self.stimuli = stimuli self.compute_size = compute_size self.resized_predictions_cache = LRU( max_size=resized_predictions_cache_size, on_miss=self._cache_miss) if library not in ['torch', 'tensorflow', 'numpy']: raise ValueError(library) self.library = library
def encodeSwoopTensorInFormat(tensor, descriptor, tensor_shape=None, cache_size=32): codec = Codec(tuple(descriptor), [True] * len(descriptor)) # get output dict based on rank names rank_names = tensor.getRankIds() # print("encode tensor: rank names {}, descriptor {}".format(rank_names, descriptor)) # TODO: move output dict generation into codec output = codec.get_output_dict(rank_names) # print("output dict {}".format(output)) output_tensor = [] for i in range(0, len(descriptor) + 1): output_tensor.append(list()) # print("encode, output {}".format(output_tensor)) codec.encode(-1, tensor.getRoot(), tensor.getRankIds(), output, output_tensor, shape=tensor_shape) # name the fibers in order from left to right per-rank rank_idx = 0 rank_names = ["root"] + tensor.getRankIds() # tensor_cache = dict() tensor_cache = LRU(max_size=cache_size) for rank in output_tensor: fiber_idx = 0 for fiber in rank: fiber_name = "_".join( [tensor.getName(), rank_names[rank_idx], str(fiber_idx)]) fiber.setName(fiber_name) # fiber.printFiber() fiber.cache = tensor_cache fiber_idx += 1 rank_idx += 1 return output_tensor
def __init__(self, method='prody', target=None, modes=None, n_samples=10000, rmsd=1.0, group_by=None, group_lambda=None, path=None, write_modes=False, **kwargs): # Fire up! GeneProvider.__init__(self, **kwargs) self.method = method self.target = target self.modes = modes if modes is not None else range(12) self.max_modes = max(self.modes) + 1 self.n_samples = n_samples self.rmsd = rmsd self.group_by = None self.group_by_options = None self.path = None self.write_modes = write_modes if method == 'prody': if path is None: self.normal_modes_function = self.calculate_prody_normal_modes self.group_by = group_by self.group_by_options = {} if group_lambda is None else { 'n': group_lambda } else: self.path = path self.normal_modes_function = self.read_prody_normal_modes else: # gaussian self.normal_modes_function = self.read_gaussian_normal_modes if path is None: raise ValueError('Path is required if method == gaussian') self.path = path if self.name not in self._cache: self._cache[self.name] = LRU(300)
from flask_sqlalchemy import BaseQuery, Pagination, SQLAlchemy from sqlalchemy import exc as sqlalchemy_err, text from sqlalchemy.engine.result import ResultProxy, RowProxy from sqlalchemy.exc import DatabaseError, IntegrityError from sqlalchemy.orm import Query, Session from sqlalchemy.orm.attributes import InstrumentedAttribute from sqlalchemy.orm.scoping import scoped_session from sqlalchemy.sql.schema import Table from .err_msg import mysql_msg from .exceptions import DBDuplicateKeyError, DBError, FuncArgsError, HttpError from .utils import gen_class_name, verify_message __all__ = ("DBClient", "DialectDriver") _lru_cache = LRU() class DialectDriver(object): """ 数据库方言驱动 """ # postgresql pg_default = "postgresql+psycopg2" # default pg_pg8000 = "postgresql+pg8000" # mysql mysql_default = "mysql+mysqldb" # default mysql_pymysql = "mysql+pymysql" # oracle oracle_cx = "oracle+cx_oracle" # default # SQL Server
def test_lru_basic(): lru = LRU(max_size=1) repr(lru) # sanity lru['hi'] = 0 lru['bye'] = 1 assert len(lru) == 1 lru['bye'] assert lru.get('hi') is None del lru['bye'] assert 'bye' not in lru assert len(lru) == 0 assert not lru try: lru.pop('bye') except KeyError: pass else: assert False default = object() assert lru.pop('bye', default) is default try: lru.popitem() except KeyError: pass else: assert False lru['another'] = 1 assert lru.popitem() == ('another', 1) lru['yet_another'] = 2 assert lru.pop('yet_another') == 2 lru['yet_another'] = 3 assert lru.pop('yet_another', default) == 3 lru['yet_another'] = 4 lru.clear() assert not lru lru['yet_another'] = 5 second_lru = LRU(max_size=1) assert lru.copy() == lru second_lru['yet_another'] = 5 assert second_lru == lru assert lru == second_lru lru.update(LRU(max_size=2, values=[('a', 1), ('b', 2)])) assert len(lru) == 1 assert 'yet_another' not in lru lru.setdefault('x', 2) assert dict(lru) == {'x': 2} lru.setdefault('x', 3) assert dict(lru) == {'x': 2} assert lru != second_lru assert second_lru != lru
def __init__(self, study_path='study.json'): self.gepResult = None with open(study_path) as json_file: config = json.load(json_file, object_pairs_hook=OrderedDict) #save the config with the output filename = config['parameters']['output_path'] + study_path #make sure the output_path folder exists if not os.path.exists(config['parameters']['output_path']): os.makedirs(config['parameters']['output_path']) pretty = json.dumps(config, indent=2, separators=(',', ':')) with open(filename, 'w') as outfile: outfile.write(pretty) outfile.close() # print(json.dumps(config['ontology'], indent=2)) self.parameters = config['parameters'] super().__init__(self.parameters['seed']) self.reproduction_report = self.reproduction_report() self.blackboard = config['blackboard'] self.ontology = config['ontology'] self.registry = registry self.emergent_functions = OrderedDict() self.emergent_functions_arity = OrderedDict() self.emergent_functions_call_number = 0 self.stochastic_pattern = re.compile(r'_stochastic\d+') self.prefix_pattern = re.compile(r'^f\d+_') pickle_config_path = config['parameters'][ 'output_path'] + 'pickles/' + 'index.p' if pickle_config_path and os.path.exists(pickle_config_path): with open(pickle_config_path, 'rb') as cachehandle: pickle_config = pickle.load(cachehandle) else: pickle_config = OrderedDict([("count", 0), ("pickles", OrderedDict())]) self.pickle_count = pickle_config[ 'count'] # contains the next number for the pickle file self.pickles = pickle_config['pickles'] self.resultTuple = () # self.cache = LRU(max_size = 512) self.cache = LRU() # Buyers gather offers by ranking those who offer to sell the product that have a price overlap. # call choose partners several times to ensure that all parts that the supply chain has a # chance to be settled in multiple trades, or offer networks have a chance to be filled. # In step the agent has a chance to put out a new message given the knowledge of purchases # made on the last round stage_list = [ 'step', 'gather_offers', 'choose_partners', 'choose_partners', 'choose_partners', 'choose_partners', 'choose_partners', 'choose_partners', 'choose_partners', 'choose_partners', 'choose_partners', 'choose_partners', 'choose_partners', 'choose_partners', 'choose_partners', 'choose_partners', 'choose_partners', 'choose_partners', 'choose_partners', 'choose_partners', 'choose_partners', 'choose_partners', ] self.schedule = StagedActivation(self, stage_list=stage_list, shuffle=True, shuffle_between_stages=True) # Create agents # first initial agents then random agents initial_blackboard = copy.deepcopy(self.blackboard) self.blackboard = [] agent_count = 0 for i, message in enumerate(initial_blackboard): if message['type'] in self.parameters['agent_parameters']: agent_parameters = self.parameters['agent_parameters'][ message['type']] else: agent_parameters = None for _ in range(self.parameters['blackboard_agents'][i]): a = globals()[message['type']](agent_count, self, message, agent_parameters) self.schedule.add(a) agent_count += 1 for agent_type, n in self.parameters['random_agents'].items(): if agent_type in self.parameters['agent_parameters']: agent_parameters = self.parameters['agent_parameters'][ agent_type] else: agent_parameters = None for i in range(n): a = globals()[agent_type](agent_count, self, None, agent_parameters) self.schedule.add(a) agent_count += 1 print("Final line of snet sim init")
def test_lru_add(): cache = LRU(max_size=3) for i in range(4): cache[i] = i assert len(cache) == 3 assert 0 not in cache
from typing import Any from typing import Dict import asyncpg import ujson from boltons.cacheutils import LRU from app.core.config import settings from app.postgres.db.session import db CACHE = LRU(max_size=settings.CACHE_ITEMS_MAX_SIZE) class User(db.Model): __USER_CACHE_KEY = "USERS:ID:{user_id}" __tablename__ = "users" id = db.Column(db.BigInteger(), primary_key=True) nickname = db.Column(db.Unicode(), default="unnamed") @classmethod async def get_user_by_id(cls, user_id: int) -> "User": cache_key: str = cls.__get_cache_key(user_id) if cache_key not in CACHE: print(f"User {user_id} cache MISS.") fetched_user: User = await cls.get_or_404(user_id) CACHE[cache_key]: User = fetched_user return CACHE[cache_key] @classmethod def db_event(cls, con_ref: asyncpg.Connection, pid: int, channel: str,
def load(self): self._config = convert_config(self._file_config) self._word_cache = LRU(self.cache_size)
class Cache(MutableMapping): """Cache that supports saving the items to files Set `cache_location` to save all newly set items to .npy files in cache_location. .. warning :: Items that have been set before setting `cache_location` won't be saved to files! """ def __init__(self, cache_location=None, pickle_cache=False, memory_cache_size=None): if memory_cache_size: self._cache = LRU(max_size=memory_cache_size) else: self._cache = {} self.cache_location = cache_location self.pickle_cache = pickle_cache def clear(self): """ Clear memory cache""" self._cache = {} def filename(self, key): return os.path.join(self.cache_location, '{}.npy'.format(key)) def __getitem__(self, key): if not key in self._cache: if self.cache_location is not None: filename = self.filename(key) if os.path.exists(filename): value = np.load(filename) self._cache[key] = value else: raise KeyError('Key {} neither in cache nor on disk'.format(key)) return self._cache[key] def __setitem__(self, key, value): if not isinstance(key, str): raise TypeError('Only string keys are supported right now!') if self.cache_location is not None: if not os.path.exists(self.cache_location): os.makedirs(self.cache_location) filename = self.filename(key) np.save(filename, value) self._cache[key] = value def __delitem__(self, key): if self.cache_location is not None: filename = self.filename(key) if os.path.exists(filename): os.remove(filename) del self._cache[key] def __iter__(self): if self.cache_location is not None: filenames = iglob(self.filename('*')) keys = map(lambda f: os.path.splitext(os.path.basename(f))[0], filenames) new_keys = filterfalse(lambda key: key in self._cache.keys(), keys) return chain(iterkeys(self._cache), new_keys) else: return iterkeys(self._cache) def __len__(self): i = iter(self) return len(list(i)) def __getstate__(self): # we don't want to save the cache state = dict(self.__dict__) if not self.pickle_cache: state.pop('_cache') return state def __setstate__(self, state): if not '_cache' in state: if state.get('memory_cache_size'): state['_cache'] = LRU(max_size=memory_cache_size) else: state['_cache'] = {} self.__dict__ = dict(state)
def test_lru_with_dupes(): SIZE = 2 lru = LRU(max_size=SIZE) for i in [0, 0, 1, 1, 2, 2]: lru[i] = i assert _test_linkage(lru._anchor, SIZE + 1), 'linked list invalid'
def test_popitem_should_return_a_tuple(): cache = LRU() cache['t'] = 42 assert cache.popitem() == ('t', 42)