def __init__(self, raw, buffer_size=DEFAULT_BUFFER_SIZE): if not raw.writable(): raise IOError('"raw" argument must be writable.') _BufferedIOMixin.__init__(self, raw) if buffer_size <= 0: raise ValueError('invalid buffer size') self.buffer_size = buffer_size self._write_buf = bytearray() self._write_lock = Lock()
def __init__(self, raw, buffer_size=DEFAULT_BUFFER_SIZE): if not raw.readable(): raise IOError('"raw" argument must be readable.') _BufferedIOMixin.__init__(self, raw) if buffer_size <= 0: raise ValueError('invalid buffer size') self.buffer_size = buffer_size self._reset_read_buf() self._read_lock = Lock()
def decorating_function(user_function): cache = {} # mapping of args to results use_count = Counter() # times each key has been accessed kwd_mark = object() # separate positional and keyword args lock = Lock() @wraps(user_function) def wrapper(*args, **kwds): key = args if kwds: key += (kwd_mark, ) + tuple(sorted(kwds.items())) use_count[key] += 1 # get cache entry or compute if not found try: with lock: result = cache[key] wrapper.hits += 1 except KeyError: result = user_function(*args, **kwds) with lock: cache[key] = result wrapper.misses += 1 # purge least frequently used cache entry if len(cache) > maxsize: for key, _ in nsmallest(maxsize // 10, iter(use_count.items()), key=itemgetter(1)): del cache[key], use_count[key] return result def cache_info(): """Report cache statistics""" with lock: return _CacheInfo(wrapper.hits, wrapper.misses, maxsize, len(cache)) def cache_clear(): with lock: cache.clear() use_count.clear() wrapper.hits = wrapper.misses = 0 wrapper.hits = wrapper.misses = 0 wrapper.cache_clear = cache_clear return wrapper
def decorating_function(user_function, tuple=tuple, sorted=sorted, len=len, KeyError=KeyError): hits = misses = 0 kwd_mark = (object(),) # separates positional and keyword args lock = Lock() # needed because OrderedDict isn't threadsafe if maxsize is None: cache = dict() # simple cache without ordering or size limit @wraps(user_function) def wrapper(*args, **kwds): nonlocal hits, misses key = args if kwds: key += kwd_mark + tuple(sorted(kwds.items())) try: result = cache[key] hits += 1 return result except KeyError: pass result = user_function(*args, **kwds) cache[key] = result misses += 1 return result else: cache = OrderedDict() # ordered least recent to most recent cache_popitem = cache.popitem cache_renew = cache.move_to_end @wraps(user_function) def wrapper(*args, **kwds): nonlocal hits, misses key = args if kwds: key += kwd_mark + tuple(sorted(kwds.items())) with lock: try: result = cache[key] cache_renew(key) # record recent use of this key hits += 1 return result except KeyError: pass result = user_function(*args, **kwds) with lock: cache[key] = result # record recent use of this key misses += 1 if len(cache) > maxsize: cache_popitem(0) # purge least recently used cache entry return result def cache_info(): """Report cache statistics""" with lock: return _CacheInfo(hits, misses, maxsize, len(cache)) def cache_clear(): """Clear the cache and cache statistics""" nonlocal hits, misses with lock: cache.clear() hits = misses = 0 wrapper.cache_info = cache_info wrapper.cache_clear = cache_clear return wrapper
def decorating_function(user_function): cache = {} hits = misses = currsize = 0 full = False cache_get = cache.get # bound method to lookup a key or return None lock = Lock() # because linkedlist updates aren't threadsafe root = [] # root of the circular doubly linked list root[:] = [root, root, None, None] # initialize by pointing to self if maxsize == 0: def wrapper(*args, **kwds): # no caching, just a statistics update after a successful call nonlocal misses result = user_function(*args, **kwds) misses += 1 return result elif maxsize is None: def wrapper(*args, **kwds): # simple caching without ordering or size limit nonlocal hits, misses, currsize key = make_key(args, kwds, typed) result = cache_get(key, sentinel) if result is not sentinel: hits += 1 return result result = user_function(*args, **kwds) cache[key] = result misses += 1 currsize += 1 return result else: def wrapper(*args, **kwds): # size limited caching that tracks accesses by recency nonlocal root, hits, misses, currsize, full key = make_key(args, kwds, typed) with lock: link = cache_get(key) if link is not None: # move the link to the front of the circular queue link_prev, link_next, key, result = link link_prev[NEXT] = link_next link_next[PREV] = link_prev last = root[PREV] last[NEXT] = root[PREV] = link link[PREV] = last link[NEXT] = root hits += 1 return result result = user_function(*args, **kwds) with lock: if key in cache: # getting here means that this same key was added to the # cache while the lock was released. since the link # update is already done, we need only return the # computed result and update the count of misses. pass elif full: # use root to store the new key and result root[KEY] = key root[RESULT] = result cache[key] = root # empty the oldest link and make it the new root root = root[NEXT] del cache[root[KEY]] root[KEY] = root[RESULT] = None else: # put result in a new link at the front of the queue last = root[PREV] link = [last, root, key, result] cache[key] = last[NEXT] = root[PREV] = link currsize += 1 full = (currsize == maxsize) misses += 1 return result def cache_info(): """Report cache statistics""" with lock: return _CacheInfo(hits, misses, maxsize, currsize) def cache_clear(): """Clear the cache and cache statistics""" nonlocal hits, misses, currsize, full with lock: cache.clear() root[:] = [root, root, None, None] hits = misses = currsize = 0 full = False wrapper.cache_info = cache_info wrapper.cache_clear = cache_clear return update_wrapper(wrapper, user_function)
def decorating_function(user_function, len=len, iter=iter, tuple=tuple, sorted=sorted, KeyError=KeyError): cache = {} # mapping of args to results queue = deque() # order that keys have been used refcount = Counter() # times each key is in the queue sentinel = object() # marker for looping around the queue kwd_mark = object() # separates positional and keyword args lock = Lock() # lookup optimizations (ugly but fast) queue_append, queue_popleft = queue.append, queue.popleft queue_appendleft, queue_pop = queue.appendleft, queue.pop @wraps(user_function) def wrapper(*args, **kwds): # cache key records both positional and keyword args key = args if kwds: key += (kwd_mark, ) + tuple(sorted(kwds.items())) # record recent use of this key queue_append(key) refcount[key] += 1 # get cache entry or compute if not found try: with lock: result = cache[key] wrapper.hits += 1 except KeyError: result = user_function(*args, **kwds) with lock: cache[key] = result wrapper.misses += 1 # purge least recently used cache entry if len(cache) > maxsize: key = queue_popleft() refcount[key] -= 1 while refcount[key]: key = queue_popleft() refcount[key] -= 1 del cache[key], refcount[key] # periodically compact the queue by eliminating duplicate keys # while preserving order of most recent access if len(queue) > maxqueue: with lock: refcount.clear() queue_appendleft(sentinel) for key in filterfalse(refcount.__contains__, iter(queue_pop, sentinel)): queue_appendleft(key) refcount[key] = 1 return result def cache_info(): """Report cache statistics""" with lock: return _CacheInfo(wrapper.hits, wrapper.misses, maxsize, len(cache)) def cache_clear(): with lock: cache.clear() queue.clear() refcount.clear() wrapper.hits = wrapper.misses = 0 wrapper.hits = wrapper.misses = 0 wrapper.cache_info = cache_info wrapper.cache_clear = cache_clear return wrapper
] dependent_variable = include[-1] model_directory = 'model' # model_file_name = '%s/model.pkl' % model_directory model_default_name = "default" # UIMA / features stuff # type system isaac_ts = uima.load_isaac_ts() # feature extraction extraction = FeatureExtraction() # in-memory feature data features = {} lock = Lock() # These will be populated at training time model_columns = {} clf = {} # model objects # load existing models try: for f in os.listdir(model_directory): if f.endswith(".pkl"): if "_columns" in f: model_id = f[:-12] model_columns[model_id] = joblib.load( '{}/{}_columns.pkl'.format(model_directory, model_id)) print('model columns {} loaded'.format(model_id)) else: