def __init__(self, raw, buffer_size=DEFAULT_BUFFER_SIZE):
if not raw.writable():
raise IOError('"raw" argument must be writable.')
_BufferedIOMixin.__init__(self, raw)
if buffer_size <= 0:
raise ValueError('invalid buffer size')
self.buffer_size = buffer_size
self._write_buf = bytearray()
self._write_lock = Lock()
def __init__(self, raw, buffer_size=DEFAULT_BUFFER_SIZE):
if not raw.readable():
raise IOError('"raw" argument must be readable.')
_BufferedIOMixin.__init__(self, raw)
if buffer_size <= 0:
raise ValueError('invalid buffer size')
self.buffer_size = buffer_size
self._reset_read_buf()
self._read_lock = Lock()
def decorating_function(user_function):
cache = {} # mapping of args to results
use_count = Counter() # times each key has been accessed
kwd_mark = object() # separate positional and keyword args
lock = Lock()
@wraps(user_function)
def wrapper(*args, **kwds):
key = args
if kwds:
key += (kwd_mark, ) + tuple(sorted(kwds.items()))
use_count[key] += 1
# get cache entry or compute if not found
try:
with lock:
result = cache[key]
wrapper.hits += 1
except KeyError:
result = user_function(*args, **kwds)
with lock:
cache[key] = result
wrapper.misses += 1
# purge least frequently used cache entry
if len(cache) > maxsize:
for key, _ in nsmallest(maxsize // 10,
iter(use_count.items()),
key=itemgetter(1)):
del cache[key], use_count[key]
return result
def cache_info():
"""Report cache statistics"""
with lock:
return _CacheInfo(wrapper.hits, wrapper.misses, maxsize,
len(cache))
def cache_clear():
with lock:
cache.clear()
use_count.clear()
wrapper.hits = wrapper.misses = 0
wrapper.hits = wrapper.misses = 0
wrapper.cache_clear = cache_clear
return wrapper
def decorating_function(user_function,
tuple=tuple, sorted=sorted, len=len, KeyError=KeyError):
hits = misses = 0
kwd_mark = (object(),) # separates positional and keyword args
lock = Lock() # needed because OrderedDict isn't threadsafe
if maxsize is None:
cache = dict() # simple cache without ordering or size limit
@wraps(user_function)
def wrapper(*args, **kwds):
nonlocal hits, misses
key = args
if kwds:
key += kwd_mark + tuple(sorted(kwds.items()))
try:
result = cache[key]
hits += 1
return result
except KeyError:
pass
result = user_function(*args, **kwds)
cache[key] = result
misses += 1
return result
else:
cache = OrderedDict() # ordered least recent to most recent
cache_popitem = cache.popitem
cache_renew = cache.move_to_end
@wraps(user_function)
def wrapper(*args, **kwds):
nonlocal hits, misses
key = args
if kwds:
key += kwd_mark + tuple(sorted(kwds.items()))
with lock:
try:
result = cache[key]
cache_renew(key) # record recent use of this key
hits += 1
return result
except KeyError:
pass
result = user_function(*args, **kwds)
with lock:
cache[key] = result # record recent use of this key
misses += 1
if len(cache) > maxsize:
cache_popitem(0) # purge least recently used cache entry
return result
def cache_info():
"""Report cache statistics"""
with lock:
return _CacheInfo(hits, misses, maxsize, len(cache))
def cache_clear():
"""Clear the cache and cache statistics"""
nonlocal hits, misses
with lock:
cache.clear()
hits = misses = 0
wrapper.cache_info = cache_info
wrapper.cache_clear = cache_clear
return wrapper
def decorating_function(user_function):
cache = {}
hits = misses = currsize = 0
full = False
cache_get = cache.get # bound method to lookup a key or return None
lock = Lock() # because linkedlist updates aren't threadsafe
root = [] # root of the circular doubly linked list
root[:] = [root, root, None, None] # initialize by pointing to self
if maxsize == 0:
def wrapper(*args, **kwds):
# no caching, just a statistics update after a successful call
nonlocal misses
result = user_function(*args, **kwds)
misses += 1
return result
elif maxsize is None:
def wrapper(*args, **kwds):
# simple caching without ordering or size limit
nonlocal hits, misses, currsize
key = make_key(args, kwds, typed)
result = cache_get(key, sentinel)
if result is not sentinel:
hits += 1
return result
result = user_function(*args, **kwds)
cache[key] = result
misses += 1
currsize += 1
return result
else:
def wrapper(*args, **kwds):
# size limited caching that tracks accesses by recency
nonlocal root, hits, misses, currsize, full
key = make_key(args, kwds, typed)
with lock:
link = cache_get(key)
if link is not None:
# move the link to the front of the circular queue
link_prev, link_next, key, result = link
link_prev[NEXT] = link_next
link_next[PREV] = link_prev
last = root[PREV]
last[NEXT] = root[PREV] = link
link[PREV] = last
link[NEXT] = root
hits += 1
return result
result = user_function(*args, **kwds)
with lock:
if key in cache:
# getting here means that this same key was added to the
# cache while the lock was released. since the link
# update is already done, we need only return the
# computed result and update the count of misses.
pass
elif full:
# use root to store the new key and result
root[KEY] = key
root[RESULT] = result
cache[key] = root
# empty the oldest link and make it the new root
root = root[NEXT]
del cache[root[KEY]]
root[KEY] = root[RESULT] = None
else:
# put result in a new link at the front of the queue
last = root[PREV]
link = [last, root, key, result]
cache[key] = last[NEXT] = root[PREV] = link
currsize += 1
full = (currsize == maxsize)
misses += 1
return result
def cache_info():
"""Report cache statistics"""
with lock:
return _CacheInfo(hits, misses, maxsize, currsize)
def cache_clear():
"""Clear the cache and cache statistics"""
nonlocal hits, misses, currsize, full
with lock:
cache.clear()
root[:] = [root, root, None, None]
hits = misses = currsize = 0
full = False
wrapper.cache_info = cache_info
wrapper.cache_clear = cache_clear
return update_wrapper(wrapper, user_function)
def decorating_function(user_function,
len=len,
iter=iter,
tuple=tuple,
sorted=sorted,
KeyError=KeyError):
cache = {} # mapping of args to results
queue = deque() # order that keys have been used
refcount = Counter() # times each key is in the queue
sentinel = object() # marker for looping around the queue
kwd_mark = object() # separates positional and keyword args
lock = Lock()
# lookup optimizations (ugly but fast)
queue_append, queue_popleft = queue.append, queue.popleft
queue_appendleft, queue_pop = queue.appendleft, queue.pop
@wraps(user_function)
def wrapper(*args, **kwds):
# cache key records both positional and keyword args
key = args
if kwds:
key += (kwd_mark, ) + tuple(sorted(kwds.items()))
# record recent use of this key
queue_append(key)
refcount[key] += 1
# get cache entry or compute if not found
try:
with lock:
result = cache[key]
wrapper.hits += 1
except KeyError:
result = user_function(*args, **kwds)
with lock:
cache[key] = result
wrapper.misses += 1
# purge least recently used cache entry
if len(cache) > maxsize:
key = queue_popleft()
refcount[key] -= 1
while refcount[key]:
key = queue_popleft()
refcount[key] -= 1
del cache[key], refcount[key]
# periodically compact the queue by eliminating duplicate keys
# while preserving order of most recent access
if len(queue) > maxqueue:
with lock:
refcount.clear()
queue_appendleft(sentinel)
for key in filterfalse(refcount.__contains__,
iter(queue_pop, sentinel)):
queue_appendleft(key)
refcount[key] = 1
return result
def cache_info():
"""Report cache statistics"""
with lock:
return _CacheInfo(wrapper.hits, wrapper.misses, maxsize,
len(cache))
def cache_clear():
with lock:
cache.clear()
queue.clear()
refcount.clear()
wrapper.hits = wrapper.misses = 0
wrapper.hits = wrapper.misses = 0
wrapper.cache_info = cache_info
wrapper.cache_clear = cache_clear
return wrapper
]
dependent_variable = include[-1]
model_directory = 'model'
# model_file_name = '%s/model.pkl' % model_directory
model_default_name = "default"
# UIMA / features stuff
# type system
isaac_ts = uima.load_isaac_ts()
# feature extraction
extraction = FeatureExtraction()
# in-memory feature data
features = {}
lock = Lock()
# These will be populated at training time
model_columns = {}
clf = {} # model objects
# load existing models
try:
for f in os.listdir(model_directory):
if f.endswith(".pkl"):
if "_columns" in f:
model_id = f[:-12]
model_columns[model_id] = joblib.load(
'{}/{}_columns.pkl'.format(model_directory, model_id))
print('model columns {} loaded'.format(model_id))
else:
