class Remember(object):
def __init__( self, function, filename='', write_freq=1, name='Remember'):
# store
self.function = function
self.filename = filename
self.write_freq = write_freq
self.name = name
# initialize cache from file
if filename and os.path.exists(filename) and os.path.isfile(filename):
self.load_cache()
# initialize new cache
else:
self.__cache__ = HashedDict()
return
def __func__(self,inputs):
outputs = self.function(inputs)
return outputs
def __call__(self,inputs):
# hashable type for cache
_inputs = make_hashable(inputs)
# check cache
if self.__cache__.has_key(_inputs):
#print 'PULLED FROM CACHE'
outputs = deepcopy( self.__cache__[_inputs] )
# evalute function
else:
outputs = self.__func__(inputs)
self.__cache__[_inputs] = deepcopy(outputs)
#: if cached
# save cache
if self.filename and ( len(self.__cache__) % self.write_freq ) == 0:
self.save_cache()
# done
return outputs
def load_cache(self):
if not self.filename:
raise AttributeError , 'no filename for loading cache'
self.__cache__ = load_data(filename)
def save_cache(self):
if not self.filename:
raise AttributeError , 'no filename for saving cache'
save_data(self.__cache__,self.filename)
def __init__(self, filename, initial=None, timeout=None, delay=0.5):
self.filename = os.path.abspath(filename)
self.timeout = timeout
self.delay = delay
if not (os.path.exists(filename)
and os.path.isfile(filename)) and not initial:
initial = HashedDict()
if not initial is None:
save(initial, self.filename, file_format='pickle')
def __init__(self, function, filename='', write_freq=1, name='Remember'):
# store
self.function = function
self.filename = filename
self.write_freq = write_freq
self.name = name
# check for instance method
try:
self._func_self = function.im_self
except:
self._func_self = None
# initialize cache from file
if filename and os.path.exists(filename) and os.path.isfile(filename):
self.load_cache()
# initialize new cache
else:
self.__cache__ = HashedDict()
return
def __init__( self, function, filename='', write_freq=1, name='Remember'):
# store
self.function = function
self.filename = filename
self.write_freq = write_freq
self.name = name
# initialize cache from file
if filename and os.path.exists(filename) and os.path.isfile(filename):
self.load_cache()
# initialize new cache
else:
self.__cache__ = HashedDict()
return
def get(self):
if self.outbox is None:
raise AttributeError, 'no outbox to query'
# for sorting results
results = HashedDict()
# wait for all jobs to process
self.inbox.join()
# pull results
while not self.outbox.empty():
task = self.outbox.get()
results[task.inputs] = task
return results
def __init__( self, function, filename='', write_freq=1, name='Remember'):
# store
self.function = function
self.filename = filename
self.write_freq = write_freq
self.name = name
# check for instance method
try:
self._func_self = function.im_self
except:
self._func_self = None
# initialize cache from file
if filename and os.path.exists(filename) and os.path.isfile(filename):
self.load_cache()
# initialize new cache
else:
self.__cache__ = HashedDict()
return
class Remember(object):
def __init__(self, function, filename='', write_freq=1, name='Remember'):
# store
self.function = function
self.filename = filename
self.write_freq = write_freq
self.name = name
# check for instance method
try:
self._func_self = function.im_self
except:
self._func_self = None
# initialize cache from file
if filename and os.path.exists(filename) and os.path.isfile(filename):
self.load_cache()
# initialize new cache
else:
self.__cache__ = HashedDict()
return
def __func__(self, inputs):
# evaluate function
outputs = self.function(inputs)
return outputs
def __call__(self, inputs):
# hashable type for cache
_inputs = make_hashable(inputs)
# check cache
if self.__cache__.has_key(_inputs):
#print 'PULLED FROM CACHE'
outputs = deepcopy(self.__cache__[_inputs])
# evalute function
else:
outputs = self.__func__(inputs)
self.__cache__[_inputs] = deepcopy(outputs)
#: if cached
# save cache
if self.filename and (len(self.__cache__) % self.write_freq) == 0:
self.save_cache()
# done
return outputs
def load_cache(self):
if not self.filename:
raise AttributeError, 'no filename for loading cache'
self.__cache__ = load_data(filename)
def save_cache(self):
if not self.filename:
raise AttributeError, 'no filename for saving cache'
save_data(self.__cache__, self.filename)
def __getstate__(self):
""" handles an instance method in self.function """
state = self.__dict__.copy()
if state['_func_self']:
state['function'] = state['function'].__name__
return state
def __setstate__(self, state):
""" handles an instance method in self.function """
self.__dict__.update(state)
if self._func_self:
self.function = getattr(self._func_self, self.function)
def main():
# --------------------------------------------------------
# Initialize
# --------------------------------------------------------
cache = HashedDict()
# --------------------------------------------------------
# Load up data
# --------------------------------------------------------
cache['a'] = 1 # normal dictionary keys are strings
cache[[1, 2, 3]] = 2 # HashedDict accepts lists for example
cache[[1, 2, 5]] = 5
funny_key = object()
cache[[6, 2, 5]] = HashedDict() # sub-dictionary
cache[[6, 2, 5]][funny_key] = 77
# --------------------------------------------------------
# Printing
# --------------------------------------------------------
print '>>> print cache'
print cache
print '>>> print cache[[1,2,3]]'
print cache[[1, 2, 3]]
print ''
print '>>> print cache[(1,2,3)]'
print cache[(1, 2, 3)]
print ''
print 'should be True:', cache.has_key([1, 2, 3])
assert cache.has_key([1, 2, 3])
print 'should be True:', [1, 2, 3] in cache
assert [1, 2, 3] in cache
del cache[[1, 2, 3]]
print 'should be False:', cache.has_key([1, 2, 3])
assert not cache.has_key([1, 2, 3])
print ''
# --------------------------------------------------------
# Pickling test
# --------------------------------------------------------
print '>>> pickle.dumps()'
d = pickle.dumps(cache)
print '>>> pickle.loads()'
p = pickle.loads(d)
print ''
print '>>> print p'
print p
print 'should be True:', [1, 2, 5] in p
assert [1, 2, 5] in p
# beware after pickling some objects...
print 'should be False:', funny_key in p[[6, 2, 5]]
assert not funny_key in p[[6, 2, 5]]
print ''
# --------------------------------------------------------
# Access Speed test
# --------------------------------------------------------
print 'Access speed test...'
# accessing bunch
t0 = time()
for i in range(int(1e5)):
v = cache[[6, 2, 5]][funny_key]
t1 = time() - t0
# a test dictionary
z = dict()
z['t'] = dict()
z['t']['i'] = 0
# accessing a normal dictionary
t0 = time()
for i in range(int(1e5)):
v = z['t']['i']
t2 = time() - t0
# results
print 'HashedDict: %.6f s' % (t1)
print 'dict: %.6f s' % (t2)
assert (t1 - t2) / t2 < 60.0
print ''
# --------------------------------------------------------
# Assignment Speed test
# --------------------------------------------------------
print 'Assignment speed test...'
# accessing bunch
t0 = time()
for i in range(int(1e5)):
v = cache[[6, 2, 5]][funny_key] = 10
t1 = time() - t0
# accessing a normal dictionary
t0 = time()
for i in range(int(1e5)):
z['t']['i'] = 10
t2 = time() - t0
# results
print 'HashedDict: %.6f s' % (t1)
print 'dict: %.6f s' % (t2)
assert (t1 - t2) / t2 < 60.0
print ''
def main():
# --------------------------------------------------------
# Initialize
# --------------------------------------------------------
cache = HashedDict()
# --------------------------------------------------------
# Load up data
# --------------------------------------------------------
cache['a'] = 1 # normal dictionary keys are strings
cache[[1,2,3]] = 2 # HashedDict accepts lists for example
cache[[1,2,5]] = 5
funny_key = object()
cache[[6,2,5]] = HashedDict() # sub-dictionary
cache[[6,2,5]][funny_key] = 77
# --------------------------------------------------------
# Printing
# --------------------------------------------------------
print '>>> print cache'
print cache
print '>>> print cache[[1,2,3]]'
print cache[[1,2,3]]
print ''
print '>>> print cache[(1,2,3)]'
print cache[(1,2,3)]
print ''
print 'should be True:' , cache.has_key([1,2,3])
assert cache.has_key([1,2,3])
print 'should be True:' , [1,2,3] in cache
assert [1,2,3] in cache
del cache[[1,2,3]]
print 'should be False:' , cache.has_key([1,2,3])
assert not cache.has_key([1,2,3])
print ''
# --------------------------------------------------------
# Pickling test
# --------------------------------------------------------
print '>>> pickle.dumps()'
d = pickle.dumps(cache)
print '>>> pickle.loads()'
p = pickle.loads(d)
print ''
print '>>> print p'
print p
print 'should be True:' , [1,2,5] in p
assert [1,2,5] in p
# beware after pickling some objects...
print 'should be False:' , funny_key in p[[6,2,5]]
assert not funny_key in p[[6,2,5]]
print ''
# --------------------------------------------------------
# Access Speed test
# --------------------------------------------------------
print 'Access speed test...'
# accessing bunch
t0 = time()
for i in range(int(1e5)):
v = cache[[6,2,5]][funny_key]
t1 = time()-t0
# a test dictionary
z = dict()
z['t'] = dict()
z['t']['i'] = 0
# accessing a normal dictionary
t0 = time()
for i in range(int(1e5)):
v = z['t']['i']
t2 = time()-t0
# results
print 'HashedDict: %.6f s' % (t1)
print 'dict: %.6f s' % (t2)
assert (t1-t2)/t2 < 60.0
print ''
# --------------------------------------------------------
# Assignment Speed test
# --------------------------------------------------------
print 'Assignment speed test...'
# accessing bunch
t0 = time()
for i in range(int(1e5)):
v = cache[[6,2,5]][funny_key] = 10
t1 = time()-t0
# accessing a normal dictionary
t0 = time()
for i in range(int(1e5)):
z['t']['i'] = 10
t2 = time()-t0
# results
print 'HashedDict: %.6f s' % (t1)
print 'dict: %.6f s' % (t2)
assert (t1-t2)/t2 < 60.0
print ''
class Remember(object):
def __init__( self, function, filename='', write_freq=1, name='Remember'):
# store
self.function = function
self.filename = filename
self.write_freq = write_freq
self.name = name
# check for instance method
try:
self._func_self = function.im_self
except:
self._func_self = None
# initialize cache from file
if filename and os.path.exists(filename) and os.path.isfile(filename):
self.load_cache()
# initialize new cache
else:
self.__cache__ = HashedDict()
return
def __func__(self,inputs):
# evaluate function
outputs = self.function(inputs)
return outputs
def __call__(self,inputs):
# hashable type for cache
_inputs = make_hashable(inputs)
# check cache
if self.__cache__.has_key(_inputs):
#print 'PULLED FROM CACHE'
outputs = deepcopy( self.__cache__[_inputs] )
# evalute function
else:
outputs = self.__func__(inputs)
self.__cache__[_inputs] = deepcopy(outputs)
#: if cached
# save cache
if self.filename and ( len(self.__cache__) % self.write_freq ) == 0:
self.save_cache()
# done
return outputs
def load_cache(self):
if not self.filename:
raise AttributeError , 'no filename for loading cache'
self.__cache__ = load_data(filename)
def save_cache(self):
if not self.filename:
raise AttributeError , 'no filename for saving cache'
save_data(self.__cache__,self.filename)
def __getstate__(self):
""" handles an instance method in self.function """
state = self.__dict__.copy()
if state['_func_self']:
state['function'] = state['function'].__name__
return state
def __setstate__(self,state):
""" handles an instance method in self.function """
self.__dict__.update(state)
if self._func_self:
self.function = getattr( self._func_self , self.function)