class Remember(object): def __init__( self, function, filename='', write_freq=1, name='Remember'): # store self.function = function self.filename = filename self.write_freq = write_freq self.name = name # initialize cache from file if filename and os.path.exists(filename) and os.path.isfile(filename): self.load_cache() # initialize new cache else: self.__cache__ = HashedDict() return def __func__(self,inputs): outputs = self.function(inputs) return outputs def __call__(self,inputs): # hashable type for cache _inputs = make_hashable(inputs) # check cache if self.__cache__.has_key(_inputs): #print 'PULLED FROM CACHE' outputs = deepcopy( self.__cache__[_inputs] ) # evalute function else: outputs = self.__func__(inputs) self.__cache__[_inputs] = deepcopy(outputs) #: if cached # save cache if self.filename and ( len(self.__cache__) % self.write_freq ) == 0: self.save_cache() # done return outputs def load_cache(self): if not self.filename: raise AttributeError , 'no filename for loading cache' self.__cache__ = load_data(filename) def save_cache(self): if not self.filename: raise AttributeError , 'no filename for saving cache' save_data(self.__cache__,self.filename)
def __init__(self, filename, initial=None, timeout=None, delay=0.5): self.filename = os.path.abspath(filename) self.timeout = timeout self.delay = delay if not (os.path.exists(filename) and os.path.isfile(filename)) and not initial: initial = HashedDict() if not initial is None: save(initial, self.filename, file_format='pickle')
def __init__(self, function, filename='', write_freq=1, name='Remember'): # store self.function = function self.filename = filename self.write_freq = write_freq self.name = name # check for instance method try: self._func_self = function.im_self except: self._func_self = None # initialize cache from file if filename and os.path.exists(filename) and os.path.isfile(filename): self.load_cache() # initialize new cache else: self.__cache__ = HashedDict() return
def __init__( self, function, filename='', write_freq=1, name='Remember'): # store self.function = function self.filename = filename self.write_freq = write_freq self.name = name # initialize cache from file if filename and os.path.exists(filename) and os.path.isfile(filename): self.load_cache() # initialize new cache else: self.__cache__ = HashedDict() return
def get(self): if self.outbox is None: raise AttributeError, 'no outbox to query' # for sorting results results = HashedDict() # wait for all jobs to process self.inbox.join() # pull results while not self.outbox.empty(): task = self.outbox.get() results[task.inputs] = task return results
def __init__( self, function, filename='', write_freq=1, name='Remember'): # store self.function = function self.filename = filename self.write_freq = write_freq self.name = name # check for instance method try: self._func_self = function.im_self except: self._func_self = None # initialize cache from file if filename and os.path.exists(filename) and os.path.isfile(filename): self.load_cache() # initialize new cache else: self.__cache__ = HashedDict() return
class Remember(object): def __init__(self, function, filename='', write_freq=1, name='Remember'): # store self.function = function self.filename = filename self.write_freq = write_freq self.name = name # check for instance method try: self._func_self = function.im_self except: self._func_self = None # initialize cache from file if filename and os.path.exists(filename) and os.path.isfile(filename): self.load_cache() # initialize new cache else: self.__cache__ = HashedDict() return def __func__(self, inputs): # evaluate function outputs = self.function(inputs) return outputs def __call__(self, inputs): # hashable type for cache _inputs = make_hashable(inputs) # check cache if self.__cache__.has_key(_inputs): #print 'PULLED FROM CACHE' outputs = deepcopy(self.__cache__[_inputs]) # evalute function else: outputs = self.__func__(inputs) self.__cache__[_inputs] = deepcopy(outputs) #: if cached # save cache if self.filename and (len(self.__cache__) % self.write_freq) == 0: self.save_cache() # done return outputs def load_cache(self): if not self.filename: raise AttributeError, 'no filename for loading cache' self.__cache__ = load_data(filename) def save_cache(self): if not self.filename: raise AttributeError, 'no filename for saving cache' save_data(self.__cache__, self.filename) def __getstate__(self): """ handles an instance method in self.function """ state = self.__dict__.copy() if state['_func_self']: state['function'] = state['function'].__name__ return state def __setstate__(self, state): """ handles an instance method in self.function """ self.__dict__.update(state) if self._func_self: self.function = getattr(self._func_self, self.function)
def main(): # -------------------------------------------------------- # Initialize # -------------------------------------------------------- cache = HashedDict() # -------------------------------------------------------- # Load up data # -------------------------------------------------------- cache['a'] = 1 # normal dictionary keys are strings cache[[1, 2, 3]] = 2 # HashedDict accepts lists for example cache[[1, 2, 5]] = 5 funny_key = object() cache[[6, 2, 5]] = HashedDict() # sub-dictionary cache[[6, 2, 5]][funny_key] = 77 # -------------------------------------------------------- # Printing # -------------------------------------------------------- print '>>> print cache' print cache print '>>> print cache[[1,2,3]]' print cache[[1, 2, 3]] print '' print '>>> print cache[(1,2,3)]' print cache[(1, 2, 3)] print '' print 'should be True:', cache.has_key([1, 2, 3]) assert cache.has_key([1, 2, 3]) print 'should be True:', [1, 2, 3] in cache assert [1, 2, 3] in cache del cache[[1, 2, 3]] print 'should be False:', cache.has_key([1, 2, 3]) assert not cache.has_key([1, 2, 3]) print '' # -------------------------------------------------------- # Pickling test # -------------------------------------------------------- print '>>> pickle.dumps()' d = pickle.dumps(cache) print '>>> pickle.loads()' p = pickle.loads(d) print '' print '>>> print p' print p print 'should be True:', [1, 2, 5] in p assert [1, 2, 5] in p # beware after pickling some objects... print 'should be False:', funny_key in p[[6, 2, 5]] assert not funny_key in p[[6, 2, 5]] print '' # -------------------------------------------------------- # Access Speed test # -------------------------------------------------------- print 'Access speed test...' # accessing bunch t0 = time() for i in range(int(1e5)): v = cache[[6, 2, 5]][funny_key] t1 = time() - t0 # a test dictionary z = dict() z['t'] = dict() z['t']['i'] = 0 # accessing a normal dictionary t0 = time() for i in range(int(1e5)): v = z['t']['i'] t2 = time() - t0 # results print 'HashedDict: %.6f s' % (t1) print 'dict: %.6f s' % (t2) assert (t1 - t2) / t2 < 60.0 print '' # -------------------------------------------------------- # Assignment Speed test # -------------------------------------------------------- print 'Assignment speed test...' # accessing bunch t0 = time() for i in range(int(1e5)): v = cache[[6, 2, 5]][funny_key] = 10 t1 = time() - t0 # accessing a normal dictionary t0 = time() for i in range(int(1e5)): z['t']['i'] = 10 t2 = time() - t0 # results print 'HashedDict: %.6f s' % (t1) print 'dict: %.6f s' % (t2) assert (t1 - t2) / t2 < 60.0 print ''
def main(): # -------------------------------------------------------- # Initialize # -------------------------------------------------------- cache = HashedDict() # -------------------------------------------------------- # Load up data # -------------------------------------------------------- cache['a'] = 1 # normal dictionary keys are strings cache[[1,2,3]] = 2 # HashedDict accepts lists for example cache[[1,2,5]] = 5 funny_key = object() cache[[6,2,5]] = HashedDict() # sub-dictionary cache[[6,2,5]][funny_key] = 77 # -------------------------------------------------------- # Printing # -------------------------------------------------------- print '>>> print cache' print cache print '>>> print cache[[1,2,3]]' print cache[[1,2,3]] print '' print '>>> print cache[(1,2,3)]' print cache[(1,2,3)] print '' print 'should be True:' , cache.has_key([1,2,3]) assert cache.has_key([1,2,3]) print 'should be True:' , [1,2,3] in cache assert [1,2,3] in cache del cache[[1,2,3]] print 'should be False:' , cache.has_key([1,2,3]) assert not cache.has_key([1,2,3]) print '' # -------------------------------------------------------- # Pickling test # -------------------------------------------------------- print '>>> pickle.dumps()' d = pickle.dumps(cache) print '>>> pickle.loads()' p = pickle.loads(d) print '' print '>>> print p' print p print 'should be True:' , [1,2,5] in p assert [1,2,5] in p # beware after pickling some objects... print 'should be False:' , funny_key in p[[6,2,5]] assert not funny_key in p[[6,2,5]] print '' # -------------------------------------------------------- # Access Speed test # -------------------------------------------------------- print 'Access speed test...' # accessing bunch t0 = time() for i in range(int(1e5)): v = cache[[6,2,5]][funny_key] t1 = time()-t0 # a test dictionary z = dict() z['t'] = dict() z['t']['i'] = 0 # accessing a normal dictionary t0 = time() for i in range(int(1e5)): v = z['t']['i'] t2 = time()-t0 # results print 'HashedDict: %.6f s' % (t1) print 'dict: %.6f s' % (t2) assert (t1-t2)/t2 < 60.0 print '' # -------------------------------------------------------- # Assignment Speed test # -------------------------------------------------------- print 'Assignment speed test...' # accessing bunch t0 = time() for i in range(int(1e5)): v = cache[[6,2,5]][funny_key] = 10 t1 = time()-t0 # accessing a normal dictionary t0 = time() for i in range(int(1e5)): z['t']['i'] = 10 t2 = time()-t0 # results print 'HashedDict: %.6f s' % (t1) print 'dict: %.6f s' % (t2) assert (t1-t2)/t2 < 60.0 print ''
class Remember(object): def __init__( self, function, filename='', write_freq=1, name='Remember'): # store self.function = function self.filename = filename self.write_freq = write_freq self.name = name # check for instance method try: self._func_self = function.im_self except: self._func_self = None # initialize cache from file if filename and os.path.exists(filename) and os.path.isfile(filename): self.load_cache() # initialize new cache else: self.__cache__ = HashedDict() return def __func__(self,inputs): # evaluate function outputs = self.function(inputs) return outputs def __call__(self,inputs): # hashable type for cache _inputs = make_hashable(inputs) # check cache if self.__cache__.has_key(_inputs): #print 'PULLED FROM CACHE' outputs = deepcopy( self.__cache__[_inputs] ) # evalute function else: outputs = self.__func__(inputs) self.__cache__[_inputs] = deepcopy(outputs) #: if cached # save cache if self.filename and ( len(self.__cache__) % self.write_freq ) == 0: self.save_cache() # done return outputs def load_cache(self): if not self.filename: raise AttributeError , 'no filename for loading cache' self.__cache__ = load_data(filename) def save_cache(self): if not self.filename: raise AttributeError , 'no filename for saving cache' save_data(self.__cache__,self.filename) def __getstate__(self): """ handles an instance method in self.function """ state = self.__dict__.copy() if state['_func_self']: state['function'] = state['function'].__name__ return state def __setstate__(self,state): """ handles an instance method in self.function """ self.__dict__.update(state) if self._func_self: self.function = getattr( self._func_self , self.function)