Exemplo n.º 1
0
class Remember(object):
    
    def __init__( self, function, filename='', write_freq=1, name='Remember'):
           
        # store
        self.function   = function
        self.filename   = filename
        self.write_freq = write_freq
        self.name       = name
        
        # initialize cache from file
        if filename and os.path.exists(filename) and os.path.isfile(filename):
            self.load_cache()
            
        # initialize new cache
        else:
            self.__cache__ = HashedDict()
        
        return
        
    def __func__(self,inputs):
        outputs = self.function(inputs)
        return outputs
        
    def __call__(self,inputs):
            
        # hashable type for cache
        _inputs = make_hashable(inputs)
                
        # check cache
        if self.__cache__.has_key(_inputs): 
            #print 'PULLED FROM CACHE'
            outputs = deepcopy( self.__cache__[_inputs] )
        
        # evalute function
        else:
            outputs = self.__func__(inputs)
            self.__cache__[_inputs] = deepcopy(outputs)
        
        #: if cached
        
        # save cache
        if self.filename and ( len(self.__cache__) % self.write_freq ) == 0:
            self.save_cache()
        
        # done
        return outputs
    
    def load_cache(self):
        if not self.filename:
            raise AttributeError , 'no filename for loading cache'
        self.__cache__ = load_data(filename)
    
    def save_cache(self):
        if not self.filename:
            raise AttributeError , 'no filename for saving cache'
        save_data(self.__cache__,self.filename)
Exemplo n.º 2
0
 def __init__(self, filename, initial=None, timeout=None, delay=0.5):
     self.filename = os.path.abspath(filename)
     self.timeout = timeout
     self.delay = delay
     if not (os.path.exists(filename)
             and os.path.isfile(filename)) and not initial:
         initial = HashedDict()
     if not initial is None:
         save(initial, self.filename, file_format='pickle')
Exemplo n.º 3
0
    def __init__(self, function, filename='', write_freq=1, name='Remember'):

        # store
        self.function = function
        self.filename = filename
        self.write_freq = write_freq
        self.name = name

        # check for instance method
        try:
            self._func_self = function.im_self
        except:
            self._func_self = None

        # initialize cache from file
        if filename and os.path.exists(filename) and os.path.isfile(filename):
            self.load_cache()

        # initialize new cache
        else:
            self.__cache__ = HashedDict()

        return
Exemplo n.º 4
0
 def __init__( self, function, filename='', write_freq=1, name='Remember'):
        
     # store
     self.function   = function
     self.filename   = filename
     self.write_freq = write_freq
     self.name       = name
     
     # initialize cache from file
     if filename and os.path.exists(filename) and os.path.isfile(filename):
         self.load_cache()
         
     # initialize new cache
     else:
         self.__cache__ = HashedDict()
     
     return
Exemplo n.º 5
0
    def get(self):

        if self.outbox is None:
            raise AttributeError, 'no outbox to query'

        # for sorting results
        results = HashedDict()

        # wait for all jobs to process
        self.inbox.join()

        # pull results
        while not self.outbox.empty():
            task = self.outbox.get()
            results[task.inputs] = task

        return results
Exemplo n.º 6
0
 def __init__( self, function, filename='', write_freq=1, name='Remember'):
        
     # store
     self.function   = function
     self.filename   = filename
     self.write_freq = write_freq
     self.name       = name
     
     # check for instance method
     try:
         self._func_self = function.im_self
     except:
         self._func_self = None
     
     # initialize cache from file
     if filename and os.path.exists(filename) and os.path.isfile(filename):
         self.load_cache()
         
     # initialize new cache
     else:
         self.__cache__ = HashedDict()
     
     return
Exemplo n.º 7
0
class Remember(object):
    def __init__(self, function, filename='', write_freq=1, name='Remember'):

        # store
        self.function = function
        self.filename = filename
        self.write_freq = write_freq
        self.name = name

        # check for instance method
        try:
            self._func_self = function.im_self
        except:
            self._func_self = None

        # initialize cache from file
        if filename and os.path.exists(filename) and os.path.isfile(filename):
            self.load_cache()

        # initialize new cache
        else:
            self.__cache__ = HashedDict()

        return

    def __func__(self, inputs):
        # evaluate function
        outputs = self.function(inputs)
        return outputs

    def __call__(self, inputs):

        # hashable type for cache
        _inputs = make_hashable(inputs)

        # check cache
        if self.__cache__.has_key(_inputs):
            #print 'PULLED FROM CACHE'
            outputs = deepcopy(self.__cache__[_inputs])

        # evalute function
        else:
            outputs = self.__func__(inputs)
            self.__cache__[_inputs] = deepcopy(outputs)

        #: if cached

        # save cache
        if self.filename and (len(self.__cache__) % self.write_freq) == 0:
            self.save_cache()

        # done
        return outputs

    def load_cache(self):
        if not self.filename:
            raise AttributeError, 'no filename for loading cache'
        self.__cache__ = load_data(filename)

    def save_cache(self):
        if not self.filename:
            raise AttributeError, 'no filename for saving cache'
        save_data(self.__cache__, self.filename)

    def __getstate__(self):
        """ handles an instance method in self.function """
        state = self.__dict__.copy()
        if state['_func_self']:
            state['function'] = state['function'].__name__
        return state

    def __setstate__(self, state):
        """ handles an instance method in self.function """
        self.__dict__.update(state)
        if self._func_self:
            self.function = getattr(self._func_self, self.function)
Exemplo n.º 8
0
def main():

    # --------------------------------------------------------
    #   Initialize
    # --------------------------------------------------------

    cache = HashedDict()

    # --------------------------------------------------------
    #   Load up data
    # --------------------------------------------------------

    cache['a'] = 1  # normal dictionary keys are strings
    cache[[1, 2, 3]] = 2  # HashedDict accepts lists for example
    cache[[1, 2, 5]] = 5

    funny_key = object()

    cache[[6, 2, 5]] = HashedDict()  # sub-dictionary
    cache[[6, 2, 5]][funny_key] = 77

    # --------------------------------------------------------
    #   Printing
    # --------------------------------------------------------

    print '>>> print cache'
    print cache

    print '>>> print cache[[1,2,3]]'
    print cache[[1, 2, 3]]
    print ''

    print '>>> print cache[(1,2,3)]'
    print cache[(1, 2, 3)]
    print ''

    print 'should be True:', cache.has_key([1, 2, 3])
    assert cache.has_key([1, 2, 3])

    print 'should be True:', [1, 2, 3] in cache
    assert [1, 2, 3] in cache

    del cache[[1, 2, 3]]
    print 'should be False:', cache.has_key([1, 2, 3])
    assert not cache.has_key([1, 2, 3])
    print ''

    # --------------------------------------------------------
    #   Pickling test
    # --------------------------------------------------------

    print '>>> pickle.dumps()'
    d = pickle.dumps(cache)
    print '>>> pickle.loads()'
    p = pickle.loads(d)
    print ''

    print '>>> print p'
    print p

    print 'should be True:', [1, 2, 5] in p
    assert [1, 2, 5] in p

    # beware after pickling some objects...
    print 'should be False:', funny_key in p[[6, 2, 5]]
    assert not funny_key in p[[6, 2, 5]]
    print ''

    # --------------------------------------------------------
    #   Access Speed test
    # --------------------------------------------------------
    print 'Access speed test...'

    # accessing bunch
    t0 = time()
    for i in range(int(1e5)):
        v = cache[[6, 2, 5]][funny_key]
    t1 = time() - t0

    # a test dictionary
    z = dict()
    z['t'] = dict()
    z['t']['i'] = 0

    # accessing a normal dictionary
    t0 = time()
    for i in range(int(1e5)):
        v = z['t']['i']
    t2 = time() - t0

    # results
    print 'HashedDict: %.6f s' % (t1)
    print 'dict:       %.6f s' % (t2)
    assert (t1 - t2) / t2 < 60.0
    print ''

    # --------------------------------------------------------
    #   Assignment Speed test
    # --------------------------------------------------------
    print 'Assignment speed test...'

    # accessing bunch
    t0 = time()
    for i in range(int(1e5)):
        v = cache[[6, 2, 5]][funny_key] = 10
    t1 = time() - t0

    # accessing a normal dictionary
    t0 = time()
    for i in range(int(1e5)):
        z['t']['i'] = 10
    t2 = time() - t0

    # results
    print 'HashedDict: %.6f s' % (t1)
    print 'dict:       %.6f s' % (t2)
    assert (t1 - t2) / t2 < 60.0
    print ''
Exemplo n.º 9
0
def main():

    # --------------------------------------------------------
    #   Initialize
    # --------------------------------------------------------
    
    cache = HashedDict()

    
    # --------------------------------------------------------
    #   Load up data
    # --------------------------------------------------------
    
    cache['a'] = 1      # normal dictionary keys are strings
    cache[[1,2,3]] = 2  # HashedDict accepts lists for example
    cache[[1,2,5]] = 5    
    
    funny_key = object()
    
    cache[[6,2,5]] = HashedDict()  # sub-dictionary
    cache[[6,2,5]][funny_key] = 77


    # --------------------------------------------------------    
    #   Printing
    # --------------------------------------------------------
    
    print '>>> print cache'
    print cache
    
    print '>>> print cache[[1,2,3]]'
    print cache[[1,2,3]] 
    print ''

    print '>>> print cache[(1,2,3)]'
    print cache[(1,2,3)] 
    print ''
    
    print 'should be True:' , cache.has_key([1,2,3])
    assert cache.has_key([1,2,3])
    
    print 'should be True:' , [1,2,3] in cache
    assert [1,2,3] in cache
    
    del cache[[1,2,3]]
    print 'should be False:' , cache.has_key([1,2,3])
    assert not cache.has_key([1,2,3])
    print ''
    
    
    # --------------------------------------------------------
    #   Pickling test
    # --------------------------------------------------------
    
    print '>>> pickle.dumps()'
    d = pickle.dumps(cache)
    print '>>> pickle.loads()'
    p = pickle.loads(d)
    print ''
    
    print '>>> print p'
    print p    

    print 'should be True:' , [1,2,5] in p    
    assert [1,2,5] in p  
    
    # beware after pickling some objects...
    print 'should be False:' , funny_key in p[[6,2,5]]
    assert not funny_key in p[[6,2,5]]
    print ''
    
    
    # --------------------------------------------------------
    #   Access Speed test
    # --------------------------------------------------------
    print 'Access speed test...'
    
    # accessing bunch
    t0 = time()
    for i in range(int(1e5)):
        v = cache[[6,2,5]][funny_key]
    t1 = time()-t0
    
    # a test dictionary
    z = dict()
    z['t'] = dict()
    z['t']['i'] = 0
    
    # accessing a normal dictionary
    t0 = time()
    for i in range(int(1e5)):
        v = z['t']['i']
    t2 = time()-t0
    
    # results
    print 'HashedDict: %.6f s' % (t1)
    print 'dict:       %.6f s' % (t2)    
    assert (t1-t2)/t2 < 60.0
    print ''
    
    
    # --------------------------------------------------------
    #   Assignment Speed test
    # --------------------------------------------------------
    print 'Assignment speed test...'
    
    # accessing bunch
    t0 = time()
    for i in range(int(1e5)):
        v = cache[[6,2,5]][funny_key] = 10
    t1 = time()-t0
    
    # accessing a normal dictionary
    t0 = time()
    for i in range(int(1e5)):
        z['t']['i'] = 10
    t2 = time()-t0
    
    # results
    print 'HashedDict: %.6f s' % (t1)
    print 'dict:       %.6f s' % (t2)   
    assert (t1-t2)/t2 < 60.0
    print ''
Exemplo n.º 10
0
class Remember(object):
    
    def __init__( self, function, filename='', write_freq=1, name='Remember'):
           
        # store
        self.function   = function
        self.filename   = filename
        self.write_freq = write_freq
        self.name       = name
        
        # check for instance method
        try:
            self._func_self = function.im_self
        except:
            self._func_self = None
        
        # initialize cache from file
        if filename and os.path.exists(filename) and os.path.isfile(filename):
            self.load_cache()
            
        # initialize new cache
        else:
            self.__cache__ = HashedDict()
        
        return
        
    def __func__(self,inputs):
        # evaluate function
        outputs = self.function(inputs)
        return outputs
        
    def __call__(self,inputs):
            
        # hashable type for cache
        _inputs = make_hashable(inputs)
                
        # check cache
        if self.__cache__.has_key(_inputs): 
            #print 'PULLED FROM CACHE'
            outputs = deepcopy( self.__cache__[_inputs] )
        
        # evalute function
        else:
            outputs = self.__func__(inputs)
            self.__cache__[_inputs] = deepcopy(outputs)
        
        #: if cached
        
        # save cache
        if self.filename and ( len(self.__cache__) % self.write_freq ) == 0:
            self.save_cache()
        
        # done
        return outputs
    
    def load_cache(self):
        if not self.filename:
            raise AttributeError , 'no filename for loading cache'
        self.__cache__ = load_data(filename)
    
    def save_cache(self):
        if not self.filename:
            raise AttributeError , 'no filename for saving cache'
        save_data(self.__cache__,self.filename)
    
    def __getstate__(self):
        """ handles an instance method in self.function """
        state = self.__dict__.copy()
        if state['_func_self']:
            state['function']  = state['function'].__name__
        return state
    
    def __setstate__(self,state):
        """ handles an instance method in self.function """
        self.__dict__.update(state)
        if self._func_self:
            self.function  = getattr( self._func_self , self.function)