Exemple #1
0
class EtlRecordSet(object):
    '''Encapsulate a set of records
    
    This object can be used to store multiple records.  If the volume stays
    small (under a configurable limit) the records will be stored in memory.
    If, however, the number of records based on estimated record size, then
    the records will be moved off to disk.
    
    Additionally, indexes can be added for retrieving records by values other
    than the record serial
    '''
    
    def __init__(self, size_until_disk=10000):
        self.max_size_until_disk = size_until_disk
        
        self.__store = MemoryRecordSet()
        self.__on_disk = False
    
    
    def add_record(self, etl_rec, tags=None):
        '''Add a record to the collection
        
        Use indexes to index this 
        
        @param etl_rec: Record to store
        @param tags: List of optional additional tags to be used for retrieving
            this record.  Record must be convertible to a string with str()
        '''
        # Consider switching to disk storage
        if not self.__on_disk:
            if self.__store.size > self.max_size_until_disk:
                self.convert_to_disk_storage()
                
        # Add record
        try:
            self.__store.add_record(etl_rec, tags)
        except Exception, e:
            msg = etl_rec.create_msg("Failed to store record: " + str(e))
            raise Exception(msg)
Exemple #2
0
 def __init__(self, size_until_disk=10000):
     self.max_size_until_disk = size_until_disk
     
     self.__store = MemoryRecordSet()
     self.__on_disk = False