class EtlRecordSet(object):
'''Encapsulate a set of records
This object can be used to store multiple records. If the volume stays
small (under a configurable limit) the records will be stored in memory.
If, however, the number of records based on estimated record size, then
the records will be moved off to disk.
Additionally, indexes can be added for retrieving records by values other
than the record serial
'''
def __init__(self, size_until_disk=10000):
self.max_size_until_disk = size_until_disk
self.__store = MemoryRecordSet()
self.__on_disk = False
def add_record(self, etl_rec, tags=None):
'''Add a record to the collection
Use indexes to index this
@param etl_rec: Record to store
@param tags: List of optional additional tags to be used for retrieving
this record. Record must be convertible to a string with str()
'''
# Consider switching to disk storage
if not self.__on_disk:
if self.__store.size > self.max_size_until_disk:
self.convert_to_disk_storage()
# Add record
try:
self.__store.add_record(etl_rec, tags)
except Exception, e:
msg = etl_rec.create_msg("Failed to store record: " + str(e))
raise Exception(msg)
def __init__(self, size_until_disk=10000):
self.max_size_until_disk = size_until_disk
self.__store = MemoryRecordSet()
self.__on_disk = False