def _flush_buffer(self):
if self.current_row == 0:
return
for track, column in self._iter_fields():
track.append(column[:self.current_row])
log(" %i " % self.row_counter, False)
self.current_row = 0
def iter_rows(self):
for buffer in self.iter_buffers():
for row in buffer:
self.row_counter += 1
if self.row_counter % self.dot_interval == 0:
log(".", False)
yield row
def iter_buffers(self):
buffer_counter = 0
while True:
# determin the part that will be read from disk
start = buffer_counter*len(self.buffer)*self.sub.step + self.sub.start
if start >= self.sub.stop:
break
stop = min(start + len(self.buffer)*self.sub.step, self.sub.stop)
# read the part slice(start, stop, step) from each track and store
# it in the buffer array
log(" %i " % start, False)
first_size = None
for track, column in self._iter_fields():
size = track.read_into(column, slice(start, stop, self.sub.step))
if first_size is None:
first_size = size
elif first_size != size:
raise Error("Not all tracks are of equal length!")
# yield the relevant part of the buffer array
if size == len(self.buffer):
yield self.buffer[:]
else:
yield self.buffer[:size]
break
buffer_counter += 1
log(" %i " % stop, False)
log.finish()
def dump_row(self, row):
#if row.dtype != self.buffer.dtype:
# raise Error("The row must have the same dtype as the internal buffer.")
self.buffer[self.current_row] = row
self.current_row += 1
self.row_counter += 1
if self.row_counter % self.dot_interval == 0:
log(".", False)
if self.current_row == len(self.buffer):
self._flush_buffer()
def __init__(self, filenames, dtype, buffer_size=None, dot_interval=None, clear=True):
MultiTrackBase.__init__(self)
if buffer_size is None:
buffer_size = context.default_buffer_size
if dot_interval is None:
dot_interval = context.default_dot_interval
# make sure the files can be created
for filename in filenames:
directory = os.path.dirname(filename)
if len(directory) > 0 and not os.path.exists(directory):
os.makedirs(directory)
self.init_buffer(buffer_size, dtype)
self.init_tracks(filenames, dtype, clear)
# some residual parameters
self.current_row = 0
self.dot_interval = dot_interval
self.row_counter = 0
log(" 0 ", False)
