def _flush_buffer(self): if self.current_row == 0: return for track, column in self._iter_fields(): track.append(column[:self.current_row]) log(" %i " % self.row_counter, False) self.current_row = 0
def iter_rows(self): for buffer in self.iter_buffers(): for row in buffer: self.row_counter += 1 if self.row_counter % self.dot_interval == 0: log(".", False) yield row
def iter_buffers(self): buffer_counter = 0 while True: # determin the part that will be read from disk start = buffer_counter*len(self.buffer)*self.sub.step + self.sub.start if start >= self.sub.stop: break stop = min(start + len(self.buffer)*self.sub.step, self.sub.stop) # read the part slice(start, stop, step) from each track and store # it in the buffer array log(" %i " % start, False) first_size = None for track, column in self._iter_fields(): size = track.read_into(column, slice(start, stop, self.sub.step)) if first_size is None: first_size = size elif first_size != size: raise Error("Not all tracks are of equal length!") # yield the relevant part of the buffer array if size == len(self.buffer): yield self.buffer[:] else: yield self.buffer[:size] break buffer_counter += 1 log(" %i " % stop, False) log.finish()
def dump_row(self, row): #if row.dtype != self.buffer.dtype: # raise Error("The row must have the same dtype as the internal buffer.") self.buffer[self.current_row] = row self.current_row += 1 self.row_counter += 1 if self.row_counter % self.dot_interval == 0: log(".", False) if self.current_row == len(self.buffer): self._flush_buffer()
def __init__(self, filenames, dtype, buffer_size=None, dot_interval=None, clear=True): MultiTrackBase.__init__(self) if buffer_size is None: buffer_size = context.default_buffer_size if dot_interval is None: dot_interval = context.default_dot_interval # make sure the files can be created for filename in filenames: directory = os.path.dirname(filename) if len(directory) > 0 and not os.path.exists(directory): os.makedirs(directory) self.init_buffer(buffer_size, dtype) self.init_tracks(filenames, dtype, clear) # some residual parameters self.current_row = 0 self.dot_interval = dot_interval self.row_counter = 0 log(" 0 ", False)