Example #1
0
 def __iter__(self):
   records = []
   size = 0
   for record in input_readers.RecordsReader.__iter__(self):
     records.append(record)
     size += len(record)
     if size > self.BATCH_SIZE:
       yield records
       size = 0
       records = []
       gc.collect()
   if records:
     yield records
     records = []
     gc.collect()
Example #2
0
 def __iter__(self):
     records = []
     size = 0
     for record in input_readers.RecordsReader.__iter__(self):
         records.append(record)
         size += len(record)
         if size > self.BATCH_SIZE:
             yield records
             size = 0
             records = []
             gc.collect()
     if records:
         yield records
         records = []
         gc.collect()
    def __iter__(self):

        records = []
        size = 0
        try:
            while True:
                record = next(super(_BatchGCSRecordsReader, self))
                records.append(record)
                size += len(record)
                if size > self.BATCH_SIZE:
                    yield records
                    size = 0
                    records = []
                    gc.collect()
        except StopIteration:
            pass
        if records:
            yield records
            records = []
            gc.collect()
Example #4
0
  def __iter__(self):

    records = []
    size = 0
    try:
      while True:
        record = super(_BatchGCSRecordsReader, self).next()
        records.append(record)
        size += len(record)
        if size > self.BATCH_SIZE:
          yield records
          size = 0
          records = []
          gc.collect()
    except StopIteration:
      pass
    if records:
      yield records
      records = []
      gc.collect()