def __iter__(self):
records = []
size = 0
for record in input_readers.RecordsReader.__iter__(self):
records.append(record)
size += len(record)
if size > self.BATCH_SIZE:
yield records
size = 0
records = []
gc.collect()
if records:
yield records
records = []
gc.collect()
def __iter__(self):
records = []
size = 0
for record in input_readers.RecordsReader.__iter__(self):
records.append(record)
size += len(record)
if size > self.BATCH_SIZE:
yield records
size = 0
records = []
gc.collect()
if records:
yield records
records = []
gc.collect()
def __iter__(self):
records = []
size = 0
try:
while True:
record = next(super(_BatchGCSRecordsReader, self))
records.append(record)
size += len(record)
if size > self.BATCH_SIZE:
yield records
size = 0
records = []
gc.collect()
except StopIteration:
pass
if records:
yield records
records = []
gc.collect()
def __iter__(self):
records = []
size = 0
try:
while True:
record = super(_BatchGCSRecordsReader, self).next()
records.append(record)
size += len(record)
if size > self.BATCH_SIZE:
yield records
size = 0
records = []
gc.collect()
except StopIteration:
pass
if records:
yield records
records = []
gc.collect()