def iterate_resource(resource=resource, params=params):
more_to_fetch = True
last_batch_ids = set()
count = 0
total_count = mock_api.query_set.count()
while more_to_fetch:
batch = self.get(mock_api.query_set, count, params)
batch_list = [mock_api.serialize(obj) for obj in batch]
logger.info('Received {}-{} of {}'.format(
count, count + self.limit, total_count))
if not batch_list:
more_to_fetch = False
else:
for obj in batch_list:
if obj['id'] not in last_batch_ids:
yield obj
if count < total_count:
last_batch_ids = {obj['id'] for obj in batch_list}
count += self.limit
else:
more_to_fetch = False
self.checkpoint(paginator, batch_list)
def iterate_resource(resource=resource, params=params):
more_to_fetch = True
last_batch_ids = set()
count = 0
total_count = mock_api.query_set.count()
while more_to_fetch:
batch = self.get(mock_api.query_set, count, params)
batch_list = [mock_api.serialize(obj) for obj in batch]
logger.info('Received {}-{} of {}'.format(count, count + self.limit, total_count))
if not batch_list:
more_to_fetch = False
else:
for obj in batch_list:
if obj['id'] not in last_batch_ids:
yield obj
if count < total_count:
last_batch_ids = {obj['id'] for obj in batch_list}
count += self.limit
else:
more_to_fetch = False
self.checkpoint(paginator, batch_list)
def _inner(es_query_set, start, params):
from commcare_export.cli import logger
logger.info("Fetching batch: {}-{}".format(start,
start + self.limit))
return list(es_query_set[start:start + self.limit])
def _inner(es_query_set, start, params):
from commcare_export.cli import logger
logger.info("Fetching batch: {}-{}".format(start, start + self.limit))
return es_query_set[start:start + self.limit]