def iterate_resource(resource=resource, params=params): more_to_fetch = True last_batch_ids = set() count = 0 total_count = mock_api.query_set.count() while more_to_fetch: batch = self.get(mock_api.query_set, count, params) batch_list = [mock_api.serialize(obj) for obj in batch] logger.info('Received {}-{} of {}'.format( count, count + self.limit, total_count)) if not batch_list: more_to_fetch = False else: for obj in batch_list: if obj['id'] not in last_batch_ids: yield obj if count < total_count: last_batch_ids = {obj['id'] for obj in batch_list} count += self.limit else: more_to_fetch = False self.checkpoint(paginator, batch_list)
def iterate_resource(resource=resource, params=params): more_to_fetch = True last_batch_ids = set() count = 0 total_count = mock_api.query_set.count() while more_to_fetch: batch = self.get(mock_api.query_set, count, params) batch_list = [mock_api.serialize(obj) for obj in batch] logger.info('Received {}-{} of {}'.format(count, count + self.limit, total_count)) if not batch_list: more_to_fetch = False else: for obj in batch_list: if obj['id'] not in last_batch_ids: yield obj if count < total_count: last_batch_ids = {obj['id'] for obj in batch_list} count += self.limit else: more_to_fetch = False self.checkpoint(paginator, batch_list)
def _inner(es_query_set, start, params): from commcare_export.cli import logger logger.info("Fetching batch: {}-{}".format(start, start + self.limit)) return list(es_query_set[start:start + self.limit])
def _inner(es_query_set, start, params): from commcare_export.cli import logger logger.info("Fetching batch: {}-{}".format(start, start + self.limit)) return es_query_set[start:start + self.limit]