class BaseCouchDocProcessorTest(SimpleTestCase): processor_class = None @staticmethod def _get_row(ident, doc_type="Bar"): doc_id_prefix = '{}-'.format(doc_type.lower()) doc_id = '{}{}'.format(doc_id_prefix, ident) return { 'id': doc_id, 'key': [doc_type, doc_id], 'value': None, 'doc': {'_id': doc_id, 'doc_type': doc_type} } def _get_view_results(self, total, chuck_size, doc_type="Bar"): doc_id_prefix = '{}-'.format(doc_type.lower()) results = [( {"endkey": [doc_type, {}], "group_level": 1, "reduce": True, "startkey": [doc_type]}, [{"key": doc_type, "value": total}] )] for chunk in chunked(list(range(total)), chuck_size): chunk_rows = [self._get_row(ident, doc_type=doc_type) for ident in chunk] if chunk[0] == 0: results.append(( { 'startkey': [doc_type], 'endkey': [doc_type, {}], 'reduce': False, 'limit': chuck_size, 'include_docs': True }, chunk_rows )) else: previous = '{}{}'.format(doc_id_prefix, chunk[0] - 1) results.append(( { 'endkey': [doc_type, {}], 'skip': 1, 'startkey_docid': previous, 'reduce': False, 'startkey': [doc_type, previous], 'limit': chuck_size, 'include_docs': True }, chunk_rows )) return results def setUp(self): views = { "all_docs/by_doc_type": self._get_view_results(4, chuck_size=2) } docs = [self._get_row(ident)['doc'] for ident in range(4)] self.db = FakeCouchDb(views=views, docs={ doc['_id']: doc for doc in docs }) Bar.set_db(self.db) self.processor_slug = uuid.uuid4().hex def tearDown(self): self.db.reset() def _get_processor(self, chunk_size=2, ignore_docs=None, skip_docs=None, reset=False, doc_types=None): doc_types = doc_types or [Bar] doc_processor = DemoProcessor() doc_provider = CouchDocumentProvider(self.processor_slug, doc_types) processor = self.processor_class( doc_provider, doc_processor, chunk_size=chunk_size, reset=reset ) processor.document_iterator.couch_db = self.db if ignore_docs: doc_processor.ignore_docs = ignore_docs if skip_docs: doc_processor.skip_docs = skip_docs return doc_processor, processor
class TestResumableFunctionIterator(SimpleTestCase): def setUp(self): self.couch_db = FakeCouchDb() self.batches = [ list(range(0, 3)), list(range(3, 6)), list(range(6, 8)), ] self.all_items = list(itertools.chain(*self.batches)) self.itr = self.get_iterator() def tearDown(self): self.couch_db.reset() def get_iterator(self, missing_items=None): def data_provider(batch_number): try: return self.batches[batch_number] except IndexError: return [] itr = ResumableFunctionIterator('test', data_provider, TestArgsProvider()) itr.couch_db = self.couch_db return itr def test_iteration(self): self.assertEqual(list(self.itr), self.all_items) def test_resume_iteration(self): itr = iter(self.itr) self.assertEqual([next(itr) for i in range(6)], self.all_items[:6]) # stop/resume iteration self.itr = self.get_iterator() self.assertEqual([item for item in self.itr], self.all_items[3:]) def test_resume_iteration_after_exhaustion(self): itr = iter(self.itr) self.assertEqual(list(itr), self.all_items) # resume iteration self.batches.append([8, 9]) self.itr = self.get_iterator() self.assertEqual(list(self.itr), [8, 9]) def test_resume_iteration_after_legacy_completion(self): itr = iter(self.itr) self.assertEqual(list(itr), self.all_items) state = self.itr.state state.complete = True state.args = state.kwargs = None self.itr._save_state() # attempt to resume yields no new items self.batches.append([8, 9]) self.itr = self.get_iterator() self.assertEqual(list(self.itr), []) def test_resume_iteration_after_complete_iteration(self): self.assertEqual(list(self.itr), self.all_items) # resume iteration self.itr = self.get_iterator() self.assertEqual(list(self.itr), []) def test_discard_state(self): self.assertEqual(list(self.itr), self.all_items) self.itr.discard_state() self.itr = self.get_iterator() self.assertEqual(list(self.itr), self.all_items) def test_iteration_with_iterator_detail(self): itr = iter(self.itr) self.assertEqual([next(itr) for i in range(6)], self.all_items[:6]) self.assertEqual(self.itr.get_iterator_detail('progress'), None) self.itr.set_iterator_detail('progress', {"visited": 6}) # stop/resume iteration self.itr = self.get_iterator() self.assertEqual(self.itr.get_iterator_detail('progress'), {"visited": 6}) self.itr.set_iterator_detail('progress', {"visited": "six"}) # stop/resume iteration self.itr = self.get_iterator() self.assertEqual(self.itr.get_iterator_detail('progress'), {"visited": "six"}) self.assertEqual([item for item in self.itr], self.all_items[3:])
class TestResumableFunctionIterator(SimpleTestCase): def setUp(self): self.couch_db = FakeCouchDb() self.batches = [ list(range(0, 3)), list(range(3, 6)), list(range(6, 8)), ] self.all_items = list(itertools.chain(*self.batches)) self.itr = self.get_iterator() def tearDown(self): self.couch_db.reset() def get_iterator(self, missing_items=None): def data_provider(batch_number): try: return self.batches[batch_number] except IndexError: return [] def item_getter(item_id): if missing_items and item_id in missing_items: return None return int(item_id) itr = ResumableFunctionIterator('test', data_provider, TestArgsProvider(), item_getter) itr.couch_db = self.couch_db return itr def test_iteration(self): self.assertEqual(list(self.itr), self.all_items) def test_resume_iteration(self): itr = iter(self.itr) self.assertEqual([next(itr) for i in range(6)], self.all_items[:6]) # stop/resume iteration self.itr = self.get_iterator() self.assertEqual([item for item in self.itr], self.all_items[3:]) def test_resume_iteration_after_complete_iteration(self): self.assertEqual(list(self.itr), self.all_items) # resume iteration self.itr = self.get_iterator() self.assertEqual(list(self.itr), []) def test_discard_state(self): self.assertEqual(list(self.itr), self.all_items) self.itr.discard_state() self.itr = self.get_iterator() self.assertEqual(list(self.itr), self.all_items) def test_iteration_with_iterator_detail(self): itr = iter(self.itr) self.assertEqual([next(itr) for i in range(6)], self.all_items[:6]) self.assertEqual(self.itr.get_iterator_detail('progress'), None) self.itr.set_iterator_detail('progress', {"visited": 6}) # stop/resume iteration self.itr = self.get_iterator() self.assertEqual(self.itr.get_iterator_detail('progress'), {"visited": 6}) self.itr.set_iterator_detail('progress', {"visited": "six"}) # stop/resume iteration self.itr = self.get_iterator() self.assertEqual(self.itr.get_iterator_detail('progress'), {"visited": "six"}) self.assertEqual([item for item in self.itr], self.all_items[3:]) def test_iteration_with_retry(self): itr = iter(self.itr) item = next(itr) self.itr.retry(str(item)) self.assertEqual(item, 0) self.assertEqual([0] + [d for d in itr], self.all_items + [0]) def test_iteration_complete_after_retry(self): itr = iter(self.itr) self.itr.retry(str(next(itr))) list(itr) self.itr = self.get_iterator() self.assertEqual([item for item in self.itr], []) def test_iteration_with_max_retry(self): itr = iter(self.itr) item = next(itr) ids = [item] self.assertEqual(item, 0) self.itr.retry(str(item)) retries = 1 for item in itr: ids.append(item) if item == 0: if retries < 3: self.itr.retry(str(item)) retries += 1 else: break self.assertEqual(item, 0) with self.assertRaises(TooManyRetries): self.itr.retry(str(item)) self.assertEqual(ids, self.all_items + [0, 0, 0]) self.assertEqual(list(itr), []) self.assertEqual(list(self.get_iterator()), []) def test_iteration_with_missing_retry_doc(self): iterator = self.get_iterator(missing_items=["0"]) itr = iter(iterator) item = next(itr) self.assertEqual(item, 0) iterator.retry(str(item)) self.assertEqual([0] + [d for d in itr], self.all_items)
class TestResumableFunctionIterator(SimpleTestCase): def setUp(self): self.couch_db = FakeCouchDb() self.batches = [ list(range(0, 3)), list(range(3, 6)), list(range(6, 8)), ] self.all_items = list(itertools.chain(*self.batches)) self.itr = self.get_iterator() def tearDown(self): self.couch_db.reset() def get_iterator(self, missing_items=None): def data_provider(batch_number): try: return self.batches[batch_number] except IndexError: return [] def item_getter(item_id): if missing_items and item_id in missing_items: return None return int(item_id) itr = ResumableFunctionIterator('test', data_provider, TestArgsProvider(), item_getter) itr.couch_db = self.couch_db return itr def test_iteration(self): self.assertEqual(list(self.itr), self.all_items) def test_resume_iteration(self): itr = iter(self.itr) self.assertEqual([next(itr) for i in range(6)], self.all_items[:6]) # stop/resume iteration self.itr = self.get_iterator() self.assertEqual([item for item in self.itr], self.all_items[3:]) def test_resume_iteration_after_complete_iteration(self): self.assertEqual(list(self.itr), self.all_items) # resume iteration self.itr = self.get_iterator() self.assertEqual(list(self.itr), []) def test_discard_state(self): self.assertEqual(list(self.itr), self.all_items) self.itr.discard_state() self.itr = self.get_iterator() self.assertEqual(list(self.itr), self.all_items) def test_iteration_with_iterator_detail(self): itr = iter(self.itr) self.assertEqual([next(itr) for i in range(6)], self.all_items[:6]) self.assertEqual(self.itr.get_iterator_detail('progress'), None) self.itr.set_iterator_detail('progress', {"visited": 6}) # stop/resume iteration self.itr = self.get_iterator() self.assertEqual(self.itr.get_iterator_detail('progress'), {"visited": 6}) self.itr.set_iterator_detail('progress', {"visited": "six"}) # stop/resume iteration self.itr = self.get_iterator() self.assertEqual(self.itr.get_iterator_detail('progress'), {"visited": "six"}) self.assertEqual([item for item in self.itr], self.all_items[3:]) def test_iteration_with_retry(self): itr = iter(self.itr) item = next(itr) self.itr.retry(str(item)) self.assertEqual(item, 0) self.assertEqual([0] + [d for d in itr], self.all_items + [0]) def test_iteration_complete_after_retry(self): itr = iter(self.itr) self.itr.retry(str(next(itr))) list(itr) self.itr = self.get_iterator() self.assertEqual([item for item in self.itr], []) def test_iteration_with_max_retry(self): itr = iter(self.itr) item = next(itr) ids = [item] self.assertEqual(item, 0) self.itr.retry(str(item)) retries = 1 for item in itr: ids.append(item) if item == 0: if retries < 3: self.itr.retry(str(item)) retries += 1 else: break self.assertEqual(item, 0) with self.assertRaises(TooManyRetries): self.itr.retry(str(item)) self.assertEqual(ids, self.all_items + [0, 0, 0]) self.assertEqual(list(itr), []) self.assertEqual(list(self.get_iterator()), []) def test_iteration_with_missing_retry_doc(self): iterator = self.get_iterator(missing_items=["0"]) itr = iter(iterator) item = next(itr) self.assertEqual(item, 0) iterator.retry(str(item)) self.assertEqual([0] + [d for d in itr], self.all_items)