def test__save_bulk_docs(self):
self.worker_config['bulk_save_limit'] = 3
retry_queue = Queue()
worker = ResourceItemWorker(config_dict=self.worker_config,
retry_resource_items_queue=retry_queue)
doc_id_1 = uuid.uuid4().hex
doc_id_2 = uuid.uuid4().hex
doc_id_3 = uuid.uuid4().hex
doc_id_4 = uuid.uuid4().hex
date_modified = datetime.datetime.utcnow().isoformat()
worker.bulk = {
doc_id_1: {'id': doc_id_1, 'dateModified': date_modified},
doc_id_2: {'id': doc_id_2, 'dateModified': date_modified},
doc_id_3: {'id': doc_id_3, 'dateModified': date_modified},
doc_id_4: {'id': doc_id_4, 'dateModified': date_modified}
}
update_return_value = [
(True, doc_id_1, '1-' + uuid.uuid4().hex),
(True, doc_id_2, '2-' + uuid.uuid4().hex),
(False, doc_id_3, Exception(u'New doc with oldest dateModified.')),
(False, doc_id_4, Exception(u'Document update conflict.'))
]
worker.db = MagicMock()
worker.db.update.return_value = update_return_value
# Test success response from couchdb
worker._save_bulk_docs()
sleep(0.1)
self.assertEqual(len(worker.bulk), 0)
self.assertEqual(worker.retry_resource_items_queue.qsize(), 1)
# Test failed response from couchdb
worker.db.update.side_effect = Exception('Some exceptions')
worker.bulk = {
doc_id_1: {'id': doc_id_1, 'dateModified': date_modified},
doc_id_2: {'id': doc_id_2, 'dateModified': date_modified},
doc_id_3: {'id': doc_id_3, 'dateModified': date_modified},
doc_id_4: {'id': doc_id_4, 'dateModified': date_modified}
}
worker._save_bulk_docs()
sleep(0.2)
self.assertEqual(worker.retry_resource_items_queue.qsize(), 5)
self.assertEqual(len(worker.bulk), 0)
worker.config['historical'] = True
worker.bulk = {
doc_id_1: {'id': doc_id_1, 'rev': randint(10, 99)}
}
worker._save_bulk_docs()
sleep(0.2)
self.assertEqual(worker.retry_resource_items_queue.qsize(), 6)
self.assertEqual(len(worker.bulk), 0)
worker.config['historical'] = False
def test__save_bulk_docs(self):
self.worker_config['bulk_save_limit'] = 3
retry_queue = Queue()
worker = ResourceItemWorker(config_dict=self.worker_config,
log_dict=self.log_dict,
retry_resource_items_queue=retry_queue)
doc_id_1 = uuid.uuid4().hex
doc_id_2 = uuid.uuid4().hex
doc_id_3 = uuid.uuid4().hex
doc_id_4 = uuid.uuid4().hex
date_modified = datetime.datetime.utcnow().isoformat()
worker.bulk = {
doc_id_1: {
'id': doc_id_1,
'dateModified': date_modified
},
doc_id_2: {
'id': doc_id_2,
'dateModified': date_modified
},
doc_id_3: {
'id': doc_id_3,
'dateModified': date_modified
},
doc_id_4: {
'id': doc_id_4,
'dateModified': date_modified
}
}
update_return_value = [
(True, doc_id_1, '1-' + uuid.uuid4().hex),
(True, doc_id_2, '2-' + uuid.uuid4().hex),
(False, doc_id_3, Exception(u'New doc with oldest dateModified.')),
(False, doc_id_4, Exception(u'Document update conflict.'))
]
worker.db = MagicMock()
worker.db.update.return_value = update_return_value
self.assertEqual(worker.log_dict['update_documents'], 0)
self.assertEqual(worker.log_dict['save_documents'], 0)
self.assertEqual(worker.log_dict['skiped'], 0)
self.assertEqual(worker.log_dict['add_to_retry'], 0)
# Test success response from couchdb
worker._save_bulk_docs()
self.assertEqual(worker.log_dict['update_documents'], 1)
self.assertEqual(worker.log_dict['save_documents'], 1)
self.assertEqual(worker.log_dict['skiped'], 1)
self.assertEqual(worker.log_dict['add_to_retry'], 1)
# Test failed response from couchdb
worker.db.update.side_effect = Exception('Some exceptions')
worker.bulk = {
doc_id_1: {
'id': doc_id_1,
'dateModified': date_modified
},
doc_id_2: {
'id': doc_id_2,
'dateModified': date_modified
},
doc_id_3: {
'id': doc_id_3,
'dateModified': date_modified
},
doc_id_4: {
'id': doc_id_4,
'dateModified': date_modified
}
}
worker._save_bulk_docs()
self.assertEqual(worker.log_dict['update_documents'], 1)
self.assertEqual(worker.log_dict['save_documents'], 1)
self.assertEqual(worker.log_dict['skiped'], 1)
self.assertEqual(worker.log_dict['add_to_retry'], 5)