def test__add_to_bulk(self):
retry_queue = Queue()
old_date_modified = datetime.datetime.utcnow().isoformat()
queue_resource_item = {
'doc_type': 'Tender',
'id': uuid.uuid4().hex,
'dateModified': datetime.datetime.utcnow().isoformat()
}
resource_item_doc_dict = {
'doc_type': 'Tender',
'_rev': '1-' + uuid.uuid4().hex,
'id': queue_resource_item['id'],
'dateModified': queue_resource_item['dateModified']
}
resource_item_dict = {
'doc_type': 'Tender',
'id': queue_resource_item['id'],
'dateModified': queue_resource_item['dateModified']
}
worker = ResourceItemWorker(config_dict=self.worker_config,
retry_resource_items_queue=retry_queue)
worker.db = MagicMock()
# Successfull adding to bulk
start_length = len(worker.bulk)
worker._add_to_bulk(resource_item_dict, queue_resource_item,
resource_item_doc_dict)
end_length = len(worker.bulk)
self.assertGreater(end_length, start_length)
# Update exist doc in bulk
start_length = len(worker.bulk)
new_resource_item_dict = deepcopy(resource_item_dict)
new_resource_item_dict['dateModified'] =\
datetime.datetime.utcnow().isoformat()
worker._add_to_bulk(new_resource_item_dict, queue_resource_item,
resource_item_doc_dict)
end_length = len(worker.bulk)
self.assertEqual(start_length, end_length)
# Ignored dublicate in bulk
start_length = end_length
worker._add_to_bulk(
{
'doc_type': 'Tender',
'id': queue_resource_item['id'],
'_id': queue_resource_item['id'],
'dateModified': old_date_modified
}, queue_resource_item, resource_item_dict)
end_length = len(worker.bulk)
self.assertEqual(start_length, end_length)
del worker
def test__save_bulk_docs(self):
self.worker_config['bulk_save_limit'] = 3
retry_queue = Queue()
worker = ResourceItemWorker(config_dict=self.worker_config,
retry_resource_items_queue=retry_queue)
doc_id_1 = uuid.uuid4().hex
doc_id_2 = uuid.uuid4().hex
doc_id_3 = uuid.uuid4().hex
doc_id_4 = uuid.uuid4().hex
date_modified = datetime.datetime.utcnow().isoformat()
worker.bulk = {
doc_id_1: {
'id': doc_id_1,
'dateModified': date_modified
},
doc_id_2: {
'id': doc_id_2,
'dateModified': date_modified
},
doc_id_3: {
'id': doc_id_3,
'dateModified': date_modified
},
doc_id_4: {
'id': doc_id_4,
'dateModified': date_modified
}
}
update_return_value = [
(True, doc_id_1, '1-' + uuid.uuid4().hex),
(True, doc_id_2, '2-' + uuid.uuid4().hex),
(False, doc_id_3, Exception(u'New doc with oldest dateModified.')),
(False, doc_id_4, Exception(u'Document update conflict.'))
]
worker.db = CouchDBStorage(
{'storage': {
'host': '127.0.0.1',
'port': 5984
}}, 'tenders')
worker.db.db.update = MagicMock(return_value=update_return_value)
# Test success response from couchdb
worker._save_bulk_docs()
sleep(0.1)
self.assertEqual(len(worker.bulk), 0)
self.assertEqual(worker.retry_resource_items_queue.qsize(), 1)
# Test failed response from couchdb
worker.db.db.update.side_effect = Exception('Some exceptions')
worker.bulk = {
doc_id_1: {
'id': doc_id_1,
'dateModified': date_modified
},
doc_id_2: {
'id': doc_id_2,
'dateModified': date_modified
},
doc_id_3: {
'id': doc_id_3,
'dateModified': date_modified
},
doc_id_4: {
'id': doc_id_4,
'dateModified': date_modified
}
}
worker._save_bulk_docs()
sleep(0.2)
self.assertEqual(worker.retry_resource_items_queue.qsize(), 5)
self.assertEqual(len(worker.bulk), 0)