def test__add_to_bulk(self):
retry_queue = Queue()
old_date_modified = datetime.datetime.utcnow().isoformat()
queue_resource_item = {
'doc_type': 'Tender',
'id': uuid.uuid4().hex,
'dateModified': datetime.datetime.utcnow().isoformat()
}
resource_item_doc_dict = {
'doc_type': 'Tender',
'_rev': '1-' + uuid.uuid4().hex,
'id': queue_resource_item['id'],
'dateModified': queue_resource_item['dateModified']
}
resource_item_dict = {
'doc_type': 'Tender',
'id': queue_resource_item['id'],
'dateModified': queue_resource_item['dateModified']
}
worker = ResourceItemWorker(config_dict=self.worker_config,
log_dict=self.log_dict,
retry_resource_items_queue=retry_queue)
worker.db = MagicMock()
# Successfull adding to bulk
start_length = len(worker.bulk)
worker._add_to_bulk(resource_item_dict, queue_resource_item,
resource_item_doc_dict)
end_length = len(worker.bulk)
self.assertGreater(end_length, start_length)
# Update exist doc in bulk
start_length = len(worker.bulk)
new_resource_item_dict = deepcopy(resource_item_dict)
new_resource_item_dict['dateModified'] = datetime.datetime.utcnow(
).isoformat()
worker._add_to_bulk(new_resource_item_dict, queue_resource_item,
resource_item_doc_dict)
end_length = len(worker.bulk)
self.assertEqual(start_length, end_length)
# Ignored dublicate in bulk
start_length = end_length
worker._add_to_bulk(
{
'doc_type': 'Tender',
'id': queue_resource_item['id'],
'_id': queue_resource_item['id'],
'dateModified': old_date_modified
}, queue_resource_item, resource_item_dict)
end_length = len(worker.bulk)
self.assertEqual(start_length, end_length)
del worker
def test__save_bulk_docs(self):
self.worker_config['bulk_save_limit'] = 3
retry_queue = Queue()
worker = ResourceItemWorker(config_dict=self.worker_config,
retry_resource_items_queue=retry_queue)
doc_id_1 = uuid.uuid4().hex
doc_id_2 = uuid.uuid4().hex
doc_id_3 = uuid.uuid4().hex
doc_id_4 = uuid.uuid4().hex
date_modified = datetime.datetime.utcnow().isoformat()
worker.bulk = {
doc_id_1: {'id': doc_id_1, 'dateModified': date_modified},
doc_id_2: {'id': doc_id_2, 'dateModified': date_modified},
doc_id_3: {'id': doc_id_3, 'dateModified': date_modified},
doc_id_4: {'id': doc_id_4, 'dateModified': date_modified}
}
update_return_value = [
(True, doc_id_1, '1-' + uuid.uuid4().hex),
(True, doc_id_2, '2-' + uuid.uuid4().hex),
(False, doc_id_3, Exception(u'New doc with oldest dateModified.')),
(False, doc_id_4, Exception(u'Document update conflict.'))
]
worker.db = MagicMock()
worker.db.update.return_value = update_return_value
# Test success response from couchdb
worker._save_bulk_docs()
sleep(0.1)
self.assertEqual(len(worker.bulk), 0)
self.assertEqual(worker.retry_resource_items_queue.qsize(), 1)
# Test failed response from couchdb
worker.db.update.side_effect = Exception('Some exceptions')
worker.bulk = {
doc_id_1: {'id': doc_id_1, 'dateModified': date_modified},
doc_id_2: {'id': doc_id_2, 'dateModified': date_modified},
doc_id_3: {'id': doc_id_3, 'dateModified': date_modified},
doc_id_4: {'id': doc_id_4, 'dateModified': date_modified}
}
worker._save_bulk_docs()
sleep(0.2)
self.assertEqual(worker.retry_resource_items_queue.qsize(), 5)
self.assertEqual(len(worker.bulk), 0)
worker.config['historical'] = True
worker.bulk = {
doc_id_1: {'id': doc_id_1, 'rev': randint(10, 99)}
}
worker._save_bulk_docs()
sleep(0.2)
self.assertEqual(worker.retry_resource_items_queue.qsize(), 6)
self.assertEqual(len(worker.bulk), 0)
worker.config['historical'] = False
def test__add_to_bulk(self):
retry_queue = PriorityQueue()
old_date_modified = datetime.datetime.utcnow().isoformat()
resource_item_id = uuid.uuid4().hex
priority = 1
local_resource_item = {
'doc_type': 'Tender',
'_rev': '1-' + uuid.uuid4().hex,
'id': resource_item_id,
'dateModified': old_date_modified
}
new_date_modified = datetime.datetime.utcnow().isoformat()
public_resource_item = {
'id': resource_item_id,
'dateModified': new_date_modified
}
worker = ResourceItemWorker(config_dict=self.worker_config,
retry_resource_items_queue=retry_queue)
worker.db = MagicMock()
# Successfull adding to bulk
start_length = len(worker.bulk)
worker._add_to_bulk(local_resource_item, public_resource_item,
priority)
end_length = len(worker.bulk)
self.assertGreater(end_length, start_length)
# Update exist doc in bulk
start_length = len(worker.bulk)
new_public_resource_item = deepcopy(public_resource_item)
new_public_resource_item['dateModified'] =\
datetime.datetime.utcnow().isoformat()
worker._add_to_bulk(local_resource_item, new_public_resource_item,
priority)
end_length = len(worker.bulk)
self.assertEqual(start_length, end_length)
# Ignored dublicate in bulk
start_length = end_length
worker._add_to_bulk(
local_resource_item, {
'doc_type': 'Tender',
'id': local_resource_item['id'],
'_id': local_resource_item['id'],
'dateModified': old_date_modified
}, priority)
end_length = len(worker.bulk)
self.assertEqual(start_length, end_length)
del worker
def test__save_bulk_docs(self):
self.worker_config['bulk_save_limit'] = 3
retry_queue = Queue()
worker = ResourceItemWorker(config_dict=self.worker_config,
log_dict=self.log_dict,
retry_resource_items_queue=retry_queue)
doc_id_1 = uuid.uuid4().hex
doc_id_2 = uuid.uuid4().hex
doc_id_3 = uuid.uuid4().hex
doc_id_4 = uuid.uuid4().hex
date_modified = datetime.datetime.utcnow().isoformat()
worker.bulk = {
doc_id_1: {
'id': doc_id_1,
'dateModified': date_modified
},
doc_id_2: {
'id': doc_id_2,
'dateModified': date_modified
},
doc_id_3: {
'id': doc_id_3,
'dateModified': date_modified
},
doc_id_4: {
'id': doc_id_4,
'dateModified': date_modified
}
}
update_return_value = [
(True, doc_id_1, '1-' + uuid.uuid4().hex),
(True, doc_id_2, '2-' + uuid.uuid4().hex),
(False, doc_id_3, Exception(u'New doc with oldest dateModified.')),
(False, doc_id_4, Exception(u'Document update conflict.'))
]
worker.db = MagicMock()
worker.db.update.return_value = update_return_value
self.assertEqual(worker.log_dict['update_documents'], 0)
self.assertEqual(worker.log_dict['save_documents'], 0)
self.assertEqual(worker.log_dict['skiped'], 0)
self.assertEqual(worker.log_dict['add_to_retry'], 0)
# Test success response from couchdb
worker._save_bulk_docs()
self.assertEqual(worker.log_dict['update_documents'], 1)
self.assertEqual(worker.log_dict['save_documents'], 1)
self.assertEqual(worker.log_dict['skiped'], 1)
self.assertEqual(worker.log_dict['add_to_retry'], 1)
# Test failed response from couchdb
worker.db.update.side_effect = Exception('Some exceptions')
worker.bulk = {
doc_id_1: {
'id': doc_id_1,
'dateModified': date_modified
},
doc_id_2: {
'id': doc_id_2,
'dateModified': date_modified
},
doc_id_3: {
'id': doc_id_3,
'dateModified': date_modified
},
doc_id_4: {
'id': doc_id_4,
'dateModified': date_modified
}
}
worker._save_bulk_docs()
self.assertEqual(worker.log_dict['update_documents'], 1)
self.assertEqual(worker.log_dict['save_documents'], 1)
self.assertEqual(worker.log_dict['skiped'], 1)
self.assertEqual(worker.log_dict['add_to_retry'], 5)