def test_bulk_handling(self):
self.conf['bulk_process'] = 'true'
with mock.patch('container_crawler.utils.InternalClient',
return_value=self.mock_ic), \
mock.patch('container_crawler.Ring',
return_value=self.mock_ring):
self.crawler = container_crawler.ContainerCrawler(
self.conf, self.mock_handler)
self.assertEqual(self.crawler.bulk, True)
total_rows = 20
items = [{'ROWID': x} for x in range(0, total_rows)]
mock_handler = mock.Mock()
self.crawler.handler_class = mock.Mock()
mock_handler.handle.return_value = []
self.crawler.handler_class.return_value = mock_handler
self.crawler.process_items(mock_handler, items, 1, 0,
self.container_job)
expected = [
mock.call([{
'ROWID': x
} for x in range(0, total_rows)], self.mock_ic)
]
self.assertEqual(expected, mock_handler.handle.call_args_list)
self.assertEqual(0, self.container_job._outstanding)
def test_no_bulk_init(self, mock_ring, mock_ic):
self.mock_ring = mock.Mock()
mock_ring.return_value = self.mock_ring
self.mock_handler = mock.Mock()
self.mock_handler.__name__ = 'MockHandler'
self.mock_ic = mock.Mock()
mock_ic.return_value = self.mock_ic
self.conf = {
'devices': '/devices',
'items_chunk': 1000,
'status_dir': '/var/scratch',
'bulk_process': False,
'workers': 50,
'enumerator_workers': 84
}
crawler = container_crawler.ContainerCrawler(self.conf,
self.mock_handler)
self.assertEqual(False, crawler.bulk)
self.assertEqual(50, crawler._swift_pool.min_size)
self.assertEqual(50, crawler._swift_pool.max_size)
self.assertEqual(50, crawler._swift_pool.current_size)
self.assertEqual(50, crawler.worker_pool.size)
self.assertEqual(100, crawler.work_queue.maxsize)
self.assertEqual(84, crawler.enumerator_pool.size)
self.assertEqual(84, crawler.enumerator_queue.maxsize)
def setUp(self, mock_ring):
self.mock_ring = mock.Mock()
mock_ring.return_value = self.mock_ring
self.conf = {'devices': '/devices',
'items_chunk': 1000,
'status_dir': '/var/scratch'}
self.crawler = container_crawler.ContainerCrawler(self.conf, None)
def test_bulk_errors(self):
self.mock_ring.get_nodes.return_value = ['part', []]
self.conf['bulk_process'] = 'true'
with mock.patch('container_crawler.utils.InternalClient',
return_value=self.mock_ic), \
mock.patch('container_crawler.Ring',
return_value=self.mock_ring):
self.crawler = container_crawler.ContainerCrawler(
self.conf, self.mock_handler)
self.assertEqual(self.crawler.bulk, True)
mock_handler = mock.Mock()
mock_handler.handle.side_effect = RuntimeError('error')
with self.assertRaises(RuntimeError):
self.crawler.process_items(mock_handler, [], 1, 0)
def setUp(self, mock_ring, mock_ic):
self.mock_ring = mock.Mock()
mock_ring.return_value = self.mock_ring
self.mock_handler = mock.Mock()
self.mock_handler.__name__ = 'MockHandler'
self.mock_ic = mock.Mock()
mock_ic.return_value = self.mock_ic
self.conf = {
'devices': '/devices',
'items_chunk': 1000,
'status_dir': '/var/scratch'
}
self.crawler = container_crawler.ContainerCrawler(
self.conf, self.mock_handler)
def test_bulk_init(self, mock_ring, mock_ic):
self.mock_ring = mock.Mock()
mock_ring.return_value = self.mock_ring
self.mock_handler = mock.Mock()
self.mock_handler.__name__ = 'MockHandler'
self.mock_ic = mock.Mock()
mock_ic.return_value = self.mock_ic
self.conf = {
'devices': '/devices',
'items_chunk': 1000,
'status_dir': '/var/scratch',
'bulk_process': True
}
crawler = container_crawler.ContainerCrawler(self.conf,
self.mock_handler)
self.assertEqual(True, crawler.bulk)
self.assertEqual(1, crawler._swift_pool.min_size)
self.assertEqual(1, crawler._swift_pool.max_size)
self.assertEqual(1, crawler._swift_pool.current_size)