def test_find_queue(self, fake_add, fake_lock):
fake_source = Mock()
fake_source.id = 'fake-id'
fake_lock.__enter__ = Mock()
fake_lock.__exit__ = Mock()
# test
canceled = Mock()
canceled.is_set.return_value = False
batch = Batch(canceled, None, None, None, None)
batch.queues[fake_source.id] = Mock()
queue = batch.find_queue(fake_source)
# validation
self.assertFalse(fake_add.called)
self.assertEqual(queue, batch.queues[fake_source.id])
def test_find_queue(self, fake_add, fake_lock):
fake_source = Mock()
fake_source.id = 'fake-id'
fake_lock.__enter__ = Mock()
fake_lock.__exit__ = Mock()
# test
canceled = Mock()
canceled.is_set.return_value = False
batch = Batch(canceled, None, None, None, None)
batch.queues[fake_source.id] = Mock()
queue = batch.find_queue(fake_source)
# validation
self.assertFalse(fake_add.called)
self.assertEqual(queue, batch.queues[fake_source.id])
def test_download_nothing(self, fake_dispatch, fake_wait):
primary = Mock()
sources = []
requests = []
# test
canceled = Mock()
canceled.is_set.return_value = False
batch = Batch(canceled, primary, sources, iter(requests), None)
report = batch.download()
# validation
# initial dispatch
self.assertFalse(fake_dispatch.called)
self.assertTrue(isinstance(report, DownloadReport))
self.assertEqual(len(report.downloads), 0)
fake_wait.assert_called_once_with(0)
def test_download_nothing(self, fake_dispatch, fake_wait):
primary = Mock()
sources = []
requests = []
# test
canceled = Mock()
canceled.is_set.return_value = False
batch = Batch(canceled, primary, sources, iter(requests), None)
report = batch.download()
# validation
# initial dispatch
self.assertFalse(fake_dispatch.called)
self.assertTrue(isinstance(report, DownloadReport))
self.assertEqual(len(report.downloads), 0)
fake_wait.assert_called_once_with(0)
def test_find_queue_not_found(self, fake_add, fake_lock):
fake_source = Mock()
fake_source.id = 'fake-id'
fake_lock.__enter__ = Mock()
fake_lock.__exit__ = Mock()
# test
canceled = Mock()
canceled.is_set.return_value = False
batch = Batch(canceled, None, None, None, None)
queue = batch.find_queue(fake_source)
# validation
fake_add.assert_called_with(fake_source)
self.assertEqual(batch._mutex.__enter__.call_count, 1)
self.assertEqual(batch._mutex.__exit__.call_count, 1)
self.assertEqual(queue, fake_add())
def test_find_queue_not_found(self, fake_add, fake_lock):
fake_source = Mock()
fake_source.id = 'fake-id'
fake_lock.__enter__ = Mock()
fake_lock.__exit__ = Mock()
# test
canceled = Mock()
canceled.is_set.return_value = False
batch = Batch(canceled, None, None, None, None)
queue = batch.find_queue(fake_source)
# validation
fake_add.assert_called_with(fake_source)
self.assertEqual(batch._mutex.__enter__.call_count, 1)
self.assertEqual(batch._mutex.__exit__.call_count, 1)
self.assertEqual(queue, fake_add())
def test_download(self, fake_dispatch, fake_wait):
primary = Mock()
sources = [Mock(), Mock()]
requests = [Mock(), Mock(), Mock()]
queue_1 = Mock()
queue_1.downloader = Mock()
queue_1.downloader.event_listener = Mock()
queue_1.downloader.event_listener.total_succeeded = 100
queue_1.downloader.event_listener.total_failed = 3
queue_2 = Mock()
queue_2.downloader = Mock()
queue_2.downloader.event_listener = Mock()
queue_2.downloader.event_listener.total_succeeded = 200
queue_2.downloader.event_listener.total_failed = 10
# test
canceled = Mock()
canceled.is_set.return_value = False
batch = Batch(canceled, primary, sources, iter(requests), None)
batch.queues = {'source-1': queue_1, 'source-2': queue_2} # simulated
report = batch.download()
# validation
# initial dispatch
for request in requests:
request.find_sources.assert_called_with(primary, sources)
calls = fake_dispatch.call_args_list
self.assertEqual(len(calls), len(requests))
for i, request in enumerate(requests):
self.assertEqual(calls[i][0][0], requests[i])
# wait
fake_wait.assert_called_with(len(requests))
# queue shutdown
for queue in batch.queues.values():
queue.put.assert_called_with(None)
queue.halt.assert_called_with()
queue.join.assert_called_with()
# report
self.assertTrue(isinstance(report, DownloadReport))
self.assertEqual(len(report.downloads), 2)
self.assertEqual(report.downloads['source-1'].total_succeeded, 100)
self.assertEqual(report.downloads['source-1'].total_failed, 3)
self.assertEqual(report.downloads['source-2'].total_succeeded, 200)
self.assertEqual(report.downloads['source-2'].total_failed, 10)
def test_download(self, fake_dispatch, fake_wait):
primary = Mock()
sources = [Mock(), Mock()]
requests = [Mock(), Mock(), Mock()]
queue_1 = Mock()
queue_1.downloader = Mock()
queue_1.downloader.event_listener = Mock()
queue_1.downloader.event_listener.total_succeeded = 100
queue_1.downloader.event_listener.total_failed = 3
queue_2 = Mock()
queue_2.downloader = Mock()
queue_2.downloader.event_listener = Mock()
queue_2.downloader.event_listener.total_succeeded = 200
queue_2.downloader.event_listener.total_failed = 10
# test
canceled = Mock()
canceled.is_set.return_value = False
batch = Batch(canceled, primary, sources, iter(requests), None)
batch.queues = {'source-1': queue_1, 'source-2': queue_2} # simulated
report = batch.download()
# validation
# initial dispatch
for request in requests:
request.find_sources.assert_called_with(primary, sources)
calls = fake_dispatch.call_args_list
self.assertEqual(len(calls), len(requests))
for i, request in enumerate(requests):
self.assertEqual(calls[i][0][0], requests[i])
# wait
fake_wait.assert_called_with(len(requests))
# queue shutdown
for queue in batch.queues.values():
queue.put.assert_called_with(None)
queue.halt.assert_called_with()
queue.join.assert_called_with()
# report
self.assertTrue(isinstance(report, DownloadReport))
self.assertEqual(len(report.downloads), 2)
self.assertEqual(report.downloads['source-1'].total_succeeded, 100)
self.assertEqual(report.downloads['source-1'].total_failed, 3)
self.assertEqual(report.downloads['source-2'].total_succeeded, 200)
self.assertEqual(report.downloads['source-2'].total_failed, 10)
def test_dispatch(self, fake_find, fake_item, fake_decrement):
fake_queue = Mock()
fake_request = Mock()
sources = [(Mock(), 'http://')]
fake_request.sources = iter(sources)
fake_find.return_value = fake_queue
# test
canceled = Mock()
canceled.is_set.return_value = False
batch = Batch(canceled, None, None, None, None)
dispatched = batch.dispatch(fake_request)
# validation
fake_find.assert_called_with(sources[0][0])
fake_item.assert_called_with(fake_request, sources[0][1])
fake_queue.put.assert_called_with(fake_item())
self.assertTrue(dispatched)
self.assertFalse(fake_decrement.called)
def test_dispatch_no_remaining_sources(self, fake_decrement, fake_find):
fake_queue = Mock()
fake_request = Mock()
sources = []
fake_request.sources = iter(sources)
fake_find.return_value = fake_queue
# test
canceled = Mock()
canceled.is_set.return_value = False
batch = Batch(canceled, None, None, None, None)
dispatched = batch.dispatch(fake_request)
# validation
fake_decrement.assert_called_once_with()
self.assertFalse(dispatched)
self.assertFalse(fake_queue.put.called)
self.assertFalse(fake_find.called)
def test_dispatch(self, fake_find, fake_item, fake_decrement):
fake_queue = Mock()
fake_request = Mock()
sources = [(Mock(), 'http://')]
fake_request.sources = iter(sources)
fake_find.return_value = fake_queue
# test
canceled = Mock()
canceled.is_set.return_value = False
batch = Batch(canceled, None, None, None, None)
dispatched = batch.dispatch(fake_request)
# validation
fake_find.assert_called_with(sources[0][0])
fake_item.assert_called_with(fake_request, sources[0][1])
fake_queue.put.assert_called_with(fake_item())
self.assertTrue(dispatched)
self.assertFalse(fake_decrement.called)
def test_is_canceled(self):
canceled = Mock()
canceled.is_set.return_value = False
batch = Batch(canceled, None, None, None, None)
# test
self.assertFalse(batch.is_canceled)
canceled.is_set.return_value = True
self.assertTrue(batch.is_canceled)
def test_dispatch_no_remaining_sources(self, fake_decrement, fake_find):
fake_queue = Mock()
fake_request = Mock()
sources = []
fake_request.sources = iter(sources)
fake_find.return_value = fake_queue
# test
canceled = Mock()
canceled.is_set.return_value = False
batch = Batch(canceled, None, None, None, None)
dispatched = batch.dispatch(fake_request)
# validation
fake_decrement.assert_called_once_with()
self.assertFalse(dispatched)
self.assertFalse(fake_queue.put.called)
self.assertFalse(fake_find.called)
def test_add_queue(self, fake_queue, fake_listener):
fake_source = Mock()
fake_source.id = 'fake-id'
fake_queue().downloader = Mock()
# test
canceled = Mock()
canceled.is_set.return_value = False
batch = Batch(canceled, None, None, None, None)
queue = batch._add_queue(fake_source)
# validation
fake_queue.assert_called_with(canceled, fake_source)
fake_listener.assert_called_with(batch)
fake_queue().start.assert_called_with()
self.assertEqual(fake_queue().downloader.event_listener, fake_listener())
self.assertEqual(batch.queues[fake_source.id], fake_queue())
self.assertEqual(queue, fake_queue())
def test_add_queue(self, fake_queue, fake_listener):
fake_source = Mock()
fake_source.id = 'fake-id'
fake_queue().downloader = Mock()
# test
canceled = Mock()
canceled.is_set.return_value = False
batch = Batch(canceled, None, None, None, None)
queue = batch._add_queue(fake_source)
# validation
fake_queue.assert_called_with(canceled, fake_source)
fake_listener.assert_called_with(batch)
fake_queue().start.assert_called_with()
self.assertEqual(fake_queue().downloader.event_listener,
fake_listener())
self.assertEqual(batch.queues[fake_source.id], fake_queue())
self.assertEqual(queue, fake_queue())
def test_download_with_exception(self, fake_dispatch, fake_wait):
primary = Mock()
fake_dispatch.side_effect = ValueError()
sources = [Mock(), Mock()]
requests = [Mock(), Mock(), Mock()]
# test
canceled = Mock()
canceled.is_set.return_value = False
batch = Batch(canceled, primary, sources, iter(requests), None)
batch.queues = {'source-1': Mock(), 'source-2': Mock()} # simulated
self.assertRaises(ValueError, batch.download)
# validation
fake_wait.assert_called_once_with(0)
for queue in batch.queues.values():
queue.put.assert_called_with(None)
queue.halt.assert_called_with()
queue.join.assert_called_with()
def test_download_with_exception(self, fake_dispatch, fake_wait):
primary = Mock()
fake_dispatch.side_effect = ValueError()
sources = [Mock(), Mock()]
requests = [Mock(), Mock(), Mock()]
# test
canceled = Mock()
canceled.is_set.return_value = False
batch = Batch(canceled, primary, sources, iter(requests), None)
batch.queues = {'source-1': Mock(), 'source-2': Mock()} # simulated
self.assertRaises(ValueError, batch.download)
# validation
fake_wait.assert_called_once_with(0)
for queue in batch.queues.values():
queue.put.assert_called_with(None)
queue.halt.assert_called_with()
queue.join.assert_called_with()
def test_download_canceled(self, fake_dispatch, fake_wait):
queue_1 = Mock()
queue_1.downloader = Mock()
queue_1.downloader.event_listener = Mock()
queue_1.downloader.event_listener.total_succeeded = 100
queue_1.downloader.event_listener.total_failed = 3
queue_2 = Mock()
queue_2.downloader = Mock()
queue_2.downloader.event_listener = Mock()
queue_2.downloader.event_listener.total_succeeded = 200
queue_2.downloader.event_listener.total_failed = 10
# test
canceled = Mock()
canceled.is_set.return_value = True
batch = Batch(canceled, None, [], [Mock()], None)
batch.queues = {'source-1': queue_1, 'source-2': queue_2} # simulated
report = batch.download()
# validation
# initial dispatch
self.assertFalse(fake_dispatch.called)
# queue shutdown
for queue in batch.queues.values():
queue.put.assert_called_with(None)
queue.halt.assert_called_with()
queue.join.assert_called_with()
# wait
fake_wait.assert_called_once_with(0)
# report
self.assertTrue(isinstance(report, DownloadReport))
self.assertEqual(len(report.downloads), 2)
self.assertEqual(report.downloads['source-1'].total_succeeded, 100)
self.assertEqual(report.downloads['source-1'].total_failed, 3)
self.assertEqual(report.downloads['source-2'].total_succeeded, 200)
self.assertEqual(report.downloads['source-2'].total_failed, 10)
def test_download_canceled(self, fake_dispatch, fake_wait):
queue_1 = Mock()
queue_1.downloader = Mock()
queue_1.downloader.event_listener = Mock()
queue_1.downloader.event_listener.total_succeeded = 100
queue_1.downloader.event_listener.total_failed = 3
queue_2 = Mock()
queue_2.downloader = Mock()
queue_2.downloader.event_listener = Mock()
queue_2.downloader.event_listener.total_succeeded = 200
queue_2.downloader.event_listener.total_failed = 10
# test
canceled = Mock()
canceled.is_set.return_value = True
batch = Batch(canceled, None, [], [Mock()], None)
batch.queues = {'source-1': queue_1, 'source-2': queue_2} # simulated
report = batch.download()
# validation
# initial dispatch
self.assertFalse(fake_dispatch.called)
# queue shutdown
for queue in batch.queues.values():
queue.put.assert_called_with(None)
queue.halt.assert_called_with()
queue.join.assert_called_with()
# wait
fake_wait.assert_called_once_with(0)
# report
self.assertTrue(isinstance(report, DownloadReport))
self.assertEqual(len(report.downloads), 2)
self.assertEqual(report.downloads['source-1'].total_succeeded, 100)
self.assertEqual(report.downloads['source-1'].total_failed, 3)
self.assertEqual(report.downloads['source-2'].total_succeeded, 200)
self.assertEqual(report.downloads['source-2'].total_failed, 10)
def test_init(self):
primary = Mock()
container = Mock()
requests = Mock()
listener = Mock()
# test
batch = Batch(primary, container, requests, listener)
# validation
self.assertEqual(batch.primary, primary)
self.assertEqual(batch.container, container)
self.assertEqual(batch.requests, requests)
self.assertEqual(batch.listener, listener)
self.assertRaises(NotImplementedError, batch)
def test_construction(self, fake_lock):
primary = Mock()
sources = Mock()
requests = Mock()
listener = Mock()
# test
canceled = Mock()
canceled.is_set.return_value = False
batch = Batch(canceled, primary, sources, requests, listener)
# validation
self.assertEqual(batch._mutex, fake_lock())
self.assertEqual(batch.canceled, canceled)
self.assertEqual(batch.primary, primary)
self.assertEqual(batch.sources, sources)
self.assertEqual(batch.requests, requests)
self.assertEqual(batch.listener, listener)
self.assertEqual(batch.in_progress.canceled, canceled)
self.assertEqual(len(batch.queues), 0)
self.assertTrue(isinstance(batch.in_progress, Tracker))
self.assertTrue(isinstance(batch.queues, dict))