示例#1
0
    def test_find_queue(self, fake_add, fake_lock):
        fake_source = Mock()
        fake_source.id = 'fake-id'
        fake_lock.__enter__ = Mock()
        fake_lock.__exit__ = Mock()

        # test
        batch = Threaded(None, None, None, None)
        batch.queues[fake_source.id] = Mock()
        queue = batch.find_queue(fake_source)

        # validation
        self.assertFalse(fake_add.called)
        self.assertEqual(queue, batch.queues[fake_source.id])
示例#2
0
    def test_find_queue(self, fake_add, fake_lock):
        fake_source = Mock()
        fake_source.id = 'fake-id'
        fake_lock.__enter__ = Mock()
        fake_lock.__exit__ = Mock()

        # test
        batch = Threaded(None, None, None, None)
        batch.queues[fake_source.id] = Mock()
        queue = batch.find_queue(fake_source)

        # validation
        self.assertFalse(fake_add.called)
        self.assertEqual(queue, batch.queues[fake_source.id])
示例#3
0
    def test_find_queue_not_found(self, fake_add, fake_lock):
        fake_source = Mock()
        fake_source.id = 'fake-id'
        fake_lock.__enter__ = Mock()
        fake_lock.__exit__ = Mock()

        # test
        batch = Threaded(None, None, None, None)
        queue = batch.find_queue(fake_source)

        # validation
        fake_add.assert_called_with(fake_source)
        self.assertEqual(batch._mutex.__enter__.call_count, 1)
        self.assertEqual(batch._mutex.__exit__.call_count, 1)
        self.assertEqual(queue, fake_add())
示例#4
0
    def test_dispatch_no_remaining_sources(self, fake_decrement, fake_find):
        fake_queue = Mock()
        fake_request = Mock()
        fake_request.sources = iter([])
        fake_find.return_value = fake_queue

        # test
        batch = Threaded(None, None, None, None)
        dispatched = batch.dispatch(fake_request)

        # validation
        fake_decrement.assert_called_once_with()
        self.assertFalse(dispatched)
        self.assertFalse(fake_queue.put.called)
        self.assertFalse(fake_find.called)
示例#5
0
    def test_find_queue_not_found(self, fake_add, fake_lock):
        fake_source = Mock()
        fake_source.id = 'fake-id'
        fake_lock.__enter__ = Mock()
        fake_lock.__exit__ = Mock()

        # test
        batch = Threaded(None, None, None, None)
        queue = batch.find_queue(fake_source)

        # validation
        fake_add.assert_called_with(fake_source)
        self.assertEqual(batch._mutex.__enter__.call_count, 1)
        self.assertEqual(batch._mutex.__exit__.call_count, 1)
        self.assertEqual(queue, fake_add())
示例#6
0
    def test_dispatch_no_remaining_sources(self, fake_decrement, fake_find):
        fake_queue = Mock()
        fake_request = Mock()
        fake_request.sources = iter([])
        fake_find.return_value = fake_queue

        # test
        batch = Threaded(None, None, None, None)
        dispatched = batch.dispatch(fake_request)

        # validation
        fake_decrement.assert_called_once_with()
        self.assertFalse(dispatched)
        self.assertFalse(fake_queue.put.called)
        self.assertFalse(fake_find.called)
示例#7
0
    def test_dispatch(self, fake_find, fake_item, fake_decrement):
        fake_queue = Mock()
        fake_request = Mock()
        sources = [(Mock(), 'http://')]
        fake_request.sources = iter(sources)
        fake_find.return_value = fake_queue
        # test
        batch = Threaded(None, None, None, None)
        dispatched = batch.dispatch(fake_request)

        # validation
        fake_find.assert_called_with(sources[0][0])
        fake_item.assert_called_with(fake_request, sources[0][1])
        fake_queue.put.assert_called_with(fake_item())
        self.assertTrue(dispatched)
        self.assertFalse(fake_decrement.called)
示例#8
0
    def test_add_queue(self, fake_queue, fake_listener):
        fake_source = Mock()
        fake_source.id = 'fake-id'
        fake_queue().downloader = Mock()

        # test
        batch = Threaded(None, None, None, None)
        queue = batch._add_queue(fake_source)

        # validation
        fake_queue.assert_called_with(fake_source)
        fake_listener.assert_called_with(batch)
        fake_queue().start.assert_called_with()
        self.assertEqual(fake_queue().downloader.event_listener, fake_listener())
        self.assertEqual(batch.queues[fake_source.id], fake_queue())
        self.assertEqual(queue, fake_queue())
示例#9
0
    def test_dispatch(self, fake_find, fake_item, fake_decrement):
        fake_queue = Mock()
        fake_request = Mock()
        sources = [(Mock(), 'http://')]
        fake_request.sources = iter(sources)
        fake_find.return_value = fake_queue
        # test
        batch = Threaded(None, None, None, None)
        dispatched = batch.dispatch(fake_request)

        # validation
        fake_find.assert_called_with(sources[0][0])
        fake_item.assert_called_with(fake_request, sources[0][1])
        fake_queue.put.assert_called_with(fake_item())
        self.assertTrue(dispatched)
        self.assertFalse(fake_decrement.called)
示例#10
0
    def test_add_queue(self, fake_queue, fake_listener):
        fake_source = Mock()
        fake_source.id = "fake-id"
        fake_queue().downloader = Mock()

        # test
        batch = Threaded(None, None, None, None)
        queue = batch._add_queue(fake_source)

        # validation
        fake_queue.assert_called_with(fake_source)
        fake_listener.assert_called_with(batch)
        fake_queue().start.assert_called_with()
        self.assertEqual(fake_queue().downloader.event_listener, fake_listener())
        self.assertEqual(batch.queues[fake_source.id], fake_queue())
        self.assertEqual(queue, fake_queue())
示例#11
0
    def test_download_with_exception(self, fake_dispatch, fake_wait):
        primary = Mock()
        fake_dispatch.side_effect = ValueError()
        sources = [Mock(), Mock()]
        container = Mock(sources=sources)
        requests = [Mock(), Mock(), Mock()]

        # test
        batch = Threaded(primary, container, iter(requests), None)
        batch.queues = {'source-1': Mock(), 'source-2': Mock()}  # simulated
        self.assertRaises(ValueError, batch)

        # validation
        fake_wait.assert_called_once_with(0)
        for queue in batch.queues.values():
            queue.put.assert_called_with(None)
            queue.halt.assert_called_with()
            queue.join.assert_called_with()
示例#12
0
    def test_download_with_exception(self, fake_dispatch, fake_wait):
        primary = Mock()
        fake_dispatch.side_effect = ValueError()
        sources = [Mock(), Mock()]
        container = Mock(sources=sources)
        requests = [Mock(), Mock(), Mock()]

        # test
        batch = Threaded(primary, container, iter(requests), None)
        batch.queues = {'source-1': Mock(), 'source-2': Mock()}  # simulated
        self.assertRaises(ValueError, batch)

        # validation
        fake_wait.assert_called_once_with(0)
        for queue in batch.queues.values():
            queue.put.assert_called_with(None)
            queue.halt.assert_called_with()
            queue.join.assert_called_with()
示例#13
0
    def test_download(self, fake_dispatch, fake_wait):
        primary = Mock()
        sources = [Mock(), Mock()]
        container = Mock(sources=sources)
        requests = [Mock(), Mock(), Mock()]

        queue_1 = Mock()
        queue_1.downloader = Mock()
        queue_1.downloader.event_listener = Mock()
        queue_1.downloader.event_listener.total_succeeded = 100
        queue_1.downloader.event_listener.total_failed = 3
        queue_2 = Mock()
        queue_2.downloader = Mock()
        queue_2.downloader.event_listener = Mock()
        queue_2.downloader.event_listener.total_succeeded = 200
        queue_2.downloader.event_listener.total_failed = 10

        # test
        batch = Threaded(primary, container, iter(requests), None)
        batch.queues = {'source-1': queue_1, 'source-2': queue_2}  # simulated
        report = batch()

        # validation
        # initial dispatch
        for request in requests:
            request.find_sources.assert_called_with(primary, sources)
        calls = fake_dispatch.call_args_list
        self.assertEqual(len(calls), len(requests))
        for i, request in enumerate(requests):
            self.assertEqual(calls[i][0][0], requests[i])
        # wait
        fake_wait.assert_called_with(len(requests))
        # queue shutdown
        for queue in batch.queues.values():
            queue.put.assert_called_with(None)
            queue.halt.assert_called_with()
            queue.join.assert_called_with()
        # report
        self.assertTrue(isinstance(report, DownloadReport))
        self.assertEqual(len(report.downloads), 2)
        self.assertEqual(report.downloads['source-1'].total_succeeded, 100)
        self.assertEqual(report.downloads['source-1'].total_failed, 3)
        self.assertEqual(report.downloads['source-2'].total_succeeded, 200)
        self.assertEqual(report.downloads['source-2'].total_failed, 10)
示例#14
0
    def test_download(self, fake_dispatch, fake_wait):
        primary = Mock()
        sources = [Mock(), Mock()]
        container = Mock(sources=sources)
        requests = [Mock(), Mock(), Mock()]

        queue_1 = Mock()
        queue_1.downloader = Mock()
        queue_1.downloader.event_listener = Mock()
        queue_1.downloader.event_listener.total_succeeded = 100
        queue_1.downloader.event_listener.total_failed = 3
        queue_2 = Mock()
        queue_2.downloader = Mock()
        queue_2.downloader.event_listener = Mock()
        queue_2.downloader.event_listener.total_succeeded = 200
        queue_2.downloader.event_listener.total_failed = 10

        # test
        batch = Threaded(primary, container, iter(requests), None)
        batch.queues = {'source-1': queue_1, 'source-2': queue_2}  # simulated
        report = batch()

        # validation
        # initial dispatch
        for request in requests:
            request.find_sources.assert_called_with(primary, sources)
        calls = fake_dispatch.call_args_list
        self.assertEqual(len(calls), len(requests))
        for i, request in enumerate(requests):
            self.assertEqual(calls[i][0][0], requests[i])
        # wait
        fake_wait.assert_called_with(len(requests))
        # queue shutdown
        for queue in batch.queues.values():
            queue.put.assert_called_with(None)
            queue.halt.assert_called_with()
            queue.join.assert_called_with()
        # report
        self.assertTrue(isinstance(report, DownloadReport))
        self.assertEqual(len(report.downloads), 2)
        self.assertEqual(report.downloads['source-1'].total_succeeded, 100)
        self.assertEqual(report.downloads['source-1'].total_failed, 3)
        self.assertEqual(report.downloads['source-2'].total_succeeded, 200)
        self.assertEqual(report.downloads['source-2'].total_failed, 10)
示例#15
0
    def test_download_nothing(self, fake_dispatch, fake_wait):
        primary = Mock()
        container = Mock(sources=[])
        requests = []

        # test
        canceled = Mock()
        canceled.is_set.return_value = False
        batch = Threaded(primary, container, iter(requests), None)
        report = batch()

        # validation
        # initial dispatch
        self.assertFalse(fake_dispatch.called)
        self.assertTrue(isinstance(report, DownloadReport))
        self.assertEqual(len(report.downloads), 0)
        fake_wait.assert_called_once_with(0)
示例#16
0
    def test_init(self, fake_lock):
        primary = Mock()
        container = Mock()
        requests = Mock()
        listener = Mock()

        # test
        batch = Threaded(primary, container, requests, listener)

        # validation
        self.assertEqual(batch._mutex, fake_lock())
        self.assertEqual(batch.primary, primary)
        self.assertEqual(batch.container, container)
        self.assertEqual(batch.requests, requests)
        self.assertEqual(batch.listener, listener)
        self.assertEqual(len(batch.queues), 0)
        self.assertTrue(isinstance(batch.in_progress, Tracker))
        self.assertTrue(isinstance(batch.queues, dict))