def test_download(self, fake_load): sources = [] for n in range(3): s = ContentSource('s-%d' % n, {}) s.get_downloader = Mock() sources.append(s) fake_load.return_value = sources request_list = [] for n in range(6): r = Request('T', {}, 'url-%d' % n, 'path-%d' % n) r.find_sources = Mock(return_value=sources[n % 3:]) request_list.append(r) collated = [ { sources[0]: ['nectar-1'], sources[1]: ['nectar-2', 'nectar-3', 'nectar-4'], sources[2]: ['nectar-5', 'nectar-6'] }, {} ] fake_collated = Mock(side_effect=collated) fake_listener = Mock() canceled = FakeEvent() fake_primary = PrimarySource(Mock()) # test container = ContentContainer('') container.refresh = Mock() container.collated = fake_collated report = container.download(canceled, fake_primary, request_list, fake_listener) # validation container.refresh.assert_called_with(canceled) for r in request_list: r.find_sources.assert_called_with(fake_primary, container.sources) self.assertEqual(report.total_passes, 1) self.assertEqual(report.total_sources, len(sources)) self.assertEqual(len(report.downloads), 3) for source in sources: self.assertEqual(report.downloads[source.id].total_succeeded, 0) self.assertEqual(report.downloads[source.id].total_failed, 0) for source in sources: source.get_downloader.assert_called_with() downloader = source.get_downloader() listener = downloader.event_listener self.assertEqual(listener.cancel_event, canceled) self.assertEqual(listener.downloader, downloader) self.assertEqual(listener.listener, fake_listener) downloader.download.assert_called_with(collated[0][source])
def test_download(self, fake_load): sources = [] for n in range(3): s = ContentSource('s-%d' % n, {}) s.get_downloader = Mock() sources.append(s) fake_load.return_value = sources request_list = [] for n in range(6): r = Request('T', {}, 'url-%d' % n, 'path-%d' % n) r.find_sources = Mock(return_value=sources[n % 3:]) request_list.append(r) collated = [{ sources[0]: ['nectar-1'], sources[1]: ['nectar-2', 'nectar-3', 'nectar-4'], sources[2]: ['nectar-5', 'nectar-6'] }, {}] fake_collated = Mock(side_effect=collated) fake_listener = Mock() canceled = FakeEvent() fake_primary = PrimarySource(Mock()) # test container = ContentContainer('') container.refresh = Mock() container.collated = fake_collated report = container.download(canceled, fake_primary, request_list, fake_listener) # validation container.refresh.assert_called_with(canceled) for r in request_list: r.find_sources.assert_called_with(fake_primary, container.sources) self.assertEqual(report.total_passes, 1) self.assertEqual(report.total_sources, len(sources)) self.assertEqual(len(report.downloads), 3) for source in sources: self.assertEqual(report.downloads[source.id].total_succeeded, 0) self.assertEqual(report.downloads[source.id].total_failed, 0) for source in sources: source.get_downloader.assert_called_with() downloader = source.get_downloader() listener = downloader.event_listener self.assertEqual(listener.cancel_event, canceled) self.assertEqual(listener.downloader, downloader) self.assertEqual(listener.listener, fake_listener) downloader.download.assert_called_with(collated[0][source])
def test_collated_all_downloaded(self): request_list = [] for n in range(10): r = Mock() r.downloaded = True request_list.append(r) # test collated = ContentContainer.collated(request_list) # validation self.assertEqual(len(collated), 0)
def test_download_canceled_after_collated(self, fake_load): sources = [] for n in range(3): s = ContentSource('s-%d' % n, {}) s.get_downloader = Mock() sources.append(s) fake_load.return_value = sources request_list = [] for n in range(6): r = Request('T', {}, 'url-%d' % n, 'path-%d' % n) r.find_sources = Mock(return_value=sources[n % 3:]) request_list.append(r) collated = [ { sources[0]: ['nectar-1'], sources[1]: ['nectar-2', 'nectar-3', 'nectar-4'], sources[2]: ['nectar-5', 'nectar-6'] }, {} ] fake_collated = Mock(side_effect=collated) fake_listener = Mock() canceled = Mock() canceled.isSet.side_effect = [False, True, True] fake_primary = PrimarySource(Mock()) # test container = ContentContainer('') container.refresh = Mock() container.collated = fake_collated report = container.download(canceled, fake_primary, request_list, fake_listener) # validation container.refresh.assert_called_with(canceled) for r in request_list: r.find_sources.assert_called_with(fake_primary, container.sources) called = 0 for s in sources: if s.get_downloader.called: called += 1 self.assertEqual(called, 1) self.assertEqual(report.total_passes, 1) self.assertEqual(report.total_sources, len(sources)) self.assertEqual(len(report.downloads), 1) self.assertEqual(report.downloads[sources[2].id].total_succeeded, 0) self.assertEqual(report.downloads[sources[2].id].total_failed, 0)
def test_download_canceled_after_collated(self, fake_load): sources = [] for n in range(3): s = ContentSource('s-%d' % n, {}) s.get_downloader = Mock() sources.append(s) fake_load.return_value = sources request_list = [] for n in range(6): r = Request('T', {}, 'url-%d' % n, 'path-%d' % n) r.find_sources = Mock(return_value=sources[n % 3:]) request_list.append(r) collated = [{ sources[0]: ['nectar-1'], sources[1]: ['nectar-2', 'nectar-3', 'nectar-4'], sources[2]: ['nectar-5', 'nectar-6'] }, {}] fake_collated = Mock(side_effect=collated) fake_listener = Mock() canceled = Mock() canceled.isSet.side_effect = [False, True, True] fake_primary = PrimarySource(Mock()) # test container = ContentContainer('') container.refresh = Mock() container.collated = fake_collated report = container.download(canceled, fake_primary, request_list, fake_listener) # validation container.refresh.assert_called_with(canceled) for r in request_list: r.find_sources.assert_called_with(fake_primary, container.sources) called = 0 for s in sources: if s.get_downloader.called: called += 1 self.assertEqual(called, 1) self.assertEqual(report.total_passes, 1) self.assertEqual(report.total_sources, len(sources)) self.assertEqual(len(report.downloads), 1) self.assertEqual(report.downloads[sources[2].id].total_succeeded, 0) self.assertEqual(report.downloads[sources[2].id].total_failed, 0)
def test_collated(self): source_id = 1 request_list = [] # simulated requests # 0-8 need to be downloaded and have a next source. # 6-7 have no next source. # 9-10 are already downloaded. for n in range(10): r = Mock() r.downloaded = n > 8 r.destination = 'path-%d' % n if n < 6: r.next_source.return_value = ('s-%d' % source_id, 'url-%d' % n) else: r.next_source.return_value = None request_list.append(r) if n % 3 == 0: source_id += 1 # test collated = ContentContainer.collated(request_list) # validation self.assertEqual(len(collated), 3) for requests in collated.values(): for r in requests: self.assertTrue(isinstance(r, DownloadRequest)) self.assertEqual( [s.__dict__ for s in collated['s-1']], [ DownloadRequest('url-0', 'path-0', data=request_list[0]).__dict__ ]) self.assertEqual( [s.__dict__ for s in collated['s-2']], [ DownloadRequest('url-1', 'path-1', data=request_list[1]).__dict__, DownloadRequest('url-2', 'path-2', data=request_list[2]).__dict__, DownloadRequest('url-3', 'path-3', data=request_list[3]).__dict__ ]) self.assertEqual( [s.__dict__ for s in collated['s-3']], [ DownloadRequest('url-4', 'path-4', data=request_list[4]).__dict__, DownloadRequest('url-5', 'path-5', data=request_list[5]).__dict__, ])
def test_download_canceled_before_collated(self, fake_load): fake_load.return_value = [] canceled = FakeEvent() canceled.set() # test container = ContentContainer('') container.refresh = Mock() container.collated = Mock() report = container.download(canceled, None, [], None) container.refresh.assert_called_with(canceled) self.assertFalse(container.collated.called) self.assertEqual(report.total_passes, 0) self.assertEqual(report.total_sources, 0) self.assertEqual(len(report.downloads), 0)
def test_collated(self): source_id = 1 request_list = [] # simulated requests # 0-8 need to be downloaded and have a next source. # 6-7 have no next source. # 9-10 are already downloaded. for n in range(10): r = Mock() r.downloaded = n > 8 r.destination = 'path-%d' % n if n < 6: r.next_source.return_value = ('s-%d' % source_id, 'url-%d' % n) else: r.next_source.return_value = None request_list.append(r) if n % 3 == 0: source_id += 1 # test collated = ContentContainer.collated(request_list) # validation self.assertEqual(len(collated), 3) for requests in collated.values(): for r in requests: self.assertTrue(isinstance(r, DownloadRequest)) self.assertEqual([s.__dict__ for s in collated['s-1']], [ DownloadRequest('url-0', 'path-0', data=request_list[0]).__dict__ ]) self.assertEqual([s.__dict__ for s in collated['s-2']], [ DownloadRequest('url-1', 'path-1', data=request_list[1]).__dict__, DownloadRequest('url-2', 'path-2', data=request_list[2]).__dict__, DownloadRequest('url-3', 'path-3', data=request_list[3]).__dict__ ]) self.assertEqual([s.__dict__ for s in collated['s-3']], [ DownloadRequest('url-4', 'path-4', data=request_list[4]).__dict__, DownloadRequest('url-5', 'path-5', data=request_list[5]).__dict__, ])