def test_download(self, fake_load): sources = [] for n in range(3): s = ContentSource('s-%d' % n, {}) s.get_downloader = Mock() sources.append(s) fake_load.return_value = sources request_list = [] for n in range(6): r = Request('T', {}, 'url-%d' % n, 'path-%d' % n) r.find_sources = Mock(return_value=sources[n % 3:]) request_list.append(r) collated = [ { sources[0]: ['nectar-1'], sources[1]: ['nectar-2', 'nectar-3', 'nectar-4'], sources[2]: ['nectar-5', 'nectar-6'] }, {} ] fake_collated = Mock(side_effect=collated) fake_listener = Mock() canceled = FakeEvent() fake_primary = PrimarySource(Mock()) # test container = ContentContainer('') container.refresh = Mock() container.collated = fake_collated report = container.download(canceled, fake_primary, request_list, fake_listener) # validation container.refresh.assert_called_with(canceled) for r in request_list: r.find_sources.assert_called_with(fake_primary, container.sources) self.assertEqual(report.total_passes, 1) self.assertEqual(report.total_sources, len(sources)) self.assertEqual(len(report.downloads), 3) for source in sources: self.assertEqual(report.downloads[source.id].total_succeeded, 0) self.assertEqual(report.downloads[source.id].total_failed, 0) for source in sources: source.get_downloader.assert_called_with() downloader = source.get_downloader() listener = downloader.event_listener self.assertEqual(listener.cancel_event, canceled) self.assertEqual(listener.downloader, downloader) self.assertEqual(listener.listener, fake_listener) downloader.download.assert_called_with(collated[0][source])
def test_download(self, fake_load): sources = [] for n in range(3): s = ContentSource('s-%d' % n, {}) s.get_downloader = Mock() sources.append(s) fake_load.return_value = sources request_list = [] for n in range(6): r = Request('T', {}, 'url-%d' % n, 'path-%d' % n) r.find_sources = Mock(return_value=sources[n % 3:]) request_list.append(r) collated = [{ sources[0]: ['nectar-1'], sources[1]: ['nectar-2', 'nectar-3', 'nectar-4'], sources[2]: ['nectar-5', 'nectar-6'] }, {}] fake_collated = Mock(side_effect=collated) fake_listener = Mock() canceled = FakeEvent() fake_primary = PrimarySource(Mock()) # test container = ContentContainer('') container.refresh = Mock() container.collated = fake_collated report = container.download(canceled, fake_primary, request_list, fake_listener) # validation container.refresh.assert_called_with(canceled) for r in request_list: r.find_sources.assert_called_with(fake_primary, container.sources) self.assertEqual(report.total_passes, 1) self.assertEqual(report.total_sources, len(sources)) self.assertEqual(len(report.downloads), 3) for source in sources: self.assertEqual(report.downloads[source.id].total_succeeded, 0) self.assertEqual(report.downloads[source.id].total_failed, 0) for source in sources: source.get_downloader.assert_called_with() downloader = source.get_downloader() listener = downloader.event_listener self.assertEqual(listener.cancel_event, canceled) self.assertEqual(listener.downloader, downloader) self.assertEqual(listener.listener, fake_listener) downloader.download.assert_called_with(collated[0][source])
def test_download_canceled_after_collated(self, fake_load): sources = [] for n in range(3): s = ContentSource('s-%d' % n, {}) s.get_downloader = Mock() sources.append(s) fake_load.return_value = sources request_list = [] for n in range(6): r = Request('T', {}, 'url-%d' % n, 'path-%d' % n) r.find_sources = Mock(return_value=sources[n % 3:]) request_list.append(r) collated = [ { sources[0]: ['nectar-1'], sources[1]: ['nectar-2', 'nectar-3', 'nectar-4'], sources[2]: ['nectar-5', 'nectar-6'] }, {} ] fake_collated = Mock(side_effect=collated) fake_listener = Mock() canceled = Mock() canceled.isSet.side_effect = [False, True, True] fake_primary = PrimarySource(Mock()) # test container = ContentContainer('') container.refresh = Mock() container.collated = fake_collated report = container.download(canceled, fake_primary, request_list, fake_listener) # validation container.refresh.assert_called_with(canceled) for r in request_list: r.find_sources.assert_called_with(fake_primary, container.sources) called = 0 for s in sources: if s.get_downloader.called: called += 1 self.assertEqual(called, 1) self.assertEqual(report.total_passes, 1) self.assertEqual(report.total_sources, len(sources)) self.assertEqual(len(report.downloads), 1) self.assertEqual(report.downloads[sources[2].id].total_succeeded, 0) self.assertEqual(report.downloads[sources[2].id].total_failed, 0)
def test_is_valid_bad_descriptor(self, mock_descriptor_is_valid): source = ContentSource('s-1', {'A': 1}) source.get_downloader = Mock() source.get_cataloger = Mock() mock_descriptor_is_valid.side_effect = ValueError() # Test valid = source.is_valid() # validation self.assertFalse(valid)
def test_download_canceled_after_collated(self, fake_load): sources = [] for n in range(3): s = ContentSource('s-%d' % n, {}) s.get_downloader = Mock() sources.append(s) fake_load.return_value = sources request_list = [] for n in range(6): r = Request('T', {}, 'url-%d' % n, 'path-%d' % n) r.find_sources = Mock(return_value=sources[n % 3:]) request_list.append(r) collated = [{ sources[0]: ['nectar-1'], sources[1]: ['nectar-2', 'nectar-3', 'nectar-4'], sources[2]: ['nectar-5', 'nectar-6'] }, {}] fake_collated = Mock(side_effect=collated) fake_listener = Mock() canceled = Mock() canceled.isSet.side_effect = [False, True, True] fake_primary = PrimarySource(Mock()) # test container = ContentContainer('') container.refresh = Mock() container.collated = fake_collated report = container.download(canceled, fake_primary, request_list, fake_listener) # validation container.refresh.assert_called_with(canceled) for r in request_list: r.find_sources.assert_called_with(fake_primary, container.sources) called = 0 for s in sources: if s.get_downloader.called: called += 1 self.assertEqual(called, 1) self.assertEqual(report.total_passes, 1) self.assertEqual(report.total_sources, len(sources)) self.assertEqual(len(report.downloads), 1) self.assertEqual(report.downloads[sources[2].id].total_succeeded, 0) self.assertEqual(report.downloads[sources[2].id].total_failed, 0)
def test_is_valid(self, mock_descriptor_is_valid): source = ContentSource('s-1', {'A': 1}) source.get_downloader = Mock() source.get_cataloger = Mock() # Test valid = source.is_valid() # validation source.get_cataloger.assert_called_with() source.get_downloader.assert_called_with() mock_descriptor_is_valid.assert_called_with(source.id, source.descriptor) self.assertTrue(valid)
def test_is_valid_no_downloader(self, mock_descriptor_is_valid): source = ContentSource('s-1', {'A': 1}) source.get_downloader = Mock(side_effect=NotImplementedError()) source.get_cataloger = Mock() # Test valid = source.is_valid() # validation source.get_cataloger.assert_called_with() source.get_downloader.assert_called_with() self.assertFalse(mock_descriptor_is_valid.called) self.assertFalse(valid)
def test_is_valid_no_plugin(self, mock_descriptor_is_valid): mock_descriptor_is_valid.return_value = True source = ContentSource("s-1", {"A": 1}) source.get_downloader = Mock() source.get_cataloger = Mock(side_effect=NotImplementedError()) # Test valid = source.is_valid() # validation source.get_cataloger.assert_called_with() self.assertFalse(source.get_downloader.called) self.assertTrue(mock_descriptor_is_valid.called) self.assertFalse(valid)
def test_downloader(self): url = 'http://xyz.com' fake_conduit = Mock() fake_cataloger = Mock() fake_downloader = Mock() fake_cataloger.get_downloader = Mock(return_value=fake_downloader) source = ContentSource('s-1', {constants.BASE_URL: url}) source.get_conduit = Mock(return_value=fake_conduit) source.get_cataloger = Mock(return_value=fake_cataloger) # test downloader = source.get_downloader() # validation source.get_cataloger.assert_called_with() fake_cataloger.get_downloader.assert_called_with(fake_conduit, source.descriptor, url) self.assertEqual(downloader, fake_downloader)
def test_refresh(self, fake_manager, fake_load): sources = {} for n in range(3): s = ContentSource('s-%d' % n, {}) s.refresh = Mock(return_value=[n]) s.get_downloader = Mock() sources[s.id] = s fake_manager().has_entries.return_value = False fake_load.return_value = sources # test container = ContentContainer('') report = container.refresh() # validation for s in sources.values(): s.refresh.assert_called_with() self.assertEqual(sorted(report), [0, 1, 2])
def test_refresh_raised(self, fake_manager, fake_load): sources = {} for n in range(3): s = ContentSource('s-%d' % n, {}) s.refresh = Mock(side_effect=ValueError('must be int')) s.get_downloader = Mock() sources[s.id] = s fake_manager().has_entries.return_value = False fake_load.return_value = sources # test container = ContentContainer('') report = container.refresh() # validation for s in sources.values(): s.refresh.assert_called_with() for r in report: r.errors = ['must be int']