def test_download_to_stream(self): request_list = [] _dir, cataloged = self.populate_catalog(ORPHANED, 0, 10) _dir, cataloged = self.populate_catalog(UNIT_WORLD, 0, 10) _dir = self.populate_content(PRIMARY, 0, 20) # unit-world for n in range(0, 10): request = Request( cataloged[n].type_id, cataloged[n].unit_key, 'file://%s/unit_%d' % (_dir, n), StringIO()) request_list.append(request) # primary for n in range(11, 20): unit_key = { 'name': 'unit_%d' % n, 'version': '1.0.%d' % n, 'release': '1', 'checksum': str(uuid4()) } request = Request( TYPE_ID, unit_key, 'file://%s/unit_%d' % (_dir, n), StringIO()) request_list.append(request) downloader = LocalFileDownloader(DownloaderConfig()) listener = Mock() container = ContentContainer(path=self.tmp_dir) container.threaded = False container.refresh = Mock() # test report = container.download(downloader, request_list, listener) # validation # unit-world for i in range(0, 10): request = request_list[i] self.assertTrue(request.downloaded) self.assertEqual(len(request.errors), 0) fp = request.destination s = fp.getvalue() self.assertTrue(UNIT_WORLD in s) # primary for i in range(11, len(request_list)): request = request_list[i] self.assertTrue(request.downloaded) self.assertEqual(len(request.errors), 0) fp = request.destination s = fp.getvalue() self.assertTrue(PRIMARY in s) self.assertEqual(report.total_sources, 2) self.assertEqual(len(report.downloads), 2) self.assertEqual(report.downloads[PRIMARY_ID].total_succeeded, 9) self.assertEqual(report.downloads[PRIMARY_ID].total_failed, 0) self.assertEqual(report.downloads[UNIT_WORLD].total_succeeded, 10) self.assertEqual(report.downloads[UNIT_WORLD].total_failed, 0)
def test_forced_refresh(self, fake_manager, fake_load): sources = {} for n in range(3): s = ContentSource('s-%d' % n, {}) s.refresh = Mock() sources[s.id] = s fake_manager().has_entries.return_value = True fake_load.return_value = sources # test container = ContentContainer('') container.refresh(force=True) # validation for s in sources.values(): s.refresh.assert_called_with()
def test_forced_refresh(self, fake_manager, fake_load): sources = {} for n in range(3): s = ContentSource('s-%d' % n, {}) s.refresh = Mock() sources[s.id] = s fake_manager().has_entries.return_value = True fake_load.return_value = sources # test container = ContentContainer('') container.refresh(force=True) # validation for s in sources.values(): s.refresh.assert_called_with()
def test_refresh_canceled(self, fake_load): sources = {} for n in range(3): s = ContentSource('s-%d' % n, {}) s.refresh = Mock() sources[s.id] = s fake_load.return_value = sources # test canceled = FakeEvent() canceled.set() container = ContentContainer('') container.refresh(canceled, force=True) # validation for s in sources.values(): self.assertFalse(s.refresh.called)
def test_refresh_canceled(self, fake_load): sources = {} for n in range(3): s = ContentSource('s-%d' % n, {}) s.refresh = Mock() sources[s.id] = s fake_load.return_value = sources # test canceled = FakeEvent() canceled.set() container = ContentContainer('') container.refresh(canceled, force=True) # validation for s in sources.values(): self.assertFalse(s.refresh.called)
def test_download(self, fake_load): sources = [] for n in range(3): s = ContentSource('s-%d' % n, {}) s.get_downloader = Mock() sources.append(s) fake_load.return_value = sources request_list = [] for n in range(6): r = Request('T', {}, 'url-%d' % n, 'path-%d' % n) r.find_sources = Mock(return_value=sources[n % 3:]) request_list.append(r) collated = [ { sources[0]: ['nectar-1'], sources[1]: ['nectar-2', 'nectar-3', 'nectar-4'], sources[2]: ['nectar-5', 'nectar-6'] }, {} ] fake_collated = Mock(side_effect=collated) fake_listener = Mock() canceled = FakeEvent() fake_primary = PrimarySource(Mock()) # test container = ContentContainer('') container.refresh = Mock() container.collated = fake_collated report = container.download(canceled, fake_primary, request_list, fake_listener) # validation container.refresh.assert_called_with(canceled) for r in request_list: r.find_sources.assert_called_with(fake_primary, container.sources) self.assertEqual(report.total_passes, 1) self.assertEqual(report.total_sources, len(sources)) self.assertEqual(len(report.downloads), 3) for source in sources: self.assertEqual(report.downloads[source.id].total_succeeded, 0) self.assertEqual(report.downloads[source.id].total_failed, 0) for source in sources: source.get_downloader.assert_called_with() downloader = source.get_downloader() listener = downloader.event_listener self.assertEqual(listener.cancel_event, canceled) self.assertEqual(listener.downloader, downloader) self.assertEqual(listener.listener, fake_listener) downloader.download.assert_called_with(collated[0][source])
def test_download(self, fake_load): sources = [] for n in range(3): s = ContentSource('s-%d' % n, {}) s.get_downloader = Mock() sources.append(s) fake_load.return_value = sources request_list = [] for n in range(6): r = Request('T', {}, 'url-%d' % n, 'path-%d' % n) r.find_sources = Mock(return_value=sources[n % 3:]) request_list.append(r) collated = [{ sources[0]: ['nectar-1'], sources[1]: ['nectar-2', 'nectar-3', 'nectar-4'], sources[2]: ['nectar-5', 'nectar-6'] }, {}] fake_collated = Mock(side_effect=collated) fake_listener = Mock() canceled = FakeEvent() fake_primary = PrimarySource(Mock()) # test container = ContentContainer('') container.refresh = Mock() container.collated = fake_collated report = container.download(canceled, fake_primary, request_list, fake_listener) # validation container.refresh.assert_called_with(canceled) for r in request_list: r.find_sources.assert_called_with(fake_primary, container.sources) self.assertEqual(report.total_passes, 1) self.assertEqual(report.total_sources, len(sources)) self.assertEqual(len(report.downloads), 3) for source in sources: self.assertEqual(report.downloads[source.id].total_succeeded, 0) self.assertEqual(report.downloads[source.id].total_failed, 0) for source in sources: source.get_downloader.assert_called_with() downloader = source.get_downloader() listener = downloader.event_listener self.assertEqual(listener.cancel_event, canceled) self.assertEqual(listener.downloader, downloader) self.assertEqual(listener.listener, fake_listener) downloader.download.assert_called_with(collated[0][source])
def test_download_canceled_after_collated(self, fake_load): sources = [] for n in range(3): s = ContentSource('s-%d' % n, {}) s.get_downloader = Mock() sources.append(s) fake_load.return_value = sources request_list = [] for n in range(6): r = Request('T', {}, 'url-%d' % n, 'path-%d' % n) r.find_sources = Mock(return_value=sources[n % 3:]) request_list.append(r) collated = [ { sources[0]: ['nectar-1'], sources[1]: ['nectar-2', 'nectar-3', 'nectar-4'], sources[2]: ['nectar-5', 'nectar-6'] }, {} ] fake_collated = Mock(side_effect=collated) fake_listener = Mock() canceled = Mock() canceled.isSet.side_effect = [False, True, True] fake_primary = PrimarySource(Mock()) # test container = ContentContainer('') container.refresh = Mock() container.collated = fake_collated report = container.download(canceled, fake_primary, request_list, fake_listener) # validation container.refresh.assert_called_with(canceled) for r in request_list: r.find_sources.assert_called_with(fake_primary, container.sources) called = 0 for s in sources: if s.get_downloader.called: called += 1 self.assertEqual(called, 1) self.assertEqual(report.total_passes, 1) self.assertEqual(report.total_sources, len(sources)) self.assertEqual(len(report.downloads), 1) self.assertEqual(report.downloads[sources[2].id].total_succeeded, 0) self.assertEqual(report.downloads[sources[2].id].total_failed, 0)
def test_download_canceled_after_collated(self, fake_load): sources = [] for n in range(3): s = ContentSource('s-%d' % n, {}) s.get_downloader = Mock() sources.append(s) fake_load.return_value = sources request_list = [] for n in range(6): r = Request('T', {}, 'url-%d' % n, 'path-%d' % n) r.find_sources = Mock(return_value=sources[n % 3:]) request_list.append(r) collated = [{ sources[0]: ['nectar-1'], sources[1]: ['nectar-2', 'nectar-3', 'nectar-4'], sources[2]: ['nectar-5', 'nectar-6'] }, {}] fake_collated = Mock(side_effect=collated) fake_listener = Mock() canceled = Mock() canceled.isSet.side_effect = [False, True, True] fake_primary = PrimarySource(Mock()) # test container = ContentContainer('') container.refresh = Mock() container.collated = fake_collated report = container.download(canceled, fake_primary, request_list, fake_listener) # validation container.refresh.assert_called_with(canceled) for r in request_list: r.find_sources.assert_called_with(fake_primary, container.sources) called = 0 for s in sources: if s.get_downloader.called: called += 1 self.assertEqual(called, 1) self.assertEqual(report.total_passes, 1) self.assertEqual(report.total_sources, len(sources)) self.assertEqual(len(report.downloads), 1) self.assertEqual(report.downloads[sources[2].id].total_succeeded, 0) self.assertEqual(report.downloads[sources[2].id].total_failed, 0)
def test_refresh_exception(self, mock_refresh): container = ContentContainer(path=self.tmp_dir) # test report = container.refresh(force=True) # validation self.assertEqual(len(report), 2) for r in report: self.assertFalse(r.succeeded) self.assertEqual(r.added_count, 0) self.assertEqual(r.deleted_count, 0) self.assertEqual(len(r.errors), 1) collection = ContentCatalog.get_collection() self.assertEqual(mock_refresh.call_count, 2) self.assertEqual(collection.find().count(), 0)
def test_download_canceled_before_collated(self, fake_load): fake_load.return_value = [] canceled = FakeEvent() canceled.set() # test container = ContentContainer('') container.refresh = Mock() container.collated = Mock() report = container.download(canceled, None, [], None) container.refresh.assert_called_with(canceled) self.assertFalse(container.collated.called) self.assertEqual(report.total_passes, 0) self.assertEqual(report.total_sources, 0) self.assertEqual(len(report.downloads), 0)
def test_refresh_failure(self, mock_plugin): container = ContentContainer(path=self.tmp_dir) # test report = container.refresh(force=True) # validation self.assertEqual(len(report), 5) for r in report: self.assertFalse(r.succeeded) self.assertEqual(r.added_count, 0) self.assertEqual(r.deleted_count, 0) self.assertEqual(len(r.errors), 1) plugin = mock_plugin.return_value[0] collection = ContentCatalog.get_collection() self.assertEqual(plugin.refresh.call_count, 5) self.assertEqual(collection.find().count(), 0)
def test_download_canceled_before_collated(self, fake_load): fake_load.return_value = [] canceled = FakeEvent() canceled.set() # test container = ContentContainer('') container.refresh = Mock() container.collated = Mock() report = container.download(canceled, None, [], None) container.refresh.assert_called_with(canceled) self.assertFalse(container.collated.called) self.assertEqual(report.total_passes, 0) self.assertEqual(report.total_sources, 0) self.assertEqual(len(report.downloads), 0)
def test_refresh(self, fake_manager, fake_load): sources = {} for n in range(3): s = ContentSource('s-%d' % n, {}) s.refresh = Mock(return_value=[n]) s.get_downloader = Mock() sources[s.id] = s fake_manager().has_entries.return_value = False fake_load.return_value = sources # test container = ContentContainer('') report = container.refresh() # validation for s in sources.values(): s.refresh.assert_called_with() self.assertEqual(sorted(report), [0, 1, 2])
def test_refresh(self, fake_manager, fake_load): sources = {} for n in range(3): s = ContentSource('s-%d' % n, {}) s.refresh = Mock(return_value=[n]) s.get_downloader = Mock() sources[s.id] = s fake_manager().has_entries.return_value = False fake_load.return_value = sources # test container = ContentContainer('') report = container.refresh() # validation for s in sources.values(): s.refresh.assert_called_with() self.assertEqual(sorted(report), [0, 1, 2])
def test_refresh(self, mock_plugin): container = ContentContainer(path=self.tmp_dir) # test report = container.refresh(force=True) # validation plugin = mock_plugin.return_value[0] self.assertEqual(plugin.refresh.call_count, 5) self.assertEqual(len(report), 5) for r in report: self.assertTrue(r.succeeded) self.assertEqual(r.added_count, 100) self.assertEqual(r.deleted_count, 0) calls = iter(plugin.refresh.call_args_list) for source in ContentSource.load_all(self.tmp_dir).values(): for url in source.urls: args = calls.next()[0] self.assertTrue(isinstance(args[0], CatalogerConduit)) self.assertEqual(args[1], source.descriptor) self.assertEqual(args[2], url)
def test_refresh_raised(self, fake_manager, fake_load): sources = {} for n in range(3): s = ContentSource('s-%d' % n, {}) s.refresh = Mock(side_effect=ValueError('must be int')) s.get_downloader = Mock() sources[s.id] = s fake_manager().has_entries.return_value = False fake_load.return_value = sources # test container = ContentContainer('') report = container.refresh() # validation for s in sources.values(): s.refresh.assert_called_with() for r in report: r.errors = ['must be int']
def test_refresh_raised(self, fake_manager, fake_load): sources = {} for n in range(3): s = ContentSource('s-%d' % n, {}) s.refresh = Mock(side_effect=ValueError('must be int')) s.get_downloader = Mock() sources[s.id] = s fake_manager().has_entries.return_value = False fake_load.return_value = sources # test container = ContentContainer('') report = container.refresh() # validation for s in sources.values(): s.refresh.assert_called_with() for r in report: r.errors = ['must be int']
def test_download_with_errors(self): request_list = [] _dir, cataloged = self.populate_catalog(ORPHANED, 0, 10) _dir, cataloged = self.populate_catalog(UNDERGROUND, 0, 10) _dir, cataloged = self.populate_catalog(UNIT_WORLD, 0, 10) shutil.rmtree(_dir) _dir = self.populate_content(PRIMARY, 0, 20) # unit-world for n in range(0, 10): request = Request( cataloged[n].type_id, cataloged[n].unit_key, 'file://%s/unit_%d' % (_dir, n), os.path.join(self.downloaded, 'unit_%d' % n)) request_list.append(request) # primary for n in range(11, 20): unit_key = { 'name': 'unit_%d' % n, 'version': '1.0.%d' % n, 'release': '1', 'checksum': str(uuid4()) } request = Request( TYPE_ID, unit_key, 'file://%s/unit_%d' % (_dir, n), os.path.join(self.downloaded, 'unit_%d' % n)) request_list.append(request) downloader = LocalFileDownloader(DownloaderConfig()) listener = Mock() container = ContentContainer(path=self.tmp_dir) container.refresh = Mock() # test report = container.download(downloader, request_list, listener) # validation # unit-world for i in range(0, 10): request = request_list[i] self.assertTrue(request.downloaded, msg='URL: %s' % request.url) self.assertEqual(len(request.errors), 1) with open(request.destination) as fp: s = fp.read() self.assertTrue(UNDERGROUND in s) # primary for i in range(11, len(request_list)): request = request_list[i] self.assertTrue(request.downloaded, msg='URL: %s' % request.url) self.assertEqual(len(request.errors), 0) with open(request.destination) as fp: s = fp.read() self.assertTrue(PRIMARY in s) self.assertEqual(report.total_sources, 2) self.assertEqual(len(report.downloads), 3) self.assertEqual(report.downloads[PRIMARY_ID].total_succeeded, 9) self.assertEqual(report.downloads[PRIMARY_ID].total_failed, 0) self.assertEqual(report.downloads[UNDERGROUND].total_succeeded, 10) self.assertEqual(report.downloads[UNDERGROUND].total_failed, 0) self.assertEqual(report.downloads[UNIT_WORLD].total_succeeded, 0) self.assertEqual(report.downloads[UNIT_WORLD].total_failed, 10)