def test_find_sources(self, fake_manager): type_id = 'test_1' unit_key = 1 destination = '/tmp/123' url = 'http://redhat.com/repository' primary = PrimarySource(None) alternatives = dict([(s, ContentSource(s, d)) for s, d in DESCRIPTOR]) fake_manager().find.return_value = CATALOG # test request = Request(type_id, unit_key, url, destination) request.find_sources(primary, alternatives) # validation # validate sources sorted by priority with the primary last. # should only have matched on s-1 and s-3. request.sources = list(request.sources) self.assertEqual(len(request.sources), 5) self.assertEqual(request.sources[0][0].id, 's-3') self.assertEqual(request.sources[0][1], CATALOG[2][constants.URL]) self.assertEqual(request.sources[1][0].id, 's-3') self.assertEqual(request.sources[1][1], CATALOG[3][constants.URL]) self.assertEqual(request.sources[2][0].id, 's-1') self.assertEqual(request.sources[2][1], CATALOG[0][constants.URL]) self.assertEqual(request.sources[3][0].id, 's-1') self.assertEqual(request.sources[3][1], CATALOG[1][constants.URL]) self.assertEqual(request.sources[4][0].id, primary.id) self.assertEqual(request.sources[4][1], url)
def test_find_sources(self, fake_manager): type_id = 'test_1' unit_key = 1 destination = '/tmp/123' url = 'http://redhat.com/repository' primary = PrimarySource(None) alternatives = dict([(s, ContentSource(s, d)) for s, d in DESCRIPTOR]) fake_manager().find.return_value = CATALOG # test request = Request(type_id, unit_key, url, destination) request.find_sources(primary, alternatives) # validation # validate sources sorted by priority with the primary last. # should only have matched on s-1 and s-3. self.assertEqual(len(request.sources), 5) self.assertEqual(request.sources[0][0].id, 's-3') self.assertEqual(request.sources[0][1], CATALOG[2][constants.URL]) self.assertEqual(request.sources[1][0].id, 's-3') self.assertEqual(request.sources[1][1], CATALOG[3][constants.URL]) self.assertEqual(request.sources[2][0].id, 's-1') self.assertEqual(request.sources[2][1], CATALOG[0][constants.URL]) self.assertEqual(request.sources[3][0].id, 's-1') self.assertEqual(request.sources[3][1], CATALOG[1][constants.URL]) self.assertEqual(request.sources[4][0].id, primary.id) self.assertEqual(request.sources[4][1], url)
def test_download_to_stream(self): request_list = [] _dir, cataloged = self.populate_catalog(ORPHANED, 0, 10) _dir, cataloged = self.populate_catalog(UNIT_WORLD, 0, 10) _dir = self.populate_content(PRIMARY, 0, 20) # unit-world for n in range(0, 10): request = Request( cataloged[n].type_id, cataloged[n].unit_key, 'file://%s/unit_%d' % (_dir, n), StringIO()) request_list.append(request) # primary for n in range(11, 20): unit_key = { 'name': 'unit_%d' % n, 'version': '1.0.%d' % n, 'release': '1', 'checksum': str(uuid4()) } request = Request( TYPE_ID, unit_key, 'file://%s/unit_%d' % (_dir, n), StringIO()) request_list.append(request) downloader = LocalFileDownloader(DownloaderConfig()) listener = Mock() container = ContentContainer(path=self.tmp_dir) container.threaded = False container.refresh = Mock() # test report = container.download(downloader, request_list, listener) # validation # unit-world for i in range(0, 10): request = request_list[i] self.assertTrue(request.downloaded) self.assertEqual(len(request.errors), 0) fp = request.destination s = fp.getvalue() self.assertTrue(UNIT_WORLD in s) # primary for i in range(11, len(request_list)): request = request_list[i] self.assertTrue(request.downloaded) self.assertEqual(len(request.errors), 0) fp = request.destination s = fp.getvalue() self.assertTrue(PRIMARY in s) self.assertEqual(report.total_sources, 2) self.assertEqual(len(report.downloads), 2) self.assertEqual(report.downloads[PRIMARY_ID].total_succeeded, 9) self.assertEqual(report.downloads[PRIMARY_ID].total_failed, 0) self.assertEqual(report.downloads[UNIT_WORLD].total_succeeded, 10) self.assertEqual(report.downloads[UNIT_WORLD].total_failed, 0)
def test_next_source(self): sources = [1, 2, 3] request = Request('', {}, '', '') request.sources = sources # test and validation for i, source in enumerate(request.sources): self.assertEqual(source, sources[i])
def test_download(self, fake_load): sources = [] for n in range(3): s = ContentSource('s-%d' % n, {}) s.get_downloader = Mock() sources.append(s) fake_load.return_value = sources request_list = [] for n in range(6): r = Request('T', {}, 'url-%d' % n, 'path-%d' % n) r.find_sources = Mock(return_value=sources[n % 3:]) request_list.append(r) collated = [ { sources[0]: ['nectar-1'], sources[1]: ['nectar-2', 'nectar-3', 'nectar-4'], sources[2]: ['nectar-5', 'nectar-6'] }, {} ] fake_collated = Mock(side_effect=collated) fake_listener = Mock() canceled = FakeEvent() fake_primary = PrimarySource(Mock()) # test container = ContentContainer('') container.refresh = Mock() container.collated = fake_collated report = container.download(canceled, fake_primary, request_list, fake_listener) # validation container.refresh.assert_called_with(canceled) for r in request_list: r.find_sources.assert_called_with(fake_primary, container.sources) self.assertEqual(report.total_passes, 1) self.assertEqual(report.total_sources, len(sources)) self.assertEqual(len(report.downloads), 3) for source in sources: self.assertEqual(report.downloads[source.id].total_succeeded, 0) self.assertEqual(report.downloads[source.id].total_failed, 0) for source in sources: source.get_downloader.assert_called_with() downloader = source.get_downloader() listener = downloader.event_listener self.assertEqual(listener.cancel_event, canceled) self.assertEqual(listener.downloader, downloader) self.assertEqual(listener.listener, fake_listener) downloader.download.assert_called_with(collated[0][source])
def test_has_source(self): request = Request('', {}, '', '') request.sources = [1] # test and validation self.assertTrue(request.has_source()) request.index = 1 self.assertFalse(request.has_source())
def test_download(self, fake_load): sources = [] for n in range(3): s = ContentSource('s-%d' % n, {}) s.get_downloader = Mock() sources.append(s) fake_load.return_value = sources request_list = [] for n in range(6): r = Request('T', {}, 'url-%d' % n, 'path-%d' % n) r.find_sources = Mock(return_value=sources[n % 3:]) request_list.append(r) collated = [{ sources[0]: ['nectar-1'], sources[1]: ['nectar-2', 'nectar-3', 'nectar-4'], sources[2]: ['nectar-5', 'nectar-6'] }, {}] fake_collated = Mock(side_effect=collated) fake_listener = Mock() canceled = FakeEvent() fake_primary = PrimarySource(Mock()) # test container = ContentContainer('') container.refresh = Mock() container.collated = fake_collated report = container.download(canceled, fake_primary, request_list, fake_listener) # validation container.refresh.assert_called_with(canceled) for r in request_list: r.find_sources.assert_called_with(fake_primary, container.sources) self.assertEqual(report.total_passes, 1) self.assertEqual(report.total_sources, len(sources)) self.assertEqual(len(report.downloads), 3) for source in sources: self.assertEqual(report.downloads[source.id].total_succeeded, 0) self.assertEqual(report.downloads[source.id].total_failed, 0) for source in sources: source.get_downloader.assert_called_with() downloader = source.get_downloader() listener = downloader.event_listener self.assertEqual(listener.cancel_event, canceled) self.assertEqual(listener.downloader, downloader) self.assertEqual(listener.listener, fake_listener) downloader.download.assert_called_with(collated[0][source])
def test_next_source(self): request = Request('', {}, '', '') request.sources = [1, 2, 3] # test and validation for n in request.sources: source = request.next_source() self.assertEqual(source, n) self.assertTrue(request.next_source() is None)
def test_download_canceled_after_collated(self, fake_load): sources = [] for n in range(3): s = ContentSource('s-%d' % n, {}) s.get_downloader = Mock() sources.append(s) fake_load.return_value = sources request_list = [] for n in range(6): r = Request('T', {}, 'url-%d' % n, 'path-%d' % n) r.find_sources = Mock(return_value=sources[n % 3:]) request_list.append(r) collated = [ { sources[0]: ['nectar-1'], sources[1]: ['nectar-2', 'nectar-3', 'nectar-4'], sources[2]: ['nectar-5', 'nectar-6'] }, {} ] fake_collated = Mock(side_effect=collated) fake_listener = Mock() canceled = Mock() canceled.isSet.side_effect = [False, True, True] fake_primary = PrimarySource(Mock()) # test container = ContentContainer('') container.refresh = Mock() container.collated = fake_collated report = container.download(canceled, fake_primary, request_list, fake_listener) # validation container.refresh.assert_called_with(canceled) for r in request_list: r.find_sources.assert_called_with(fake_primary, container.sources) called = 0 for s in sources: if s.get_downloader.called: called += 1 self.assertEqual(called, 1) self.assertEqual(report.total_passes, 1) self.assertEqual(report.total_sources, len(sources)) self.assertEqual(len(report.downloads), 1) self.assertEqual(report.downloads[sources[2].id].total_succeeded, 0) self.assertEqual(report.downloads[sources[2].id].total_failed, 0)
def test_download_canceled_after_collated(self, fake_load): sources = [] for n in range(3): s = ContentSource('s-%d' % n, {}) s.get_downloader = Mock() sources.append(s) fake_load.return_value = sources request_list = [] for n in range(6): r = Request('T', {}, 'url-%d' % n, 'path-%d' % n) r.find_sources = Mock(return_value=sources[n % 3:]) request_list.append(r) collated = [{ sources[0]: ['nectar-1'], sources[1]: ['nectar-2', 'nectar-3', 'nectar-4'], sources[2]: ['nectar-5', 'nectar-6'] }, {}] fake_collated = Mock(side_effect=collated) fake_listener = Mock() canceled = Mock() canceled.isSet.side_effect = [False, True, True] fake_primary = PrimarySource(Mock()) # test container = ContentContainer('') container.refresh = Mock() container.collated = fake_collated report = container.download(canceled, fake_primary, request_list, fake_listener) # validation container.refresh.assert_called_with(canceled) for r in request_list: r.find_sources.assert_called_with(fake_primary, container.sources) called = 0 for s in sources: if s.get_downloader.called: called += 1 self.assertEqual(called, 1) self.assertEqual(report.total_passes, 1) self.assertEqual(report.total_sources, len(sources)) self.assertEqual(len(report.downloads), 1) self.assertEqual(report.downloads[sources[2].id].total_succeeded, 0) self.assertEqual(report.downloads[sources[2].id].total_failed, 0)
def test_on_succeeded(self): request = Request('T1', {'A': 1}, 'http://test', '/tmp/test') request.data = Mock() content_listener = Mock() # test listener = ContainerListener(content_listener) listener.on_succeeded(request) # validation calls = content_listener.download_succeeded.mock_calls self.assertEqual(len(calls), 1) report = calls[0][1][0] self.assertEqual(report.url, request.url) self.assertEqual(report.destination, request.destination) self.assertEqual(report.data, request.data)
def test_download_succeeded(self): request = Request("T1", {"A": 1}, "http://test", "/tmp/test") request.data = Mock() content_listener = Mock() # test listener = ContainerListener(content_listener) listener.download_succeeded(request) # validation calls = content_listener.download_succeeded.mock_calls self.assertEqual(len(calls), 1) report = calls[0][1][0] self.assertEqual(report.url, request.url) self.assertEqual(report.destination, request.destination) self.assertEqual(report.data, request.data)
def get_requests(self): """ Get requests for the units requested to be downloaded. :return: An iterable of: Request :rtype: iterable """ for unit in self.units: base_url = unit.base_url or self.base_url url = self.url_modify(base_url, path_append=unit.download_path) destination = os.path.join(self.dst_dir, unit.filename) request = Request(type_id=unit.type_id, unit_key=unit.unit_key, url=url, destination=destination) request.data = unit yield request
def get_requests(self): """ Get requests for the units requested to be downloaded. :return: An iterable of: Request :rtype: iterable """ for unit in self.units: base_url = unit.base_url or self.base_url url = self.url_modify(base_url, path_append=unit.filename) destination = os.path.join(self.dst_dir, unit.filename) request = Request( type_id=unit.type_id, unit_key=unit.unit_key, url=url, destination=destination) request.data = unit yield request
def test_on_failed(self): request = Request("T1", {"A": 1}, "http://test", "/tmp/test") request.data = Mock() request.errors = [1, 2, 3] content_listener = Mock() # test listener = ContainerListener(content_listener) listener.on_failed(request) # validation calls = content_listener.download_failed.mock_calls self.assertEqual(len(calls), 1) report = calls[0][1][0] self.assertEqual(report.url, request.url) self.assertEqual(report.destination, request.destination) self.assertEqual(report.data, request.data) self.assertEqual(report.error_report["errors"], request.errors)
def test_download_failed(self): request = Request('T1', {'A': 1}, 'http://test', '/tmp/test') request.data = Mock() request.errors = [1, 2, 3] content_listener = Mock() # test listener = ContainerListener(content_listener) listener.download_failed(request) # validation calls = content_listener.download_failed.mock_calls self.assertEqual(len(calls), 1) report = calls[0][1][0] self.assertEqual(report.url, request.url) self.assertEqual(report.destination, request.destination) self.assertEqual(report.data, request.data) self.assertEqual(report.error_report['errors'], request.errors)
def get_requests(self): """ Get requests for the units requested to be downloaded. :return: An iterable of: Request :rtype: iterable """ for unit in self.units: if unit.metadata.get('base_url'): url = urljoin(unit.metadata.get('base_url'), unit.download_path) else: url = urljoin(self.base_url, unit.download_path) file_name = os.path.basename(unit.relative_path) destination = os.path.join(self.dst_dir, file_name) request = Request( type_id=unit.TYPE, unit_key=unit.unit_key, url=url, destination=destination) request.data = unit yield request
def create_request(url, destination, unit, unit_ref): """ Create a content container download request that is compatible with the listener. The destination directory is created as needed. :param url: The download URL. :type url: str :param destination: The absolute path to where the file is to be downloaded. :type destination: str :param unit: A published content unit. :type unit: dict :param unit_ref: A reference to the unit association. :type unit_ref: pulp_node.manifest.UnitRef. :return: A download request. :rtype: Request """ data = {STORAGE_PATH: unit[constants.STORAGE_PATH], UNIT_REF: unit_ref} dir_path = os.path.dirname(destination) pathlib.mkdir(dir_path) request = Request(unit[constants.TYPE_ID], unit[constants.UNIT_KEY], url, destination) request.data = data return request
def create_request(url, destination, unit, unit_ref): """ Create a content container download request that is compatible with the listener. The destination directory is created as needed. :param url: The download URL. :type url: str :param destination: The absolute path to where the file is to be downloaded. :type destination: str :param unit: A published content unit. :type unit: dict :param unit_ref: A reference to the unit association. :type unit_ref: pulp_node.manifest.UnitRef. :return: A download request. :rtype: Request """ data = { STORAGE_PATH: unit[constants.STORAGE_PATH], UNIT_REF: unit_ref } dir_path = os.path.dirname(destination) pathlib.mkdir(dir_path) request = Request(unit[constants.TYPE_ID], unit[constants.UNIT_KEY], url, destination) request.data = data return request
def test_construction(self): type_id = 'test_1' unit_key = {'name': 'A', 'version': '1.0'} destination = '/tmp/123' url = 'http://redhat.com/repository' # test request = Request(type_id, unit_key, url, destination) # validation self.assertEqual(request.type_id, type_id) self.assertEqual(request.unit_key, unit_key) self.assertEqual(request.url, url) self.assertEqual(request.destination, destination) self.assertFalse(request.downloaded) self.assertEqual(request.sources, []) self.assertEqual(request.index, 0) self.assertEqual(request.errors, []) self.assertEqual(request.data, None)
def __init__(self, *args, **kwargs): DownloadRequest.__init__(self, *args, **kwargs) self.url = 'http:/NOWHERE/FAIL_ME_%f' % random.random()
def test_download_with_errors(self): request_list = [] _dir, cataloged = self.populate_catalog(ORPHANED, 0, 10) _dir, cataloged = self.populate_catalog(UNDERGROUND, 0, 10) _dir, cataloged = self.populate_catalog(UNIT_WORLD, 0, 10) shutil.rmtree(_dir) _dir = self.populate_content(PRIMARY, 0, 20) # unit-world for n in range(0, 10): request = Request( cataloged[n].type_id, cataloged[n].unit_key, 'file://%s/unit_%d' % (_dir, n), os.path.join(self.downloaded, 'unit_%d' % n)) request_list.append(request) # primary for n in range(11, 20): unit_key = { 'name': 'unit_%d' % n, 'version': '1.0.%d' % n, 'release': '1', 'checksum': str(uuid4()) } request = Request( TYPE_ID, unit_key, 'file://%s/unit_%d' % (_dir, n), os.path.join(self.downloaded, 'unit_%d' % n)) request_list.append(request) downloader = LocalFileDownloader(DownloaderConfig()) listener = Mock() container = ContentContainer(path=self.tmp_dir) container.refresh = Mock() # test report = container.download(downloader, request_list, listener) # validation # unit-world for i in range(0, 10): request = request_list[i] self.assertTrue(request.downloaded, msg='URL: %s' % request.url) self.assertEqual(len(request.errors), 1) with open(request.destination) as fp: s = fp.read() self.assertTrue(UNDERGROUND in s) # primary for i in range(11, len(request_list)): request = request_list[i] self.assertTrue(request.downloaded, msg='URL: %s' % request.url) self.assertEqual(len(request.errors), 0) with open(request.destination) as fp: s = fp.read() self.assertTrue(PRIMARY in s) self.assertEqual(report.total_sources, 2) self.assertEqual(len(report.downloads), 3) self.assertEqual(report.downloads[PRIMARY_ID].total_succeeded, 9) self.assertEqual(report.downloads[PRIMARY_ID].total_failed, 0) self.assertEqual(report.downloads[UNDERGROUND].total_succeeded, 10) self.assertEqual(report.downloads[UNDERGROUND].total_failed, 0) self.assertEqual(report.downloads[UNIT_WORLD].total_succeeded, 0) self.assertEqual(report.downloads[UNIT_WORLD].total_failed, 10)