def test_download_with_errors(self): request_list = [] _dir, cataloged = self.populate_catalog(ORPHANED, 0, 10) _dir, cataloged = self.populate_catalog(UNDERGROUND, 0, 10) _dir, cataloged = self.populate_catalog(UNIT_WORLD, 0, 10) shutil.rmtree(_dir) _dir = self.populate_content(PRIMARY, 0, 20) # unit-world for n in range(0, 10): request = Request(cataloged[n].type_id, cataloged[n].unit_key, 'file://%s/unit_%d' % (_dir, n), os.path.join(self.downloaded, 'unit_%d' % n)) request_list.append(request) # primary for n in range(11, 20): unit_key = { 'name': 'unit_%d' % n, 'version': '1.0.%d' % n, 'release': '1', 'checksum': str(uuid4()) } request = Request(TYPE_ID, unit_key, 'file://%s/unit_%d' % (_dir, n), os.path.join(self.downloaded, 'unit_%d' % n)) request_list.append(request) downloader = LocalFileDownloader(DownloaderConfig()) listener = Mock() container = ContentContainer(path=self.tmp_dir) container.refresh = Mock() event = Event() # test report = container.download(event, downloader, request_list, listener) # validation # unit-world for i in range(0, 10): request = request_list[i] self.assertTrue(request.downloaded, msg='URL: %s' % request.url) self.assertEqual(len(request.errors), 1) with open(request.destination) as fp: s = fp.read() self.assertTrue(UNDERGROUND in s) # primary for i in range(11, len(request_list)): request = request_list[i] self.assertTrue(request.downloaded, msg='URL: %s' % request.url) self.assertEqual(len(request.errors), 0) with open(request.destination) as fp: s = fp.read() self.assertTrue(PRIMARY in s) self.assertEqual(report.total_sources, 2) self.assertEqual(len(report.downloads), 3) self.assertEqual(report.downloads[PRIMARY_ID].total_succeeded, 9) self.assertEqual(report.downloads[PRIMARY_ID].total_failed, 0) self.assertEqual(report.downloads[UNDERGROUND].total_succeeded, 10) self.assertEqual(report.downloads[UNDERGROUND].total_failed, 0) self.assertEqual(report.downloads[UNIT_WORLD].total_succeeded, 0) self.assertEqual(report.downloads[UNIT_WORLD].total_failed, 10)
def test_download_cancelled_in_failed(self, mock_started, mock_cancel): request_list = [] for n in range(0, 5): unit_key = { 'name': 'unit_%d' % n, 'version': '1.0.%d' % n, 'release': '1', 'checksum': str(uuid4()) } request = Request(TYPE_ID, unit_key, 'http://unit-city/unit_%d' % n, os.path.join(self.downloaded, 'unit_%d' % n)) request_list.append(request) downloader = HTTPThreadedDownloader(DownloaderConfig()) container = ContentContainer(path=self.tmp_dir) container.refresh = Mock() event = CancelEvent(2) report = container.download(event, downloader, request_list) self.assertTrue(mock_started.called) self.assertTrue(mock_cancel.called) self.assertEqual(report.total_passes, 1) self.assertEqual(report.total_sources, 2) self.assertEqual(len(report.downloads), 1) self.assertEqual(report.downloads[PRIMARY_ID].total_succeeded, 0) self.assertEqual(report.downloads[PRIMARY_ID].total_failed, 5)
def test_download_with_unsupported_url(self): request_list = [] _dir, cataloged = self.populate_catalog(UNSUPPORTED_PROTOCOL, 0, 10) _dir = self.populate_content(PRIMARY, 0, 20) # unit-world for n in range(0, 10): request = Request(cataloged[n].type_id, cataloged[n].unit_key, 'file://%s/unit_%d' % (_dir, n), os.path.join(self.downloaded, 'unit_%d' % n)) request_list.append(request) # primary for n in range(11, 20): unit_key = { 'name': 'unit_%d' % n, 'version': '1.0.%d' % n, 'release': '1', 'checksum': str(uuid4()) } request = Request(TYPE_ID, unit_key, 'file://%s/unit_%d' % (_dir, n), os.path.join(self.downloaded, 'unit_%d' % n)) request_list.append(request) downloader = LocalFileDownloader(DownloaderConfig()) listener = MockListener() container = ContentContainer(path=self.tmp_dir) container.refresh = Mock() event = Event() report = container.download(event, downloader, request_list, listener) for i in range(0, len(request_list)): request = request_list[i] self.assertTrue(request.downloaded) self.assertEqual(len(request.errors), 0) with open(request.destination) as fp: s = fp.read() self.assertTrue(PRIMARY in s) self.assertEqual(listener.download_started.call_count, len(request_list)) self.assertEqual(listener.download_succeeded.call_count, len(request_list)) self.assertEqual(listener.download_failed.call_count, 0) self.assertEqual(report.total_passes, 1) self.assertEqual(report.total_sources, 2) self.assertEqual(len(report.downloads), 1) self.assertEqual(report.downloads[PRIMARY_ID].total_succeeded, 19) self.assertEqual(report.downloads[PRIMARY_ID].total_failed, 0)
def test_download_with_errors(self): request_list = [] _dir, cataloged = self.populate_catalog(ORPHANED, 0, 1000) _dir, cataloged = self.populate_catalog(UNDERGROUND, 0, 1000) _dir, cataloged = self.populate_catalog(UNIT_WORLD, 0, 1000) shutil.rmtree(_dir) _dir = self.populate_content(PRIMARY, 0, 2000) # unit-world for n in range(0, 1000): request = Request(cataloged[n].type_id, cataloged[n].unit_key, 'file://%s/unit_%d' % (_dir, n), os.path.join(self.downloaded, 'unit_%d' % n)) request_list.append(request) # primary for n in range(1001, 2000): unit_key = { 'name': 'unit_%d' % n, 'version': '1.0.%d' % n, 'release': '1', 'checksum': str(uuid4()) } request = Request(TYPE_ID, unit_key, 'file://%s/unit_%d' % (_dir, n), os.path.join(self.downloaded, 'unit_%d' % n)) request_list.append(request) downloader = LocalFileDownloader(DownloaderConfig()) event = Event() threshold = len(request_list) * 0.10 # cancel after 10% started listener = TestListener(event, threshold) container = ContentContainer(path=self.tmp_dir) container.refresh = Mock() # test report = container.download(event, downloader, request_list, listener) # validation self.assertEqual(report.total_sources, 2) self.assertEqual(len(report.downloads), 2) self.assertTrue( 0 < report.downloads[UNDERGROUND].total_succeeded < 500) self.assertEqual(report.downloads[UNDERGROUND].total_failed, 0) self.assertEqual(report.downloads[UNIT_WORLD].total_succeeded, 0) self.assertTrue(0 < report.downloads[UNIT_WORLD].total_failed < 1000)
def get_requests(self): """ Get requests for the units requested to be downloaded. :return: An iterable of: Request :rtype: iterable """ for unit in self.units: if unit.metadata.get('base_url'): url = urljoin(unit.metadata.get('base_url'), unit.download_path) else: url = urljoin(self.base_url, unit.download_path) file_name = os.path.basename(unit.relative_path) destination = os.path.join(self.dst_dir, file_name) request = Request( type_id=unit.TYPE, unit_key=unit.unit_key, url=url, destination=destination) request.data = unit yield request
def create_request(url, destination, unit, unit_ref): """ Create a content container download request that is compatible with the listener. The destination directory is created as needed. :param url: The download URL. :type url: str :param destination: The absolute path to where the file is to be downloaded. :type destination: str :param unit: A published content unit. :type unit: dict :param unit_ref: A reference to the unit association. :type unit_ref: pulp_node.manifest.UnitRef. :return: A download request. :rtype: Request """ data = {STORAGE_PATH: unit[constants.STORAGE_PATH], UNIT_REF: unit_ref} dir_path = os.path.dirname(destination) pathlib.mkdir(dir_path) request = Request(unit[constants.TYPE_ID], unit[constants.UNIT_KEY], url, destination) request.data = data return request
def create_request(url, destination, unit, unit_ref): """ Create a content container download request that is compatible with the listener. The destination directory is created as needed. :param url: The download URL. :type url: str :param destination: The absolute path to where the file is to be downloaded. :type destination: str :param unit: A published content unit. :type unit: dict :param unit_ref: A reference to the unit association. :type unit_ref: pulp_node.manifest.UnitRef. :return: A download request. :rtype: Request """ data = { STORAGE_PATH: unit[constants.STORAGE_PATH], UNIT_REF: unit_ref } dir_path = os.path.dirname(destination) pathlib.mkdir(dir_path) request = Request(unit[constants.TYPE_ID], unit[constants.UNIT_KEY], url, destination) request.data = data return request
def test_download_fail_completely(self): request_list = [] _dir, cataloged = self.populate_catalog(UNIT_WORLD, 0, 10) shutil.rmtree(_dir) _dir = self.populate_content(PRIMARY, 0, 20) # primary for n in range(0, 10): unit_key = { 'name': 'unit_%d' % n, 'version': '1.0.%d' % n, 'release': '1', 'checksum': str(uuid4()) } request = Request(TYPE_ID, unit_key, 'http://redhat.com/%s/unit_%d' % (_dir, n), os.path.join(self.downloaded, 'unit_%d' % n)) request_list.append(request) downloader = HTTPThreadedDownloader(DownloaderConfig()) listener = MockListener() container = ContentContainer(path=self.tmp_dir) container.refresh = Mock() event = Event() report = container.download(event, downloader, request_list, listener) # primary for i in range(0, len(request_list)): request = request_list[i] self.assertFalse(request.downloaded) self.assertEqual(len(request.errors), 1) self.assertEqual(listener.download_started.call_count, len(request_list)) self.assertEqual(listener.download_succeeded.call_count, 0) self.assertEqual(listener.download_failed.call_count, len(request_list)) self.assertEqual(report.total_passes, 1) self.assertEqual(report.total_sources, 2) self.assertEqual(len(report.downloads), 1) self.assertEqual(report.downloads[PRIMARY_ID].total_succeeded, 0) self.assertEqual(report.downloads[PRIMARY_ID].total_failed, 10)
def __init__(self, *args, **kwargs): DownloadRequest.__init__(self, *args, **kwargs) self.url = 'http:/NOWHERE/FAIL_ME_%f' % random.random()