Beispiel #1
0
    def test_download_succeeded_fails_checksum(self, download_failed):
        """
        This test verifies that download_succeeded does the right thing if the checksum fails. Note
        that we are also implicitly testing that the default behavior is to validate downloads by
        not setting it in this test. There are two other tests that verify that setting the boolean
        explicitly is honored.
        """
        self.config.override_config[importer_constants.KEY_VALIDATE] = True

        iso_sync_run = ISOSyncRun(self.sync_conduit, self.config)

        destination = os.path.join(self.temp_dir, 'test.txt')
        with open(destination, 'w') as test_file:
            test_file.write('Boring test data.')
        unit = MagicMock()
        unit.storage_path = destination
        iso = models.ISO('test.txt', 114, 'wrong checksum', unit)
        iso.url = 'http://fake.com'
        report = DownloadReport(iso.url, destination, iso)

        # Let's fake having downloaded the whole file
        iso.bytes_downloaded = iso.size
        report.bytes_downloaded = iso.size
        iso_sync_run.progress_report._state = SyncProgressReport.STATE_ISOS_IN_PROGRESS

        iso_sync_run.download_succeeded(report)

        # Because we fail validation, the save_unit step will not be called
        self.assertEqual(self.sync_conduit.save_unit.call_count, 0)
        # The download should be marked failed
        self.assertEqual(download_failed.call_count, 1)
        download_failed.assert_called_once_with(report)
Beispiel #2
0
    def test_download_failed(self):
        report = DownloadReport('', '')
        report.error_report['response_code'] = 1234

        # test
        listener = DownloadListener(None, None)
        listener.download_failed(report)
Beispiel #3
0
    def test_download_succeeded_honors_validate_units_set_true(self, download_failed):
        """
        We have a setting that makes download validation optional. This test ensures that
        download_succeeded()
        honors that setting.
        """
        # In this config, we will set validate_units to False, which should make our
        # "wrong_checksum" OK
        config = importer_mocks.get_basic_config(
            **{importer_constants.KEY_FEED: 'http://fake.com/iso_feed/',
               importer_constants.KEY_VALIDATE: True})

        iso_sync_run = ISOSyncRun(self.sync_conduit, config)

        destination = os.path.join(self.temp_dir, 'test.txt')
        with open(destination, 'w') as test_file:
            test_file.write('Boring test data.')
        unit = MagicMock()
        unit.storage_path = destination
        iso = models.ISO('test.txt', 114, 'wrong checksum', unit)
        iso.url = 'http://fake.com'
        report = DownloadReport(iso.url, destination, iso)

        # Let's fake having downloaded the whole file
        iso.bytes_downloaded = iso.size
        report.bytes_downloaded = iso.size
        iso_sync_run.progress_report._state = SyncProgressReport.STATE_ISOS_IN_PROGRESS

        iso_sync_run.download_succeeded(report)

        # Because we fail validation, the save_unit step will not be called
        self.assertEqual(self.sync_conduit.save_unit.call_count, 0)
        # The download should be marked failed
        self.assertEqual(download_failed.call_count, 1)
        download_failed.assert_called_once_with(report)
Beispiel #4
0
    def test_download_succeeded(self, download_failed):
        destination = os.path.join(self.temp_dir, 'test.txt')
        with open(destination, 'w') as test_file:
            test_file.write(
                'Descartes walks into a bar and sits down, the bartender walks up to him and says '
                '"You, my '
                'man, look like you need a stiff drink." Descartes considers this, and shakes his '
                'head "No, '
                'I don\'t think-" and ceases to exist.')
        unit = MagicMock()
        unit.storage_path = destination
        iso = models.ISO('test.txt', 217,
                         'a1552efee6f04012bc7e1f3e02c00c6177b08217cead958c47ec83cb8f97f835',
                         unit)
        iso.url = 'http://fake.com'
        report = DownloadReport(iso.url, destination, iso)

        # Simulate having downloaded the whole file
        iso.bytes_downloaded = iso.size
        report.bytes_downloaded = iso.size
        self.iso_sync_run.progress_report._state = SyncProgressReport.STATE_ISOS_IN_PROGRESS

        self.iso_sync_run.download_succeeded(report)

        # The sync conduit should have been called to save the unit
        self.sync_conduit.save_unit.assert_any_call(unit)
        # The download should not fail
        self.assertEqual(download_failed.call_count, 0)
    def test_download_succeeded_honors_validate_units_set_false(self, download_failed):
        """
        We have a setting that makes download validation optional. This test ensures that download_succeeded()
        honors that setting.
        """
        # In this config, we will set validate_units to False, which should make our "wrong_checksum" OK
        config = importer_mocks.get_basic_config(**{importer_constants.KEY_FEED: 'http://fake.com/iso_feed/',
                                                    importer_constants.KEY_VALIDATE: False})

        iso_sync_run = ISOSyncRun(self.sync_conduit, config)

        destination = os.path.join(self.temp_dir, 'test.iso')
        with open(destination, 'w') as test_iso:
            test_iso.write('What happens when you combine a mosquito with a mountain climber? Nothing. You '
                           'can\'t cross a vector with a scalar.')
        unit = MagicMock()
        unit.storage_path = destination
        iso = models.ISO('test.txt', 114, 'wrong checksum', unit)
        iso.url = 'http://fake.com'
        report = DownloadReport(iso.url, destination, iso)

        # Let's fake having downloaded the whole file
        iso.bytes_downloaded = iso.size
        report.bytes_downloaded = iso.size
        iso_sync_run.progress_report._state = SyncProgressReport.STATE_ISOS_IN_PROGRESS

        iso_sync_run.download_succeeded(report)

        # The sync conduit should have been called to save the unit
        self.sync_conduit.save_unit.assert_any_call(unit)
        # The download should not fail
        self.assertEqual(download_failed.call_count, 0)
Beispiel #6
0
    def test_download_headers(self):
        request = Mock(uri='http://content-world.com/content/bear.rpm',
                       headers={})
        request.setHeader.side_effect = request.headers.__setitem__
        report = DownloadReport('', '')
        report.headers = {
            'A': 1,
            'B': 2,
        }

        # should be ignored.
        report.headers.update({k: '' for k in HOP_BY_HOP_HEADERS})

        config = Mock(properties={'streamer': {'cache_timeout': 100}})

        def get(s, p):
            return config.properties[s][p]

        config.get.side_effect = get
        streamer = Mock(config=config)

        # test
        listener = DownloadListener(streamer, request)
        listener.download_headers(report)

        # validation
        self.assertEqual(request.headers, {
            'Cache-Control': 'public, s-maxage=100, max-age=100',
            'A': 1,
            'B': 2,
        })
Beispiel #7
0
    def test_get_single_path_failure(self, mock_download_one):
        report = DownloadReport('http://pulpproject.org/v1/repositories/pulp/crane/images',
                                StringIO(''))
        report.headers = {}
        report.state = report.DOWNLOAD_FAILED
        mock_download_one.return_value = report

        self.assertRaises(IOError, self.repo._get_single_path, '/v1/repositories/pulp/crane/images')
    def test_get_single_path_failure(self, mock_download_one):
        report = DownloadReport('http://pulpproject.org/v1/repositories/pulp/crane/images',
                                StringIO(''))
        report.headers = {}
        report.state = report.DOWNLOAD_FAILED
        mock_download_one.return_value = report

        self.assertRaises(IOError, self.repo._get_single_path, '/v1/repositories/pulp/crane/images')
Beispiel #9
0
class TestDownloadReport(unittest.TestCase):

    def setUp(self):
        self.report = DownloadReport("fakeurl", "fakedestination")

    def test_download_connection_error(self):

        self.report.download_connection_error()
        self.assertEqual(self.report.state, self.report.DOWNLOAD_FAILED)
        self.assertEqual(self.report.error_msg, "A connection error occurred")
Beispiel #10
0
class TestDownloadReport(unittest.TestCase):

    def setUp(self):
        self.report = DownloadReport("fakeurl", "fakedestination")

    def test_download_connection_error(self):

        self.report.download_connection_error()
        self.assertEqual(self.report.state, self.report.DOWNLOAD_FAILED)
        self.assertEqual(self.report.error_msg, "A connection error occurred")
Beispiel #11
0
 def download_failed(self, request):
     """
     Notification that downloading has failed for the specified request.
     Fields mapped and forwarded to the wrapped listener.
     :param request: A download request.
     :type request: pulp.server.content.sources.model.Request
     """
     report = DownloadReport(request.url, request.destination, request.data)
     report.error_report['errors'] = request.errors
     self.content_listener.download_failed(report)
Beispiel #12
0
 def download_one(request):
     """
     Mock the download_one() method to manipulate the path.
     """
     self.assertEqual(request.url, 'https://registry.example.com/v2/pulp/tags/list')
     self.assertEqual(type(request.destination), type(StringIO()))
     report = DownloadReport(request.url, request.destination)
     report.download_succeeded()
     report.headers = {}
     report.destination.write('{"name": "pulp", "tags": ["best_ever", "latest", "decent"]}')
     return report
Beispiel #13
0
 def download_one(request):
     """
     Mock the download_one() method to manipulate the path.
     """
     self.assertEqual(request.url, 'https://registry.example.com/v2/pulp/tags/list')
     self.assertEqual(type(request.destination), type(StringIO()))
     report = DownloadReport(request.url, request.destination)
     report.download_succeeded()
     report.headers = {}
     report.destination.write('{"name": "pulp", "tags": ["best_ever", "latest", "decent"]}')
     return report
Beispiel #14
0
 def download_one(request):
     """
     Mock the download_one() method.
     """
     self.assertEqual(request.url, 'https://registry.example.com/some/path')
     self.assertEqual(type(request.destination), type(StringIO()))
     report = DownloadReport(request.url, request.destination)
     report.download_succeeded()
     report.headers = {'some': 'cool stuff'}
     report.destination.write("This is the stuff you've been waiting for.")
     return report
Beispiel #15
0
 def download_one(request):
     """
     Mock the download_one() method to manipulate the path.
     """
     self.assertEqual(request.url, 'https://registry.example.com/v2/')
     self.assertEqual(type(request.destination), type(StringIO()))
     report = DownloadReport(request.url, request.destination)
     report.download_succeeded()
     report.headers = {'Docker-Distribution-API-Version': 'registry/2.0'}
     report.destination.write("")
     return report
Beispiel #16
0
 def download_one(request):
     """
     Mock the download_one() method.
     """
     self.assertEqual(request.url, 'https://registry.example.com/some/path')
     self.assertEqual(type(request.destination), type(StringIO()))
     report = DownloadReport(request.url, request.destination)
     report.download_succeeded()
     report.headers = {'some': 'cool stuff'}
     report.destination.write("This is the stuff you've been waiting for.")
     return report
Beispiel #17
0
 def download_one(request):
     """
     Mock the download_one() method to manipulate the path.
     """
     self.assertEqual(request.url, 'https://registry.example.com/v2/')
     self.assertEqual(type(request.destination), type(StringIO()))
     report = DownloadReport(request.url, request.destination)
     report.download_succeeded()
     report.headers = {'Docker-Distribution-API-Version': 'registry/2.0'}
     report.destination.write("")
     return report
Beispiel #18
0
    def test__raise_path_error_not_found(self):
        """
        For a standard error like 404, the report's error message should be used.
        """
        report = DownloadReport('http://foo/bar', '/a/b/c')
        report.error_report = {'response_code': httplib.NOT_FOUND}
        report.error_msg = 'oops'

        with self.assertRaises(IOError) as assertion:
            registry.V2Repository._raise_path_error(report)

        self.assertEqual(assertion.exception.message, report.error_msg)
Beispiel #19
0
 def download_one(request):
     """
     Mock the download_one() method to manipulate the path.
     """
     self.assertEqual(request.url, 'https://registry.example.com/v2/')
     self.assertEqual(type(request.destination), type(StringIO()))
     report = DownloadReport(request.url, request.destination)
     report.download_succeeded()
     # The Version header is not present
     report.headers = {}
     report.destination.write("")
     return report
Beispiel #20
0
    def test__raise_path_error_not_found(self):
        """
        For a standard error like 404, the report's error message should be used.
        """
        report = DownloadReport('http://foo/bar', '/a/b/c')
        report.error_report = {'response_code': httplib.NOT_FOUND}
        report.error_msg = 'oops'

        with self.assertRaises(IOError) as assertion:
            registry.V2Repository._raise_path_error(report)

        self.assertEqual(assertion.exception.message, report.error_msg)
Beispiel #21
0
 def download_one(request):
     """
     Mock the download_one() method to manipulate the path.
     """
     self.assertEqual(request.url, 'https://registry.example.com/v2/')
     self.assertEqual(type(request.destination), type(StringIO()))
     report = DownloadReport(request.url, request.destination)
     report.download_succeeded()
     # The Version header is not present
     report.headers = {}
     report.destination.write("")
     return report
Beispiel #22
0
    def test_failed_reports(self):
        self.metadata_files.downloader.download = mock.MagicMock(
            spec_set=self.metadata_files.downloader.download)
        self.metadata_files.metadata = {
            'primary': file_info_factory('primary'),
        }

        report = DownloadReport('url', '/destination')
        report.download_failed()
        self.metadata_files.event_listener.failed_reports.append(report)

        # Ensure an exception is raised if the download failed
        self.assertRaises(IOError, self.metadata_files.download_metadata_files)
    def test_download_failed_during_manifest(self):
        self.iso_sync_run.progress_report._state = SyncProgressReport.STATE_MANIFEST_IN_PROGRESS
        url = 'http://www.theonion.com/articles/' +\
            'american-airlines-us-airways-merge-to-form-worlds,31302/'
        report = DownloadReport(url, '/fake/destination')
        report.error_report = {'why': 'because'}

        self.iso_sync_run.download_failed(report)

        # The manifest_state should be failed
        self.assertEqual(self.iso_sync_run.progress_report._state,
                         SyncProgressReport.STATE_MANIFEST_FAILED)
        self.assertEqual(self.iso_sync_run.progress_report.error_message, report.error_report)
Beispiel #24
0
    def test_download_failed_during_iso_download(self, _logger):
        self.iso_sync_run.progress_report._state = SyncProgressReport.STATE_ISOS_IN_PROGRESS
        url = 'http://www.theonion.com/articles/american-airlines-us-airways-merge-to-form' \
              '-worlds,31302/'
        iso = models.ISO('test.txt', 217,
                         'a1552efee6f04012bc7e1f3e02c00c6177b08217cead958c47ec83cb8f97f835')
        report = DownloadReport(url, '/fake/destination', iso)
        report.error_msg = 'uh oh'

        self.iso_sync_run.download_failed(report)

        self.assertEqual(_logger.error.call_count, 1)
        log_msg = _logger.error.mock_calls[0][1][0]
        self.assertTrue('uh oh' in log_msg)
Beispiel #25
0
    def test_retrieve_metadata_with_error(self, mock_downloader_download, mock_listener_constructor):
        # Setup
        mock_listener = mock.MagicMock()
        report = DownloadReport(None, None)
        report.error_msg = 'oops'
        mock_listener.failed_reports = [report]
        mock_listener_constructor.return_value = mock_listener

        # Test
        try:
            self.downloader.retrieve_metadata(self.mock_progress_report)
            self.fail()
        except exceptions.FileRetrievalException:
            pass
Beispiel #26
0
    def test_get_with_headers(self, mock_download_one):
        body = json.dumps(['abc123'])
        report = DownloadReport('http://pulpproject.org/v1/repositories/pulp/crane/images',
                                StringIO(body))
        report.headers = {
            self.repo.DOCKER_TOKEN_HEADER: 'token',
            self.repo.DOCKER_ENDPOINT_HEADER: 'endpoint',
        }
        mock_download_one.return_value = report

        self.repo._get_single_path('/v1/repositories/pulp/crane/images')

        self.assertEqual(self.repo.token, 'token')
        self.assertEqual(self.repo.endpoint, 'endpoint')
    def test_get_tags(self, mock_download_one):
        body = json.dumps({'latest': 'abc123'})
        report = DownloadReport('http://pulpproject.org/v1/repositories/pulp/crane/tags',
                                StringIO(body))
        report.headers = {}
        mock_download_one.return_value = report

        ret = self.repo._get_single_path('/v1/repositories/pulp/crane/tags')

        self.assertEqual(ret, {'latest': 'abc123'})
        self.assertEqual(mock_download_one.call_count, 1)
        self.assertTrue(isinstance(mock_download_one.call_args[0][0], DownloadRequest))
        req = mock_download_one.call_args[0][0]
        self.assertEqual(req.url, 'http://pulpproject.org/v1/repositories/pulp/crane/tags')
Beispiel #28
0
    def test_failed_reports(self):
        self.metadata_files.downloader.download = mock.MagicMock(
            spec_set=self.metadata_files.downloader.download
        )
        self.metadata_files.metadata = {
            'primary': file_info_factory('primary'),
        }

        report = DownloadReport('url', '/destination')
        report.download_failed()
        self.metadata_files.event_listener.failed_reports.append(report)

        # Ensure an exception is raised if the download failed
        self.assertRaises(IOError, self.metadata_files.download_metadata_files)
    def test_get_with_headers(self, mock_download_one):
        body = json.dumps(['abc123'])
        report = DownloadReport('http://pulpproject.org/v1/repositories/pulp/crane/images',
                                StringIO(body))
        report.headers = {
            self.repo.DOCKER_TOKEN_HEADER: 'token',
            self.repo.DOCKER_ENDPOINT_HEADER: 'endpoint',
        }
        mock_download_one.return_value = report

        self.repo._get_single_path('/v1/repositories/pulp/crane/images')

        self.assertEqual(self.repo.token, 'token')
        self.assertEqual(self.repo.endpoint, 'endpoint')
Beispiel #30
0
    def test_get_tags(self, mock_download_one):
        body = json.dumps({'latest': 'abc123'})
        report = DownloadReport('http://pulpproject.org/v1/repositories/pulp/crane/tags',
                                StringIO(body))
        report.headers = {}
        mock_download_one.return_value = report

        ret = self.repo._get_single_path('/v1/repositories/pulp/crane/tags')

        self.assertEqual(ret, {'latest': 'abc123'})
        self.assertEqual(mock_download_one.call_count, 1)
        self.assertTrue(isinstance(mock_download_one.call_args[0][0], DownloadRequest))
        req = mock_download_one.call_args[0][0]
        self.assertEqual(req.url, 'http://pulpproject.org/v1/repositories/pulp/crane/tags')
Beispiel #31
0
    def test_retrieve_metadata_with_error(self, mock_downloader_download,
                                          mock_listener_constructor):
        # Setup
        mock_listener = mock.MagicMock()
        report = DownloadReport(None, None)
        report.error_msg = 'oops'
        mock_listener.failed_reports = [report]
        mock_listener_constructor.return_value = mock_listener

        # Test
        try:
            self.downloader.retrieve_metadata(self.mock_progress_report)
            self.fail()
        except exceptions.FileRetrievalException:
            pass
Beispiel #32
0
 def download_one(request):
     """
     Mock the download_one() method to manipulate the path.
     """
     self.assertEqual(request.url,
                      'https://registry.example.com/v2/pulp/manifests/best_version_ever')
     self.assertEqual(type(request.destination), type(StringIO()))
     report = DownloadReport(request.url, request.destination)
     report.download_succeeded()
     schema2 = 'application/vnd.docker.distribution.manifest.v2+json'
     report.headers = {'Docker-Distribution-API-Version': 'registry/2.0',
                       'docker-content-digest': digest,
                       'content-type': schema2}
     report.destination.write(manifest)
     return report
Beispiel #33
0
    def _common_link(self, link_method, request, report=None):
        """
        Link files using either a hard link or symbolic link method.

        :param link_method: hard link or symbolic link method
        :type link_method: callable
        :param request: request instance
        :type request: nectar.request.DownloadRequest
        :param report: report instance for the request
        :type report: nectar.report.DownloadReport
        :return: report instance
        :rtype: nectar.report.DownloadReport
        """

        report = report or DownloadReport.from_download_request(request)

        report.download_started()
        self.fire_download_started(report)

        if self.is_canceled:
            report.download_cancelled()
            return report

        try:
            if not isinstance(request.destination, basestring):
                raise UnlinkableDestination(request.destination)

            src_path = self._file_path_from_url(request.url)
            link_method(src_path, request.destination)

        except Exception, e:
            _LOG.exception(e)
            report.error_msg = str(e)
            report.download_failed()
Beispiel #34
0
    def test_handle_get_all_failed(self, model, _download, _on_all_failed,
                                   responder):
        """
         Three catalog entries.
         All (3) failed.
        """
        request = Mock(uri='http://content-world.com/content/bear.rpm')
        responder.return_value.__enter__.return_value = responder.return_value
        report = DownloadReport('', '')
        _download.side_effect = SideEffect(PluginNotFound(), DoesNotExist(),
                                           DownloadFailed(report))
        catalog = [
            Mock(url='url-a'),
            Mock(url='url-b'),
            Mock(url='url-c'),
        ]
        model.objects.filter.return_value.order_by.return_value.all.return_value = catalog
        model.objects.filter.return_value.order_by.return_value.count.return_value = len(
            catalog)

        # test
        streamer = Streamer(Mock())
        streamer._handle_get(request)

        # validation
        model.objects.filter.assert_called_once_with(path='/content/bear.rpm')
        model.objects.filter.return_value.order_by.\
            assert_called_once_with('-_id', '-revision')
        responder.assert_called_once_with(request)
        _on_all_failed.assert_called_once_with(request)
        self.assertEqual(_download.call_args_list, [
            call(request, catalog[0], responder.return_value),
            call(request, catalog[1], responder.return_value),
            call(request, catalog[2], responder.return_value)
        ])
Beispiel #35
0
    def test_handle_get(self, model, _download, _on_succeeded, responder):
        """
         Three catalog entries.
         The 1st download fails but succeeds on the 2nd.
         The 3rd is not tried.
        """
        request = Mock(uri='http://content-world.com/content/bear.rpm')
        responder.return_value.__enter__.return_value = responder.return_value
        report = DownloadReport('', '')
        _download.side_effect = SideEffect(DownloadFailed(report), report,
                                           None)
        catalog = [
            Mock(url='url-a'),
            Mock(url='url-b'),
            Mock(url='url-c'),  # not tried.
        ]
        model.objects.filter.return_value.order_by.return_value.all.return_value = catalog
        model.objects.filter.return_value.order_by.return_value.count.return_value = len(
            catalog)

        # test
        streamer = Streamer(Mock())
        streamer._handle_get(request)

        # validation
        model.objects.filter.assert_called_once_with(path='/content/bear.rpm')
        model.objects.filter.return_value.order_by.\
            assert_called_once_with('-_id', '-revision')
        responder.assert_called_once_with(request)
        _on_succeeded.assert_called_once_with(catalog[1], request, report)
        self.assertEqual(_download.call_args_list, [
            call(request, catalog[0], responder.return_value),
            call(request, catalog[1], responder.return_value)
        ])
Beispiel #36
0
    def test_retrieve_module_missing_module(self, mock_downloader_download, mock_listener_constructor):
        # Setup
        mock_listener = mock.MagicMock()
        report = DownloadReport(None, None)
        report.error_msg = 'oops'
        mock_listener.failed_reports = [report]
        mock_listener_constructor.return_value = mock_listener

        # Test
        try:
            self.downloader.retrieve_module(self.mock_progress_report, self.module)
            self.fail()
        except exceptions.FileRetrievalException:
            expected_filename = web._create_download_tmp_dir(self.working_dir)
            expected_filename = os.path.join(expected_filename, self.module.filename())
            self.assertFalse(os.path.exists(os.path.join(expected_filename)))
Beispiel #37
0
 def drain(self):
     """
     Read and fail all requests remaining in the queue.
     """
     for request in NectarFeed(self):
         report = NectarDownloadReport.from_download_request(request)
         self.downloader.fire_download_failed(report)
Beispiel #38
0
 def drain(self):
     """
     Read and fail all requests remaining in the queue.
     """
     for request in NectarFeed(self):
         report = NectarDownloadReport.from_download_request(request)
         self.downloader.fire_download_failed(report)
Beispiel #39
0
    def test__raise_path_error_unathorized(self):
        """
        Specifically for a 401, a custom error message should be used explaining that the cause
        could be either that the client is unauthorized, or that the resource was not found.
        Docker hub responds 401 for the not found case, which is why this function exists.
        """
        report = DownloadReport('http://foo/bar', '/a/b/c')
        report.error_report = {'response_code': httplib.UNAUTHORIZED}
        report.error_msg = 'oops'

        with self.assertRaises(IOError) as assertion:
            registry.V2Repository._raise_path_error(report)

        # not worrying about what the exact contents are; just that the function added its
        # own message
        self.assertNotEqual(assertion.exception.message, report.error_msg)
        self.assertTrue(len(assertion.exception.message) > 0)
Beispiel #40
0
    def test_retrieve_module_missing_module(self, mock_downloader_download, mock_listener_constructor):
        # Setup
        self.module.author = 'asdf'
        self.module.puppet_standard_filename.return_value = 'puppet-filename.tar.gz'
        mock_listener = mock.MagicMock()
        report = DownloadReport(None, None)
        report.error_msg = 'oops'
        mock_listener.failed_reports = [report]
        mock_listener_constructor.return_value = mock_listener

        # Test
        try:
            self.downloader.retrieve_module(self.mock_progress_report, self.module)
            self.fail()
        except exceptions.FileRetrievalException:
            expected_filename = web._create_download_tmp_dir(self.working_dir)
            expected_filename = os.path.join(expected_filename, self.module.filename())
Beispiel #41
0
    def test__get_path_failed(self, mock_download_one, mock_request_token):
        """
        Test _get_path() for the case when an IOError is raised by the downloader.
        """
        name = 'pulp'
        download_config = DownloaderConfig(max_concurrent=25)
        registry_url = 'https://registry.example.com'
        working_dir = '/a/working/dir'
        r = registry.V2Repository(name, download_config, registry_url, working_dir)

        report = DownloadReport(registry_url + '/some/path', StringIO())
        report.error_report['response_code'] = httplib.UNAUTHORIZED
        report.state = DownloadReport.DOWNLOAD_FAILED
        mock_download_one.return_value = report

        # The request will fail because the requested path does not exist
        self.assertRaises(IOError, r._get_path, '/some/path')
Beispiel #42
0
    def test__raise_path_error_unathorized(self):
        """
        Specifically for a 401, a custom error message should be used explaining that the cause
        could be either that the client is unauthorized, or that the resource was not found.
        Docker hub responds 401 for the not found case, which is why this function exists.
        """
        report = DownloadReport('http://foo/bar', '/a/b/c')
        report.error_report = {'response_code': httplib.UNAUTHORIZED}
        report.error_msg = 'oops'

        with self.assertRaises(IOError) as assertion:
            registry.V2Repository._raise_path_error(report)

        # not worrying about what the exact contents are; just that the function added its
        # own message
        self.assertNotEqual(assertion.exception.message, report.error_msg)
        self.assertTrue(len(assertion.exception.message) > 0)
Beispiel #43
0
    def test_download_failed_during_manifest(self, _logger):
        self.iso_sync_run.progress_report._state = SyncProgressReport.STATE_MANIFEST_IN_PROGRESS
        url = 'http://www.theonion.com/articles/' + \
              'american-airlines-us-airways-merge-to-form-worlds,31302/'
        report = DownloadReport(url, '/fake/destination')
        report.error_report = {'why': 'because'}
        report.error_msg = 'uh oh'

        self.iso_sync_run.download_failed(report)

        # The manifest_state should be failed
        self.assertEqual(self.iso_sync_run.progress_report._state,
                         SyncProgressReport.STATE_MANIFEST_FAILED)
        self.assertEqual(self.iso_sync_run.progress_report.error_message, report.error_report)
        self.assertEqual(_logger.error.call_count, 1)
        log_msg = _logger.error.mock_calls[0][1][0]
        self.assertTrue('uh oh' in log_msg)
Beispiel #44
0
 def setUp(self):
     self.mock_sync = mock.MagicMock()
     # this causes validation to be skipped
     self.mock_sync.config.get.return_value = False
     self.mock_metadata_files = mock.MagicMock()
     self.listener = listener.DRPMListener(self.mock_sync,
                                           self.mock_metadata_files)
     self.report = DownloadReport('http://pulpproject.org', '/a/b/c')
    def test_get_images(self, mock_download_one):
        body = json.dumps(['abc123'])
        report = DownloadReport('http://pulpproject.org/v1/repositories/pulp/crane/images',
                                StringIO(body))
        report.headers = {}
        mock_download_one.return_value = report

        ret = self.repo._get_single_path('/v1/repositories/pulp/crane/images')

        self.assertEqual(ret, ['abc123'])
        self.assertEqual(mock_download_one.call_count, 1)
        self.assertTrue(isinstance(mock_download_one.call_args[0][0], DownloadRequest))
        req = mock_download_one.call_args[0][0]
        self.assertEqual(req.url, 'http://pulpproject.org/v1/repositories/pulp/crane/images')
        # make sure this header is set, which is required by the docker API in order
        # to give us an auth token
        self.assertEqual(req.headers[self.repo.DOCKER_TOKEN_HEADER], 'true')
Beispiel #46
0
    def test_get_images(self, mock_download_one):
        body = json.dumps(['abc123'])
        report = DownloadReport('http://pulpproject.org/v1/repositories/pulp/crane/images',
                                StringIO(body))
        report.headers = {}
        mock_download_one.return_value = report

        ret = self.repo._get_single_path('/v1/repositories/pulp/crane/images')

        self.assertEqual(ret, ['abc123'])
        self.assertEqual(mock_download_one.call_count, 1)
        self.assertTrue(isinstance(mock_download_one.call_args[0][0], DownloadRequest))
        req = mock_download_one.call_args[0][0]
        self.assertEqual(req.url, 'http://pulpproject.org/v1/repositories/pulp/crane/images')
        # make sure this header is set, which is required by the docker API in order
        # to give us an auth token
        self.assertEqual(req.headers[self.repo.DOCKER_TOKEN_HEADER], 'true')
Beispiel #47
0
    def test_on_succeeded_pulp_requested(self, _insert_deferred):
        entry = Mock(url='url-a')
        request = Mock(uri='http://content-world.com/content/bear.rpm')
        request.getHeader.side_effect = {
            constants.PULP_STREAM_REQUEST_HEADER: True
        }.__getitem__
        report = DownloadReport('', '')
        report.headers = {
            'A': 1,
            'B': 2,
        }

        # test
        streamer = Streamer(Mock())
        streamer._on_succeeded(entry, request, report)

        # validation
        self.assertFalse(_insert_deferred.called)
Beispiel #48
0
    def test__get_path_failed(self, mock_download_one, mock_request_token):
        """
        Test _get_path() for the case when an IOError is raised by the downloader.
        """
        name = 'pulp'
        download_config = DownloaderConfig(max_concurrent=25)
        registry_url = 'https://registry.example.com'
        working_dir = '/a/working/dir'
        r = registry.V2Repository(name, download_config, registry_url, working_dir)

        report = DownloadReport(registry_url + '/some/path', StringIO())
        report.error_report['response_code'] = httplib.UNAUTHORIZED
        report.state = DownloadReport.DOWNLOAD_FAILED
        report.headers = {}
        mock_download_one.return_value = report

        # The request will fail because the requested path does not exist
        self.assertRaises(IOError, r._get_path, '/some/path')
Beispiel #49
0
    def _copy(self, request, report=None):
        """
        Copy the source file to the destination.

        This is the default behavior and most useful for files that live on
        different disk partitions or networked file systems.

        :param request: request instance
        :type request: nectar.request.DownloadRequest
        :param report: report instance for the request
        :type report: nectar.report.DownloadReport
        :return: report instance
        :rtype: nectar.report.DownloadReport
        """

        report = report or DownloadReport.from_download_request(request)
        report.download_started()
        src_handle = None

        try:
            src_path = self._file_path_from_url(request.url)
            src_handle = open(src_path, 'rb')
            dst_handle = request.initialize_file_handle()
            buffer_size = self.buffer_size

            self.fire_download_started(report)
            last_progress_update = datetime.datetime.now()

            while True:

                if self.is_canceled or request.canceled:
                    report.download_canceled()
                    # NOTE the control flow here will pass through the finally
                    # block on the way out, but not the else block :D
                    return report

                chunk = src_handle.read(buffer_size)

                if not chunk:
                    break

                dst_handle.write(chunk)
                report.bytes_downloaded += len(chunk)

                now = datetime.datetime.now()

                if now - last_progress_update < self.progress_interval:
                    continue

                self.fire_download_progress(report)
                last_progress_update = now

        except IOError, e:
            logger.debug(e)
            report.error_msg = str(e)
            report.download_failed()
Beispiel #50
0
    def _copy(self, request, report=None):
        """
        Copy the source file to the destination.

        This is the default behavior and most useful for files that live on
        different disk partitions or networked file systems.

        :param request: request instance
        :type request: nectar.request.DownloadRequest
        :param report: report instance for the request
        :type report: nectar.report.DownloadReport
        :return: report instance
        :rtype: nectar.report.DownloadReport
        """

        report = report or DownloadReport.from_download_request(request)
        report.download_started()
        src_handle = None

        try:
            src_path = self._file_path_from_url(request.url)
            src_handle = open(src_path, 'rb')
            dst_handle = request.initialize_file_handle()
            buffer_size = self.buffer_size

            self.fire_download_started(report)
            last_progress_update = datetime.datetime.now()

            while True:

                if self.is_canceled:
                    report.download_canceled()
                    # NOTE the control flow here will pass through the finally
                    # block on the way out, but not the else block :D
                    return report

                chunk = src_handle.read(buffer_size)

                if not chunk:
                    break

                dst_handle.write(chunk)
                report.bytes_downloaded += len(chunk)

                now = datetime.datetime.now()

                if now - last_progress_update < self.progress_interval:
                    continue

                self.fire_download_progress(report)
                last_progress_update = now

        except Exception, e:
            logger.exception(e)
            report.error_msg = str(e)
            report.download_failed()
Beispiel #51
0
    def test_retrieve_module_missing_module(self, mock_downloader_download,
                                            mock_listener_constructor):
        # Setup
        mock_listener = mock.MagicMock()
        report = DownloadReport(None, None)
        report.error_msg = 'oops'
        mock_listener.failed_reports = [report]
        mock_listener_constructor.return_value = mock_listener

        # Test
        try:
            self.downloader.retrieve_module(self.mock_progress_report,
                                            self.module)
            self.fail()
        except exceptions.FileRetrievalException:
            expected_filename = web._create_download_tmp_dir(self.working_dir)
            expected_filename = os.path.join(expected_filename,
                                             self.module.filename())
            self.assertFalse(os.path.exists(os.path.join(expected_filename)))
    def test_get_tags_from_endpoint(self, mock_download_one):
        body = json.dumps({'latest': 'abc123'})
        report = DownloadReport('http://some-endpoint.org/v1/repositories/pulp/crane/tags',
                                StringIO(body))
        report.headers = {}
        mock_download_one.return_value = report
        self.repo.endpoint = 'some-endpoint.org'
        # this lets us test that auth was added to the request
        self.repo.token = 'letmein'

        ret = self.repo._get_single_path('/v1/repositories/pulp/crane/tags')

        self.assertEqual(ret, {'latest': 'abc123'})
        self.assertEqual(mock_download_one.call_count, 1)
        self.assertTrue(isinstance(mock_download_one.call_args[0][0], DownloadRequest))
        req = mock_download_one.call_args[0][0]
        self.assertEqual(req.url, 'http://some-endpoint.org/v1/repositories/pulp/crane/tags')
        # make sure the authorization was added, which is usually required by an endpoint
        self.assertTrue('Authorization' in req.headers)
Beispiel #53
0
    def test_calls_fetch(self, mock_fetch):
        config = DownloaderConfig()
        request = DownloadRequest('http://foo', StringIO())
        report = DownloadReport.from_download_request(request)
        downloader = threaded.HTTPThreadedDownloader(config)
        mock_fetch.return_value = report

        ret = downloader._download_one(request)

        self.assertEqual(mock_fetch.call_count, 1)
        self.assertTrue(ret is report)
        self.assertTrue(mock_fetch.call_args[0][0] is request)