Ejemplo n.º 1
0
    def test__init___ssl_validation(self):
        """
        Make sure the SSL validation is on by default.
        """
        # It should default to True
        config = importer_mocks.get_basic_config(
            **{importer_constants.KEY_FEED: 'http://fake.com/iso_feed/'})
        iso_sync_run = ISOSyncRun(self.sync_conduit, config)
        self.assertEqual(iso_sync_run.downloader.config.ssl_validation, True)

        # It should be possible to explicitly set it to False
        config = importer_mocks.get_basic_config(
            **{
                importer_constants.KEY_FEED: 'http://fake.com/iso_feed/',
                importer_constants.KEY_SSL_VALIDATION: False
            })
        iso_sync_run = ISOSyncRun(self.sync_conduit, config)
        self.assertEqual(iso_sync_run.downloader.config.ssl_validation, False)

        # It should be possible to explicitly set it to True
        config = importer_mocks.get_basic_config(
            **{
                importer_constants.KEY_FEED: 'http://fake.com/iso_feed/',
                importer_constants.KEY_SSL_VALIDATION: True
            })
        iso_sync_run = ISOSyncRun(self.sync_conduit, config)
        self.assertEqual(iso_sync_run.downloader.config.ssl_validation, True)
Ejemplo n.º 2
0
    def test__init__(self):
        """
        Make sure the __init__ method does cool stuff.
        """
        iso_sync_run = ISOSyncRun(self.sync_conduit, self.config)

        # Now let's assert that all the right things happened during initialization
        self.assertEqual(iso_sync_run.sync_conduit, self.sync_conduit)
        self.assertEqual(iso_sync_run._repo_url, 'http://fake.com/iso_feed/')
        # Validation of downloads should be disabled by default
        self.assertEqual(iso_sync_run._validate_downloads, False)
        # Deleting missing ISOs should be enabled by default
        self.assertEqual(iso_sync_run._remove_missing_units, False)

        # Inspect the downloader
        downloader = iso_sync_run.downloader
        # The iso_sync_run should be the event listener for the downloader
        self.assertEqual(downloader.event_listener, iso_sync_run)
        # Inspect the downloader config
        expected_downloader_config = {
            'max_speed': 500.0, 'max_concurrent': 5,
            'ssl_client_cert': "Trust me, I'm who I say I am.",
            'ssl_client_key': 'Secret Key',
            'ssl_ca_cert': "Uh, I guess that's the right server.", 'ssl_validation': False,
            'proxy_url': 'proxy.com',
            'proxy_port': 1234,
            'proxy_username': '******',
            'proxy_password': '******'}
        for key, value in expected_downloader_config.items():
            self.assertEquals(getattr(downloader.config, key), value)
        self.assertEquals(type(iso_sync_run.progress_report), SyncProgressReport)
Ejemplo n.º 3
0
    def test__filter_missing_isos_available_isos(self):
        """
        Test that when there are units in Pulp that match those in the manifest, but that are
        not currently associated with the repository, they are returned by _filter_missing_isos
        as the second list in the 3-tuple.
        """
        # Let's put all three mammajammas in the manifest
        manifest = ['%s,%s,%s' % (iso.unit_key['name'], iso.unit_key['checksum'],
                                  iso.unit_key['size']) for iso in self.existing_units]
        manifest = '\n'.join(manifest)
        manifest = StringIO(manifest)
        manifest = models.ISOManifest(manifest, 'http://test.com')

        # Set up the sync conduit to return all three units as units in Pulp, but only the first
        # is associated with the repository
        sync_conduit = importer_mocks.get_sync_conduit(pkg_dir=self.pkg_dir,
                                                       existing_units=[self.existing_units[0]],
                                                       pulp_units=self.existing_units)
        iso_sync_run = ISOSyncRun(sync_conduit, self.config)

        filtered_isos = iso_sync_run._filter_missing_isos(manifest)
        local_missing_isos, local_available_isos, remote_missing_isos = filtered_isos

        # Everything except the first unit should be in the list of local available isos
        self.assertEqual(0, len(local_missing_isos))
        self.assertEqual(2, len(local_available_isos))
        for expected, actual in zip(sorted(self.existing_units[1:]), sorted(local_available_isos)):
            self.assertEqual(expected, actual)
Ejemplo n.º 4
0
    def test_perform_local_sync(self, mock_download):
        """
        Assert that perform_sync() works equally well with a local feed
        """
        mock_download.side_effect = self.fake_download
        self.config.override_config[importer_constants.KEY_FEED] = 'file:///a/b/c'
        self.iso_sync_run = ISOSyncRun(self.sync_conduit, self.config)

        report = self.iso_sync_run.perform_sync()

        # There should now be three Units in the DB, but only test3.iso is the new one
        units = [tuple(call)[1][0] for call in self.sync_conduit.save_unit.mock_calls]
        self.assertEqual(len(units), 1)
        expected_unit = {
            'checksum': '94f7fe923212286855dea858edac1b4a292301045af0ddb275544e5251a50b3c',
            'size': 34, 'contents': 'Are you starting to get the idea?\n', 'name': 'test3.iso'}
        unit = units[0]
        self.assertEqual(unit.unit_key['checksum'], expected_unit['checksum'])
        self.assertEqual(unit.unit_key['size'], expected_unit['size'])
        expected_storage_path = os.path.join(
            self.pkg_dir, unit.unit_key['name'], unit.unit_key['checksum'],
            str(unit.unit_key['size']), unit.unit_key['name'])
        self.assertEqual(unit.storage_path, expected_storage_path)
        # The state should now be COMPLETE
        self.assertEqual(self.iso_sync_run.progress_report._state,
                         SyncProgressReport.STATE_COMPLETE)
        # There should be 0 calls to sync_conduit.remove_unit, since remove_missing_units is False
        # by default
        self.assertEqual(self.sync_conduit.remove_unit.call_count, 0)

        self.assertEqual(report.summary['state'], ISOProgressReport.STATE_COMPLETE)
Ejemplo n.º 5
0
    def test_perform_sync_available_local(self, mock_download):
        """
        Test that when content is already available within Pulp it is associated with the
        repository if necessary.
        """
        # Set up a sync where two of the units already exist in Pulp
        mock_download.side_effect = self.fake_download
        self.sync_conduit = importer_mocks.get_sync_conduit(pkg_dir=self.pkg_dir,
                                                            existing_units=[],
                                                            pulp_units=self.existing_units)
        self.iso_sync_run = ISOSyncRun(self.sync_conduit, self.config)
        self.iso_sync_run.perform_sync()

        # Confirm the list of unit key dictionaries was given to associate_existing
        expected_units = [unit.unit_key for unit in self.existing_units
                          if unit.unit_key['name'] != 'test4.iso']
        self.sync_conduit.associate_existing.assert_called_once_with(models.ISO.TYPE,
                                                                     expected_units)
        self.assertEqual(1, self.sync_conduit.associate_existing.call_count)

        # test3.iso is in the manifest, but is not present locally, so we'd better download it.
        self.assertEqual(2, mock_download.call_count)
        expected_url = mock_download.call_args_list[0][0][0][0].url
        self.assertEqual('http://fake.com/iso_feed/PULP_MANIFEST', expected_url)
        expected_url = mock_download.call_args_list[1][0][0][0].url
        self.assertEqual('http://fake.com/iso_feed/test3.iso', expected_url)
Ejemplo n.º 6
0
    def test_download_succeeded_fails_checksum(self, download_failed):
        """
        This test verifies that download_succeeded does the right thing if the checksum fails. Note
        that we are also implicitly testing that the default behavior is to validate downloads by
        not setting it in this test. There are two other tests that verify that setting the boolean
        explicitly is honored.
        """
        self.config.override_config[importer_constants.KEY_VALIDATE] = True

        iso_sync_run = ISOSyncRun(self.sync_conduit, self.config)

        destination = os.path.join(self.temp_dir, 'test.txt')
        with open(destination, 'w') as test_file:
            test_file.write('Boring test data.')
        unit = MagicMock()
        unit.storage_path = destination
        iso = models.ISO('test.txt', 114, 'wrong checksum', unit)
        iso.url = 'http://fake.com'
        report = DownloadReport(iso.url, destination, iso)

        # Let's fake having downloaded the whole file
        iso.bytes_downloaded = iso.size
        report.bytes_downloaded = iso.size
        iso_sync_run.progress_report._state = SyncProgressReport.STATE_ISOS_IN_PROGRESS

        iso_sync_run.download_succeeded(report)

        # Because we fail validation, the save_unit step will not be called
        self.assertEqual(self.sync_conduit.save_unit.call_count, 0)
        # The download should be marked failed
        self.assertEqual(download_failed.call_count, 1)
        download_failed.assert_called_once_with(report)
Ejemplo n.º 7
0
    def test_download_succeeded_honors_validate_units_set_true(self, download_failed):
        """
        We have a setting that makes download validation optional. This test ensures that
        download_succeeded()
        honors that setting.
        """
        # In this config, we will set validate_units to False, which should make our
        # "wrong_checksum" OK
        config = importer_mocks.get_basic_config(
            **{importer_constants.KEY_FEED: 'http://fake.com/iso_feed/',
               importer_constants.KEY_VALIDATE: True})

        iso_sync_run = ISOSyncRun(self.sync_conduit, config)

        destination = os.path.join(self.temp_dir, 'test.txt')
        with open(destination, 'w') as test_file:
            test_file.write('Boring test data.')
        unit = MagicMock()
        unit.storage_path = destination
        iso = models.ISO('test.txt', 114, 'wrong checksum', unit)
        iso.url = 'http://fake.com'
        report = DownloadReport(iso.url, destination, iso)

        # Let's fake having downloaded the whole file
        iso.bytes_downloaded = iso.size
        report.bytes_downloaded = iso.size
        iso_sync_run.progress_report._state = SyncProgressReport.STATE_ISOS_IN_PROGRESS

        iso_sync_run.download_succeeded(report)

        # Because we fail validation, the save_unit step will not be called
        self.assertEqual(self.sync_conduit.save_unit.call_count, 0)
        # The download should be marked failed
        self.assertEqual(download_failed.call_count, 1)
        download_failed.assert_called_once_with(report)
Ejemplo n.º 8
0
    def setUp(self):
        config = {
            importer_constants.KEY_FEED: 'http://fake.com/iso_feed/',
            importer_constants.KEY_MAX_SPEED: 500.0,
            importer_constants.KEY_MAX_DOWNLOADS: 5,
            importer_constants.KEY_SSL_VALIDATION: False,
            importer_constants.KEY_SSL_CLIENT_CERT:
            "Trust me, I'm who I say I am.",
            importer_constants.KEY_SSL_CLIENT_KEY: "Secret Key",
            importer_constants.KEY_SSL_CA_CERT:
            "Uh, I guess that's the right server.",
            importer_constants.KEY_PROXY_HOST: 'proxy.com',
            importer_constants.KEY_PROXY_PORT: 1234,
            importer_constants.KEY_PROXY_USER: "******",
            importer_constants.KEY_PROXY_PASS: '******',
            importer_constants.KEY_VALIDATE: False,
        }

        self.config = importer_mocks.get_basic_config(**config)

        self.temp_dir = tempfile.mkdtemp()
        self.pkg_dir = os.path.join(self.temp_dir, 'content')
        os.mkdir(self.pkg_dir)

        # These checksums correspond to the checksums of the files that our curl mocks will
        # generate. Our curl mocks do not have a test4.iso, so that one is to test removal of
        # old ISOs during sync
        self.existing_units = [
            Unit(
                TYPE_ID_ISO, {
                    'name':
                    'test.iso',
                    'size':
                    16,
                    'checksum':
                    'f02d5a72cd2d57fa802840a76b44c6c6920a8b8e6b90b20e26c03876275069e0'
                }, {}, '/path/test.iso'),
            Unit(
                TYPE_ID_ISO, {
                    'name':
                    'test2.iso',
                    'size':
                    22,
                    'checksum':
                    'c7fbc0e821c0871805a99584c6a384533909f68a6bbe9a2a687d28d9f3b10c16'
                }, {}, '/path/test2.iso'),
            Unit(TYPE_ID_ISO, {
                'name': 'test4.iso',
                'size': 4,
                'checksum': 'sum4'
            }, {}, '/path/test4.iso')
        ]
        self.sync_conduit = importer_mocks.get_sync_conduit(
            pkg_dir=self.pkg_dir,
            existing_units=self.existing_units,
            pulp_units=self.existing_units)

        self.iso_sync_run = ISOSyncRun(self.sync_conduit, self.config)
Ejemplo n.º 9
0
    def test_perform_sync_remove_missing_units_set_false(self, mock_download):
        mock_download.side_effect = self.fake_download

        # Make sure the missing ISOs don't get removed if they aren't supposed to
        config = importer_mocks.get_basic_config(
            **{
                importer_constants.KEY_FEED: 'http://fake.com/iso_feed/',
                importer_constants.KEY_MAX_SPEED: 500.0,
                importer_constants.KEY_MAX_DOWNLOADS: 5,
                importer_constants.KEY_PROXY_HOST: 'proxy.com',
                importer_constants.KEY_PROXY_PORT: 1234,
                importer_constants.KEY_PROXY_USER: "******",
                importer_constants.KEY_PROXY_PASS: '******',
                importer_constants.KEY_UNITS_REMOVE_MISSING: False,
                importer_constants.KEY_SSL_CLIENT_CERT:
                "Trust me, I'm who I say I am.",
                importer_constants.KEY_SSL_CLIENT_KEY: "Secret Key",
                importer_constants.KEY_SSL_CA_CERT:
                "Uh, I guess that's the right server.",
                importer_constants.KEY_VALIDATE: False,
            })

        self.iso_sync_run = ISOSyncRun(self.sync_conduit, config)

        self.iso_sync_run.perform_sync()

        # There should now be three Units in the DB
        units = [
            tuple(call)[1][0]
            for call in self.sync_conduit.save_unit.mock_calls
        ]
        self.assertEqual(len(units), 1)
        expected_unit = {
            'checksum':
            '94f7fe923212286855dea858edac1b4a292301045af0ddb275544e5251a50b3c',
            'size': 34,
            'contents': 'Are you starting to get the idea?\n',
            'name': 'test3.iso'
        }
        unit = units[0]
        self.assertEqual(unit.unit_key['checksum'], expected_unit['checksum'])
        self.assertEqual(unit.unit_key['size'], expected_unit['size'])
        expected_storage_path = os.path.join(self.pkg_dir,
                                             unit.unit_key['name'],
                                             unit.unit_key['checksum'],
                                             str(unit.unit_key['size']),
                                             unit.unit_key['name'])
        self.assertEqual(unit.storage_path, expected_storage_path)
        # There should be 0 calls to sync_conduit.remove_unit, since remove_missing_units is
        # False by default
        self.assertEqual(self.sync_conduit.remove_unit.call_count, 0)
Ejemplo n.º 10
0
    def test__init___with_feed_lacking_trailing_slash(self):
        """
        In bug https://bugzilla.redhat.com/show_bug.cgi?id=949004 we had a problem where feed
        URLs that didn't
        have trailing slashes would get their last URL component clobbered when we used urljoin
        to determine
        the path to PULP_MANIFEST. The solution is to have __init__() automatically append a
        trailing slash to
        URLs that lack it so that urljoin will determine the correct path to PULP_MANIFEST.
        """
        config = importer_mocks.get_basic_config(
            **{importer_constants.KEY_FEED: 'http://fake.com/no_trailing_slash'})

        iso_sync_run = ISOSyncRun(self.sync_conduit, config)

        # Humorously enough, the _repo_url attribute named no_trailing_slash should now have a
        # trailing slash
        self.assertEqual(iso_sync_run._repo_url, 'http://fake.com/no_trailing_slash/')
Ejemplo n.º 11
0
    def test_download_succeeded_honors_validate_units_set_false(
            self, download_failed):
        """
        We have a setting that makes download validation optional. This test ensures that
        download_succeeded()
        honors that setting.
        """
        # In this config, we will set validate_units to False, which should make our
        # "wrong_checksum" OK
        config = importer_mocks.get_basic_config(
            **{
                importer_constants.KEY_FEED: 'http://fake.com/iso_feed/',
                importer_constants.KEY_VALIDATE: False
            })

        iso_sync_run = ISOSyncRun(self.sync_conduit, config)

        destination = os.path.join(self.temp_dir, 'test.iso')
        with open(destination, 'w') as test_iso:
            test_iso.write(
                'What happens when you combine a mosquito with a mountain climber? Nothing. You '
                'can\'t cross a vector with a scalar.')
        unit = MagicMock()
        unit.storage_path = destination
        iso = models.ISO('test.txt', 114, 'wrong checksum', unit)
        iso.url = 'http://fake.com'
        report = DownloadReport(iso.url, destination, iso)

        # Let's fake having downloaded the whole file
        iso.bytes_downloaded = iso.size
        report.bytes_downloaded = iso.size
        iso_sync_run.progress_report._state = SyncProgressReport.STATE_ISOS_IN_PROGRESS

        iso_sync_run.download_succeeded(report)

        # The sync conduit should have been called to save the unit
        self.sync_conduit.save_unit.assert_any_call(unit)
        # The download should not fail
        self.assertEqual(download_failed.call_count, 0)