def test__init___ssl_validation(self): """ Make sure the SSL validation is on by default. """ # It should default to True config = importer_mocks.get_basic_config( **{importer_constants.KEY_FEED: 'http://fake.com/iso_feed/'}) iso_sync_run = ISOSyncRun(self.sync_conduit, config) self.assertEqual(iso_sync_run.downloader.config.ssl_validation, True) # It should be possible to explicitly set it to False config = importer_mocks.get_basic_config( **{ importer_constants.KEY_FEED: 'http://fake.com/iso_feed/', importer_constants.KEY_SSL_VALIDATION: False }) iso_sync_run = ISOSyncRun(self.sync_conduit, config) self.assertEqual(iso_sync_run.downloader.config.ssl_validation, False) # It should be possible to explicitly set it to True config = importer_mocks.get_basic_config( **{ importer_constants.KEY_FEED: 'http://fake.com/iso_feed/', importer_constants.KEY_SSL_VALIDATION: True }) iso_sync_run = ISOSyncRun(self.sync_conduit, config) self.assertEqual(iso_sync_run.downloader.config.ssl_validation, True)
def test__init__(self): """ Make sure the __init__ method does cool stuff. """ iso_sync_run = ISOSyncRun(self.sync_conduit, self.config) # Now let's assert that all the right things happened during initialization self.assertEqual(iso_sync_run.sync_conduit, self.sync_conduit) self.assertEqual(iso_sync_run._repo_url, 'http://fake.com/iso_feed/') # Validation of downloads should be disabled by default self.assertEqual(iso_sync_run._validate_downloads, False) # Deleting missing ISOs should be enabled by default self.assertEqual(iso_sync_run._remove_missing_units, False) # Inspect the downloader downloader = iso_sync_run.downloader # The iso_sync_run should be the event listener for the downloader self.assertEqual(downloader.event_listener, iso_sync_run) # Inspect the downloader config expected_downloader_config = { 'max_speed': 500.0, 'max_concurrent': 5, 'ssl_client_cert': "Trust me, I'm who I say I am.", 'ssl_client_key': 'Secret Key', 'ssl_ca_cert': "Uh, I guess that's the right server.", 'ssl_validation': False, 'proxy_url': 'proxy.com', 'proxy_port': 1234, 'proxy_username': '******', 'proxy_password': '******'} for key, value in expected_downloader_config.items(): self.assertEquals(getattr(downloader.config, key), value) self.assertEquals(type(iso_sync_run.progress_report), SyncProgressReport)
def test__filter_missing_isos_available_isos(self): """ Test that when there are units in Pulp that match those in the manifest, but that are not currently associated with the repository, they are returned by _filter_missing_isos as the second list in the 3-tuple. """ # Let's put all three mammajammas in the manifest manifest = ['%s,%s,%s' % (iso.unit_key['name'], iso.unit_key['checksum'], iso.unit_key['size']) for iso in self.existing_units] manifest = '\n'.join(manifest) manifest = StringIO(manifest) manifest = models.ISOManifest(manifest, 'http://test.com') # Set up the sync conduit to return all three units as units in Pulp, but only the first # is associated with the repository sync_conduit = importer_mocks.get_sync_conduit(pkg_dir=self.pkg_dir, existing_units=[self.existing_units[0]], pulp_units=self.existing_units) iso_sync_run = ISOSyncRun(sync_conduit, self.config) filtered_isos = iso_sync_run._filter_missing_isos(manifest) local_missing_isos, local_available_isos, remote_missing_isos = filtered_isos # Everything except the first unit should be in the list of local available isos self.assertEqual(0, len(local_missing_isos)) self.assertEqual(2, len(local_available_isos)) for expected, actual in zip(sorted(self.existing_units[1:]), sorted(local_available_isos)): self.assertEqual(expected, actual)
def test_perform_local_sync(self, mock_download): """ Assert that perform_sync() works equally well with a local feed """ mock_download.side_effect = self.fake_download self.config.override_config[importer_constants.KEY_FEED] = 'file:///a/b/c' self.iso_sync_run = ISOSyncRun(self.sync_conduit, self.config) report = self.iso_sync_run.perform_sync() # There should now be three Units in the DB, but only test3.iso is the new one units = [tuple(call)[1][0] for call in self.sync_conduit.save_unit.mock_calls] self.assertEqual(len(units), 1) expected_unit = { 'checksum': '94f7fe923212286855dea858edac1b4a292301045af0ddb275544e5251a50b3c', 'size': 34, 'contents': 'Are you starting to get the idea?\n', 'name': 'test3.iso'} unit = units[0] self.assertEqual(unit.unit_key['checksum'], expected_unit['checksum']) self.assertEqual(unit.unit_key['size'], expected_unit['size']) expected_storage_path = os.path.join( self.pkg_dir, unit.unit_key['name'], unit.unit_key['checksum'], str(unit.unit_key['size']), unit.unit_key['name']) self.assertEqual(unit.storage_path, expected_storage_path) # The state should now be COMPLETE self.assertEqual(self.iso_sync_run.progress_report._state, SyncProgressReport.STATE_COMPLETE) # There should be 0 calls to sync_conduit.remove_unit, since remove_missing_units is False # by default self.assertEqual(self.sync_conduit.remove_unit.call_count, 0) self.assertEqual(report.summary['state'], ISOProgressReport.STATE_COMPLETE)
def test_perform_sync_available_local(self, mock_download): """ Test that when content is already available within Pulp it is associated with the repository if necessary. """ # Set up a sync where two of the units already exist in Pulp mock_download.side_effect = self.fake_download self.sync_conduit = importer_mocks.get_sync_conduit(pkg_dir=self.pkg_dir, existing_units=[], pulp_units=self.existing_units) self.iso_sync_run = ISOSyncRun(self.sync_conduit, self.config) self.iso_sync_run.perform_sync() # Confirm the list of unit key dictionaries was given to associate_existing expected_units = [unit.unit_key for unit in self.existing_units if unit.unit_key['name'] != 'test4.iso'] self.sync_conduit.associate_existing.assert_called_once_with(models.ISO.TYPE, expected_units) self.assertEqual(1, self.sync_conduit.associate_existing.call_count) # test3.iso is in the manifest, but is not present locally, so we'd better download it. self.assertEqual(2, mock_download.call_count) expected_url = mock_download.call_args_list[0][0][0][0].url self.assertEqual('http://fake.com/iso_feed/PULP_MANIFEST', expected_url) expected_url = mock_download.call_args_list[1][0][0][0].url self.assertEqual('http://fake.com/iso_feed/test3.iso', expected_url)
def test_download_succeeded_fails_checksum(self, download_failed): """ This test verifies that download_succeeded does the right thing if the checksum fails. Note that we are also implicitly testing that the default behavior is to validate downloads by not setting it in this test. There are two other tests that verify that setting the boolean explicitly is honored. """ self.config.override_config[importer_constants.KEY_VALIDATE] = True iso_sync_run = ISOSyncRun(self.sync_conduit, self.config) destination = os.path.join(self.temp_dir, 'test.txt') with open(destination, 'w') as test_file: test_file.write('Boring test data.') unit = MagicMock() unit.storage_path = destination iso = models.ISO('test.txt', 114, 'wrong checksum', unit) iso.url = 'http://fake.com' report = DownloadReport(iso.url, destination, iso) # Let's fake having downloaded the whole file iso.bytes_downloaded = iso.size report.bytes_downloaded = iso.size iso_sync_run.progress_report._state = SyncProgressReport.STATE_ISOS_IN_PROGRESS iso_sync_run.download_succeeded(report) # Because we fail validation, the save_unit step will not be called self.assertEqual(self.sync_conduit.save_unit.call_count, 0) # The download should be marked failed self.assertEqual(download_failed.call_count, 1) download_failed.assert_called_once_with(report)
def test_download_succeeded_honors_validate_units_set_true(self, download_failed): """ We have a setting that makes download validation optional. This test ensures that download_succeeded() honors that setting. """ # In this config, we will set validate_units to False, which should make our # "wrong_checksum" OK config = importer_mocks.get_basic_config( **{importer_constants.KEY_FEED: 'http://fake.com/iso_feed/', importer_constants.KEY_VALIDATE: True}) iso_sync_run = ISOSyncRun(self.sync_conduit, config) destination = os.path.join(self.temp_dir, 'test.txt') with open(destination, 'w') as test_file: test_file.write('Boring test data.') unit = MagicMock() unit.storage_path = destination iso = models.ISO('test.txt', 114, 'wrong checksum', unit) iso.url = 'http://fake.com' report = DownloadReport(iso.url, destination, iso) # Let's fake having downloaded the whole file iso.bytes_downloaded = iso.size report.bytes_downloaded = iso.size iso_sync_run.progress_report._state = SyncProgressReport.STATE_ISOS_IN_PROGRESS iso_sync_run.download_succeeded(report) # Because we fail validation, the save_unit step will not be called self.assertEqual(self.sync_conduit.save_unit.call_count, 0) # The download should be marked failed self.assertEqual(download_failed.call_count, 1) download_failed.assert_called_once_with(report)
def setUp(self): config = { importer_constants.KEY_FEED: 'http://fake.com/iso_feed/', importer_constants.KEY_MAX_SPEED: 500.0, importer_constants.KEY_MAX_DOWNLOADS: 5, importer_constants.KEY_SSL_VALIDATION: False, importer_constants.KEY_SSL_CLIENT_CERT: "Trust me, I'm who I say I am.", importer_constants.KEY_SSL_CLIENT_KEY: "Secret Key", importer_constants.KEY_SSL_CA_CERT: "Uh, I guess that's the right server.", importer_constants.KEY_PROXY_HOST: 'proxy.com', importer_constants.KEY_PROXY_PORT: 1234, importer_constants.KEY_PROXY_USER: "******", importer_constants.KEY_PROXY_PASS: '******', importer_constants.KEY_VALIDATE: False, } self.config = importer_mocks.get_basic_config(**config) self.temp_dir = tempfile.mkdtemp() self.pkg_dir = os.path.join(self.temp_dir, 'content') os.mkdir(self.pkg_dir) # These checksums correspond to the checksums of the files that our curl mocks will # generate. Our curl mocks do not have a test4.iso, so that one is to test removal of # old ISOs during sync self.existing_units = [ Unit( TYPE_ID_ISO, { 'name': 'test.iso', 'size': 16, 'checksum': 'f02d5a72cd2d57fa802840a76b44c6c6920a8b8e6b90b20e26c03876275069e0' }, {}, '/path/test.iso'), Unit( TYPE_ID_ISO, { 'name': 'test2.iso', 'size': 22, 'checksum': 'c7fbc0e821c0871805a99584c6a384533909f68a6bbe9a2a687d28d9f3b10c16' }, {}, '/path/test2.iso'), Unit(TYPE_ID_ISO, { 'name': 'test4.iso', 'size': 4, 'checksum': 'sum4' }, {}, '/path/test4.iso') ] self.sync_conduit = importer_mocks.get_sync_conduit( pkg_dir=self.pkg_dir, existing_units=self.existing_units, pulp_units=self.existing_units) self.iso_sync_run = ISOSyncRun(self.sync_conduit, self.config)
def test_perform_sync_remove_missing_units_set_false(self, mock_download): mock_download.side_effect = self.fake_download # Make sure the missing ISOs don't get removed if they aren't supposed to config = importer_mocks.get_basic_config( **{ importer_constants.KEY_FEED: 'http://fake.com/iso_feed/', importer_constants.KEY_MAX_SPEED: 500.0, importer_constants.KEY_MAX_DOWNLOADS: 5, importer_constants.KEY_PROXY_HOST: 'proxy.com', importer_constants.KEY_PROXY_PORT: 1234, importer_constants.KEY_PROXY_USER: "******", importer_constants.KEY_PROXY_PASS: '******', importer_constants.KEY_UNITS_REMOVE_MISSING: False, importer_constants.KEY_SSL_CLIENT_CERT: "Trust me, I'm who I say I am.", importer_constants.KEY_SSL_CLIENT_KEY: "Secret Key", importer_constants.KEY_SSL_CA_CERT: "Uh, I guess that's the right server.", importer_constants.KEY_VALIDATE: False, }) self.iso_sync_run = ISOSyncRun(self.sync_conduit, config) self.iso_sync_run.perform_sync() # There should now be three Units in the DB units = [ tuple(call)[1][0] for call in self.sync_conduit.save_unit.mock_calls ] self.assertEqual(len(units), 1) expected_unit = { 'checksum': '94f7fe923212286855dea858edac1b4a292301045af0ddb275544e5251a50b3c', 'size': 34, 'contents': 'Are you starting to get the idea?\n', 'name': 'test3.iso' } unit = units[0] self.assertEqual(unit.unit_key['checksum'], expected_unit['checksum']) self.assertEqual(unit.unit_key['size'], expected_unit['size']) expected_storage_path = os.path.join(self.pkg_dir, unit.unit_key['name'], unit.unit_key['checksum'], str(unit.unit_key['size']), unit.unit_key['name']) self.assertEqual(unit.storage_path, expected_storage_path) # There should be 0 calls to sync_conduit.remove_unit, since remove_missing_units is # False by default self.assertEqual(self.sync_conduit.remove_unit.call_count, 0)
def test__init___with_feed_lacking_trailing_slash(self): """ In bug https://bugzilla.redhat.com/show_bug.cgi?id=949004 we had a problem where feed URLs that didn't have trailing slashes would get their last URL component clobbered when we used urljoin to determine the path to PULP_MANIFEST. The solution is to have __init__() automatically append a trailing slash to URLs that lack it so that urljoin will determine the correct path to PULP_MANIFEST. """ config = importer_mocks.get_basic_config( **{importer_constants.KEY_FEED: 'http://fake.com/no_trailing_slash'}) iso_sync_run = ISOSyncRun(self.sync_conduit, config) # Humorously enough, the _repo_url attribute named no_trailing_slash should now have a # trailing slash self.assertEqual(iso_sync_run._repo_url, 'http://fake.com/no_trailing_slash/')
def test_download_succeeded_honors_validate_units_set_false( self, download_failed): """ We have a setting that makes download validation optional. This test ensures that download_succeeded() honors that setting. """ # In this config, we will set validate_units to False, which should make our # "wrong_checksum" OK config = importer_mocks.get_basic_config( **{ importer_constants.KEY_FEED: 'http://fake.com/iso_feed/', importer_constants.KEY_VALIDATE: False }) iso_sync_run = ISOSyncRun(self.sync_conduit, config) destination = os.path.join(self.temp_dir, 'test.iso') with open(destination, 'w') as test_iso: test_iso.write( 'What happens when you combine a mosquito with a mountain climber? Nothing. You ' 'can\'t cross a vector with a scalar.') unit = MagicMock() unit.storage_path = destination iso = models.ISO('test.txt', 114, 'wrong checksum', unit) iso.url = 'http://fake.com' report = DownloadReport(iso.url, destination, iso) # Let's fake having downloaded the whole file iso.bytes_downloaded = iso.size report.bytes_downloaded = iso.size iso_sync_run.progress_report._state = SyncProgressReport.STATE_ISOS_IN_PROGRESS iso_sync_run.download_succeeded(report) # The sync conduit should have been called to save the unit self.sync_conduit.save_unit.assert_any_call(unit) # The download should not fail self.assertEqual(download_failed.call_count, 0)