def test_perform_sync_available_local(self, mock_download):
"""
Test that when content is already available within Pulp it is associated with the
repository if necessary.
"""
# Set up a sync where two of the units already exist in Pulp
mock_download.side_effect = self.fake_download
self.sync_conduit = importer_mocks.get_sync_conduit(pkg_dir=self.pkg_dir,
existing_units=[],
pulp_units=self.existing_units)
self.iso_sync_run = ISOSyncRun(self.sync_conduit, self.config)
self.iso_sync_run.perform_sync()
# Confirm the list of unit key dictionaries was given to associate_existing
expected_units = [unit.unit_key for unit in self.existing_units
if unit.unit_key['name'] != 'test4.iso']
self.sync_conduit.associate_existing.assert_called_once_with(models.ISO.TYPE,
expected_units)
self.assertEqual(1, self.sync_conduit.associate_existing.call_count)
# test3.iso is in the manifest, but is not present locally, so we'd better download it.
self.assertEqual(2, mock_download.call_count)
expected_url = mock_download.call_args_list[0][0][0][0].url
self.assertEqual('http://fake.com/iso_feed/PULP_MANIFEST', expected_url)
expected_url = mock_download.call_args_list[1][0][0][0].url
self.assertEqual('http://fake.com/iso_feed/test3.iso', expected_url)
def test__filter_missing_isos_available_isos(self):
"""
Test that when there are units in Pulp that match those in the manifest, but that are
not currently associated with the repository, they are returned by _filter_missing_isos
as the second list in the 3-tuple.
"""
# Let's put all three mammajammas in the manifest
manifest = ['%s,%s,%s' % (iso.unit_key['name'], iso.unit_key['checksum'],
iso.unit_key['size']) for iso in self.existing_units]
manifest = '\n'.join(manifest)
manifest = StringIO(manifest)
manifest = models.ISOManifest(manifest, 'http://test.com')
# Set up the sync conduit to return all three units as units in Pulp, but only the first
# is associated with the repository
sync_conduit = importer_mocks.get_sync_conduit(pkg_dir=self.pkg_dir,
existing_units=[self.existing_units[0]],
pulp_units=self.existing_units)
iso_sync_run = ISOSyncRun(sync_conduit, self.config)
filtered_isos = iso_sync_run._filter_missing_isos(manifest)
local_missing_isos, local_available_isos, remote_missing_isos = filtered_isos
# Everything except the first unit should be in the list of local available isos
self.assertEqual(0, len(local_missing_isos))
self.assertEqual(2, len(local_available_isos))
for expected, actual in zip(sorted(self.existing_units[1:]), sorted(local_available_isos)):
self.assertEqual(expected, actual)
def test_upload_unit_validate_unset(self, remove, validate):
"""
Assert correct behavior from upload_unit() when the validation setting is not set. This
should default to validating the upload.
"""
# Set up the test
file_data = "This is a file.\n"
working_dir = os.path.join(self.temp_dir, "working")
os.mkdir(working_dir)
pkg_dir = os.path.join(self.temp_dir, "content")
os.mkdir(pkg_dir)
repo = mock.MagicMock(spec=Repository)
repo.working_dir = working_dir
# Set the checksum incorrect. The upload should be unsuccessful since the default is to
# validate
unit_key = {"name": "test.iso", "size": 16, "checksum": "Wrong"}
metadata = {}
temp_file_location = os.path.join(self.temp_dir, "test.iso")
with open(temp_file_location, "w") as temp_file:
temp_file.write(file_data)
sync_conduit = importer_mocks.get_sync_conduit(pkg_dir=pkg_dir)
# validate isn't set, so default should happen
config = importer_mocks.get_basic_config()
# Run the upload. This should report a failure
report = self.iso_importer.upload_unit(
repo, ids.TYPE_ID_ISO, unit_key, metadata, temp_file_location, sync_conduit, config
)
self.assertEqual(report["success_flag"], False)
self.assertEqual(
report["summary"],
(
"Downloading <test.iso> failed checksum validation. The manifest specified the "
"checksum to be Wrong, but it was "
"f02d5a72cd2d57fa802840a76b44c6c6920a8b8e6b90b20e26c03876275069e0."
),
)
# The conduit's init_unit method should have been called
expected_rel_path = os.path.join(
unit_key["name"], unit_key["checksum"], str(unit_key["size"]), unit_key["name"]
)
modified_metadata = metadata.copy()
modified_metadata[server_constants.PULP_USER_METADATA_FIELDNAME] = {}
sync_conduit.init_unit.assert_called_once_with(ids.TYPE_ID_ISO, unit_key, modified_metadata, expected_rel_path)
# The file should have been moved to its final destination
self.assertFalse(os.path.exists(temp_file_location))
would_be_destination = os.path.join(pkg_dir, expected_rel_path)
self.assertFalse(os.path.exists(would_be_destination))
# The file should have been removed
remove.assert_called_once_with(would_be_destination)
# validate() should have been called with the full_validation=True flag
iso = validate.mock_calls[0][1][0]
validate.assert_called_once_with(iso, full_validation=True)
# The conduit's save_unit method should have been called
self.assertEqual(sync_conduit.save_unit.call_count, 0)
def setUp(self):
config = {
importer_constants.KEY_FEED: 'http://fake.com/iso_feed/',
importer_constants.KEY_MAX_SPEED: 500.0,
importer_constants.KEY_MAX_DOWNLOADS: 5,
importer_constants.KEY_SSL_VALIDATION: False,
importer_constants.KEY_SSL_CLIENT_CERT:
"Trust me, I'm who I say I am.",
importer_constants.KEY_SSL_CLIENT_KEY: "Secret Key",
importer_constants.KEY_SSL_CA_CERT:
"Uh, I guess that's the right server.",
importer_constants.KEY_PROXY_HOST: 'proxy.com',
importer_constants.KEY_PROXY_PORT: 1234,
importer_constants.KEY_PROXY_USER: "******",
importer_constants.KEY_PROXY_PASS: '******',
importer_constants.KEY_VALIDATE: False,
}
self.config = importer_mocks.get_basic_config(**config)
self.temp_dir = tempfile.mkdtemp()
self.pkg_dir = os.path.join(self.temp_dir, 'content')
os.mkdir(self.pkg_dir)
# These checksums correspond to the checksums of the files that our curl mocks will
# generate. Our curl mocks do not have a test4.iso, so that one is to test removal of
# old ISOs during sync
self.existing_units = [
Unit(
TYPE_ID_ISO, {
'name':
'test.iso',
'size':
16,
'checksum':
'f02d5a72cd2d57fa802840a76b44c6c6920a8b8e6b90b20e26c03876275069e0'
}, {}, '/path/test.iso'),
Unit(
TYPE_ID_ISO, {
'name':
'test2.iso',
'size':
22,
'checksum':
'c7fbc0e821c0871805a99584c6a384533909f68a6bbe9a2a687d28d9f3b10c16'
}, {}, '/path/test2.iso'),
Unit(TYPE_ID_ISO, {
'name': 'test4.iso',
'size': 4,
'checksum': 'sum4'
}, {}, '/path/test4.iso')
]
self.sync_conduit = importer_mocks.get_sync_conduit(
pkg_dir=self.pkg_dir,
existing_units=self.existing_units,
pulp_units=self.existing_units)
self.iso_sync_run = ISOSyncRun(self.sync_conduit, self.config)
def test_upload_unit_validate_true_good_checksum(self):
"""
Test behavior with good arguments.
"""
# Set up the test
file_data = 'This is a file.\n'
working_dir = os.path.join(self.temp_dir, "working")
os.mkdir(working_dir)
pkg_dir = os.path.join(self.temp_dir, 'content')
os.mkdir(pkg_dir)
repo = mock.MagicMock(spec=Repository)
repo.working_dir = working_dir
unit_key = {
'name':
'test.iso',
'size':
16,
'checksum':
'f02d5a72cd2d57fa802840a76b44c6c6920a8b8e6b90b20e26c03876275069e0'
}
metadata = {}
temp_file_location = os.path.join(self.temp_dir, 'test.iso')
with open(temp_file_location, 'w') as temp_file:
temp_file.write(file_data)
sync_conduit = importer_mocks.get_sync_conduit(pkg_dir=pkg_dir)
config = importer_mocks.get_basic_config(
**{importer_constants.KEY_VALIDATE: 'true'})
# Run the upload. This should be successful
report = self.iso_importer.upload_unit(repo, ids.TYPE_ID_ISO, unit_key,
metadata, temp_file_location,
sync_conduit, config)
self.assertEqual(report['success_flag'], True)
self.assertEqual(report['summary'], None)
# The conduit's init_unit method should have been called
expected_rel_path = os.path.join(unit_key['name'],
unit_key['checksum'],
str(unit_key['size']),
unit_key['name'])
modified_metadata = metadata.copy()
modified_metadata[server_constants.PULP_USER_METADATA_FIELDNAME] = {}
sync_conduit.init_unit.assert_called_once_with(ids.TYPE_ID_ISO,
unit_key,
modified_metadata,
expected_rel_path)
# The file should have been moved to its final destination
self.assertFalse(os.path.exists(temp_file_location))
expected_destination = os.path.join(pkg_dir, expected_rel_path)
self.assertTrue(os.path.exists(expected_destination))
with open(expected_destination) as iso_file:
self.assertEqual(iso_file.read(), file_data)
# The conduit's save_unit method should have been called
self.assertEqual(sync_conduit.save_unit.call_count, 1)
saved_unit = sync_conduit.save_unit.mock_calls[0][1][0]
self.assertEqual(saved_unit.unit_key, unit_key)
def test_sync_no_feed(self):
repo = mock.MagicMock(spec=Repository)
pkg_dir = os.path.join(self.temp_dir, "content")
sync_conduit = importer_mocks.get_sync_conduit(pkg_dir=pkg_dir)
config = {importer_constants.KEY_FEED: None}
config = importer_mocks.get_basic_config(**config)
# Now run the sync
self.assertRaises(ValueError, self.iso_importer.sync_repo, repo, sync_conduit, config)
def test_sync_no_feed(self):
repo = mock.MagicMock(spec=Repository)
pkg_dir = os.path.join(self.temp_dir, 'content')
sync_conduit = importer_mocks.get_sync_conduit(pkg_dir=pkg_dir)
config = {importer_constants.KEY_FEED: None}
config = importer_mocks.get_basic_config(**config)
# Now run the sync
self.assertRaises(ValueError, self.iso_importer.sync_repo, repo, sync_conduit, config)
def test_upload_unit_validate_false(self, validate):
"""
Assert correct behavior from upload_unit() when the validation setting is False.
"""
# Set up the test
file_data = "This is a file.\n"
working_dir = os.path.join(self.temp_dir, "working")
os.mkdir(working_dir)
pkg_dir = os.path.join(self.temp_dir, "content")
os.mkdir(pkg_dir)
repo = mock.MagicMock(spec=Repository)
repo.working_dir = working_dir
# Set the checksum incorrect. The upload should be successful no matter what since
# validation will be set to False
unit_key = {"name": "test.iso", "size": 16, "checksum": "Wrong"}
metadata = {}
temp_file_location = os.path.join(self.temp_dir, "test.iso")
with open(temp_file_location, "w") as temp_file:
temp_file.write(file_data)
sync_conduit = importer_mocks.get_sync_conduit(pkg_dir=pkg_dir)
config = importer_mocks.get_basic_config(**{importer_constants.KEY_VALIDATE: "false"})
# Run the upload. This should be successful, since we have set validation off.
report = self.iso_importer.upload_unit(
repo, ids.TYPE_ID_ISO, unit_key, metadata, temp_file_location, sync_conduit, config
)
# The import should have been successful
self.assertEqual(report["success_flag"], True)
self.assertEqual(report["summary"], None)
# The conduit's init_unit method should have been called
expected_rel_path = os.path.join(
unit_key["name"], unit_key["checksum"], str(unit_key["size"]), unit_key["name"]
)
modified_metadata = metadata.copy()
modified_metadata[server_constants.PULP_USER_METADATA_FIELDNAME] = {}
sync_conduit.init_unit.assert_called_once_with(ids.TYPE_ID_ISO, unit_key, modified_metadata, expected_rel_path)
# The file should have been moved to its final destination
self.assertFalse(os.path.exists(temp_file_location))
expected_destination = os.path.join(pkg_dir, expected_rel_path)
self.assertTrue(os.path.exists(expected_destination))
with open(expected_destination) as iso_file:
self.assertEqual(iso_file.read(), file_data)
# validate() should still have been called, but with the full_validation=False flag
# We need to get the ISO itself for our assertion, since it is technically the first
# argument
iso = validate.mock_calls[0][1][0]
validate.assert_called_once_with(iso, full_validation=False)
# The conduit's save_unit method should have been called
self.assertEqual(sync_conduit.save_unit.call_count, 1)
saved_unit = sync_conduit.save_unit.mock_calls[0][1][0]
self.assertEqual(saved_unit.unit_key, unit_key)
def test_upload_unit_validate_unset(self, remove, validate):
"""
Assert correct behavior from upload_unit() when the validation setting is not set. This
should default to validating the upload.
"""
# Set up the test
file_data = 'This is a file.\n'
working_dir = os.path.join(self.temp_dir, "working")
os.mkdir(working_dir)
pkg_dir = os.path.join(self.temp_dir, 'content')
os.mkdir(pkg_dir)
repo = mock.MagicMock(spec=Repository)
repo.working_dir = working_dir
# Set the checksum incorrect. The upload should be unsuccessful since the default is to
# validate
unit_key = {'name': 'test.iso', 'size': 16, 'checksum': 'Wrong'}
metadata = {}
temp_file_location = os.path.join(self.temp_dir, 'test.iso')
with open(temp_file_location, 'w') as temp_file:
temp_file.write(file_data)
sync_conduit = importer_mocks.get_sync_conduit(pkg_dir=pkg_dir)
# validate isn't set, so default should happen
config = importer_mocks.get_basic_config()
# Run the upload. This should report a failure
report = self.iso_importer.upload_unit(repo, ids.TYPE_ID_ISO, unit_key, metadata,
temp_file_location, sync_conduit, config)
self.assertEqual(report['success_flag'], False)
self.assertEqual(
report['summary'],
('Downloading <test.iso> failed checksum validation. The manifest specified the '
'checksum to be Wrong, but it was '
'f02d5a72cd2d57fa802840a76b44c6c6920a8b8e6b90b20e26c03876275069e0.'))
# The conduit's init_unit method should have been called
expected_rel_path = os.path.join(unit_key['name'], unit_key['checksum'],
str(unit_key['size']), unit_key['name'])
modified_metadata = metadata.copy()
modified_metadata[server_constants.PULP_USER_METADATA_FIELDNAME] = {}
sync_conduit.init_unit.assert_called_once_with(ids.TYPE_ID_ISO, unit_key, modified_metadata,
expected_rel_path)
# The file should have been moved to its final destination
self.assertFalse(os.path.exists(temp_file_location))
would_be_destination = os.path.join(pkg_dir, expected_rel_path)
self.assertFalse(os.path.exists(would_be_destination))
# The file should have been removed
remove.assert_called_once_with(would_be_destination)
# validate() should have been called with the full_validation=True flag
iso = validate.mock_calls[0][1][0]
validate.assert_called_once_with(iso, full_validation=True)
# The conduit's save_unit method should have been called
self.assertEqual(sync_conduit.save_unit.call_count, 0)
def test_sync_calls_sync(self, mock_sync_run):
repo = Repository("repo1")
sync_conduit = importer_mocks.get_sync_conduit(pkg_dir="/a/b/c")
config = importer_mocks.get_basic_config(**{importer_constants.KEY_FEED: "http://fake.com/iso_feed/"})
self.iso_importer.sync_repo(repo, sync_conduit, config)
# make sure the sync workflow is called with the right stuff
mock_sync_run.assert_called_once_with(sync_conduit, config)
mock_sync_run.return_value.perform_sync.assert_called_once_with()
def test_upload_unit_validate_false(self, validate):
"""
Assert correct behavior from upload_unit() when the validation setting is False.
"""
# Set up the test
file_data = 'This is a file.\n'
working_dir = os.path.join(self.temp_dir, "working")
os.mkdir(working_dir)
pkg_dir = os.path.join(self.temp_dir, 'content')
os.mkdir(pkg_dir)
repo = mock.MagicMock(spec=Repository)
repo.working_dir = working_dir
# Set the checksum incorrect. The upload should be successful no matter what since
# validation will be set to False
unit_key = {'name': 'test.iso', 'size': 16, 'checksum': 'Wrong'}
metadata = {}
temp_file_location = os.path.join(self.temp_dir, 'test.iso')
with open(temp_file_location, 'w') as temp_file:
temp_file.write(file_data)
sync_conduit = importer_mocks.get_sync_conduit(pkg_dir=pkg_dir)
config = importer_mocks.get_basic_config(**{importer_constants.KEY_VALIDATE: 'false'})
# Run the upload. This should be successful, since we have set validation off.
report = self.iso_importer.upload_unit(repo, ids.TYPE_ID_ISO, unit_key, metadata,
temp_file_location, sync_conduit, config)
# The import should have been successful
self.assertEqual(report['success_flag'], True)
self.assertEqual(report['summary'], None)
# The conduit's init_unit method should have been called
expected_rel_path = os.path.join(unit_key['name'], unit_key['checksum'],
str(unit_key['size']), unit_key['name'])
modified_metadata = metadata.copy()
modified_metadata[server_constants.PULP_USER_METADATA_FIELDNAME] = {}
sync_conduit.init_unit.assert_called_once_with(ids.TYPE_ID_ISO, unit_key, modified_metadata,
expected_rel_path)
# The file should have been moved to its final destination
self.assertFalse(os.path.exists(temp_file_location))
expected_destination = os.path.join(pkg_dir, expected_rel_path)
self.assertTrue(os.path.exists(expected_destination))
with open(expected_destination) as iso_file:
self.assertEqual(iso_file.read(), file_data)
# validate() should still have been called, but with the full_validation=False flag
# We need to get the ISO itself for our assertion, since it is technically the first
# argument
iso = validate.mock_calls[0][1][0]
validate.assert_called_once_with(iso, full_validation=False)
# The conduit's save_unit method should have been called
self.assertEqual(sync_conduit.save_unit.call_count, 1)
saved_unit = sync_conduit.save_unit.mock_calls[0][1][0]
self.assertEqual(saved_unit.unit_key, unit_key)
def test_upload_unit_validate_true_bad_checksum(self, remove, validate):
"""
Test behavior with a bad checksum.
"""
# Set up the test
file_data = 'This is a file.\n'
error_message = 'uh oh'
validate.side_effect = ValueError(error_message)
working_dir = os.path.join(self.temp_dir, "working")
os.mkdir(working_dir)
pkg_dir = os.path.join(self.temp_dir, 'content')
os.mkdir(pkg_dir)
repo = mock.MagicMock(spec=Repository)
repo.working_dir = working_dir
# Set the checksum incorrect. The upload should fail.
unit_key = {'name': 'test.iso', 'size': 16, 'checksum': 'Wrong'}
metadata = {}
temp_file_location = os.path.join(self.temp_dir, 'test.iso')
with open(temp_file_location, 'w') as temp_file:
temp_file.write(file_data)
sync_conduit = importer_mocks.get_sync_conduit(pkg_dir=pkg_dir)
config = importer_mocks.get_basic_config(
**{importer_constants.KEY_VALIDATE: 'true'})
# Run the upload. This should fail due to the bad checksum
report = self.iso_importer.upload_unit(repo, ids.TYPE_ID_ISO, unit_key,
metadata, temp_file_location,
sync_conduit, config)
self.assertEqual(report['success_flag'], False)
self.assertEqual(report['summary'], error_message)
# The conduit's init_unit method should have been called
expected_rel_path = os.path.join(unit_key['name'],
unit_key['checksum'],
str(unit_key['size']),
unit_key['name'])
modified_metadata = metadata.copy()
modified_metadata[server_constants.PULP_USER_METADATA_FIELDNAME] = {}
sync_conduit.init_unit.assert_called_once_with(ids.TYPE_ID_ISO,
unit_key,
modified_metadata,
expected_rel_path)
# The file should have been deleted
self.assertFalse(os.path.exists(temp_file_location))
would_be_destination = os.path.join(pkg_dir, expected_rel_path)
self.assertFalse(os.path.exists(would_be_destination))
# The file should have been removed from there
remove.assert_called_once_with(would_be_destination)
# validate() should have been called with the full_validation=True flag
validate.assert_called_once_with(full_validation=True)
# The conduit's save_unit method should not have been called
self.assertEqual(sync_conduit.save_unit.call_count, 0)
def test_upload_unit_named_PULP_MANIFEST(self, remove):
"""
We had a bug[0] due to the ISOImporter allowing units to be uploaded named PULP_MANIFEST.
This test asserts that that is no longer allowed.
[0] https://bugzilla.redhat.com/show_bug.cgi?id=973678
"""
# Set up the test
file_data = "This is a PULP_MANIFEST file. The upload should be rejected.\n"
working_dir = os.path.join(self.temp_dir, "working")
os.mkdir(working_dir)
pkg_dir = os.path.join(self.temp_dir, "content")
os.mkdir(pkg_dir)
repo = mock.MagicMock(spec=Repository)
repo.working_dir = working_dir
# We'll set validation off so the checksum doesn't matter
unit_key = {"name": "PULP_MANIFEST", "size": len(file_data), "checksum": "Doesn't matter"}
metadata = {}
temp_file_location = os.path.join(self.temp_dir, unit_key["name"])
with open(temp_file_location, "w") as temp_file:
temp_file.write(file_data)
sync_conduit = importer_mocks.get_sync_conduit(pkg_dir=pkg_dir)
# Just so we don't have to care about the checksum
config = importer_mocks.get_basic_config(**{importer_constants.KEY_VALIDATE: "false"})
report = self.iso_importer.upload_unit(
repo, ids.TYPE_ID_ISO, unit_key, metadata, temp_file_location, sync_conduit, config
)
self.assertEqual(report["success_flag"], False)
self.assertEqual(
report["summary"],
"An ISO may not be named PULP_MANIFEST, as it "
"conflicts with the name of the manifest during "
"publishing.",
)
# init_unit() should have been called
expected_rel_path = os.path.join(
unit_key["name"], unit_key["checksum"], str(unit_key["size"]), unit_key["name"]
)
modified_metadata = metadata.copy()
modified_metadata[server_constants.PULP_USER_METADATA_FIELDNAME] = {}
sync_conduit.init_unit.assert_called_once_with(ids.TYPE_ID_ISO, unit_key, modified_metadata, expected_rel_path)
# The file should have been deleted
self.assertFalse(os.path.exists(temp_file_location))
would_be_destination = os.path.join(pkg_dir, expected_rel_path)
self.assertFalse(os.path.exists(would_be_destination))
# The file should have been removed from there
remove.assert_called_once_with(would_be_destination)
# The conduit's save_unit method should not have been called
self.assertEqual(sync_conduit.save_unit.call_count, 0)
def test_sync_calls_sync(self, mock_sync_run):
repo = Repository('repo1')
sync_conduit = importer_mocks.get_sync_conduit(pkg_dir='/a/b/c')
config = importer_mocks.get_basic_config(
**{importer_constants.KEY_FEED: 'http://fake.com/iso_feed/'})
self.iso_importer.sync_repo(repo, sync_conduit, config)
# make sure the sync workflow is called with the right stuff
mock_sync_run.assert_called_once_with(sync_conduit, config)
mock_sync_run.return_value.perform_sync.assert_called_once_with()
def test_upload_unit_validate_unset(self, remove, validate):
"""
Assert correct behavior from upload_unit() when the validation setting is not set. This
should default to validating the upload.
"""
# Set up the test
file_data = 'This is a file.\n'
working_dir = os.path.join(self.temp_dir, "working")
os.mkdir(working_dir)
pkg_dir = os.path.join(self.temp_dir, 'content')
os.mkdir(pkg_dir)
repo = mock.MagicMock(spec=Repository)
repo.working_dir = working_dir
# Set the checksum incorrect. The upload should be unsuccessful since the default is to
# validate
unit_key = {'name': 'test.iso', 'size': 16, 'checksum': 'Wrong'}
metadata = {}
temp_file_location = os.path.join(self.temp_dir, 'test.iso')
with open(temp_file_location, 'w') as temp_file:
temp_file.write(file_data)
sync_conduit = importer_mocks.get_sync_conduit(type_id=ids.TYPE_ID_ISO, pkg_dir=pkg_dir)
# validate isn't set, so default should happen
config = importer_mocks.get_basic_config()
# Run the upload. This should report a failure
report = self.iso_importer.upload_unit(repo, ids.TYPE_ID_ISO, unit_key, metadata,
temp_file_location, sync_conduit, config)
self.assertEqual(report['success_flag'], False)
self.assertEqual(
report['summary'],
('Downloading <test.iso> failed checksum validation. The manifest specified the '
'checksum to be Wrong, but it was '
'f02d5a72cd2d57fa802840a76b44c6c6920a8b8e6b90b20e26c03876275069e0.'))
# The conduit's init_unit method should have been called
expected_rel_path = os.path.join(unit_key['name'], unit_key['checksum'],
str(unit_key['size']), unit_key['name'])
sync_conduit.init_unit.assert_called_once_with(ids.TYPE_ID_ISO, unit_key, metadata,
expected_rel_path)
# The file should have been moved to its final destination
self.assertFalse(os.path.exists(temp_file_location))
would_be_destination = os.path.join(pkg_dir, expected_rel_path)
self.assertFalse(os.path.exists(would_be_destination))
# The file should have been removed
remove.assert_called_once_with(would_be_destination)
# validate() should have been called with the full_validation=True flag
iso = validate.mock_calls[0][1][0]
validate.assert_called_once_with(iso, full_validation=True)
# The conduit's save_unit method should have been called
self.assertEqual(sync_conduit.save_unit.call_count, 0)
def test_upload_unit_validate_true_good_checksum(self):
"""
Test behavior with good arguments.
"""
# Set up the test
file_data = "This is a file.\n"
working_dir = os.path.join(self.temp_dir, "working")
os.mkdir(working_dir)
pkg_dir = os.path.join(self.temp_dir, "content")
os.mkdir(pkg_dir)
repo = mock.MagicMock(spec=Repository)
repo.working_dir = working_dir
unit_key = {
"name": "test.iso",
"size": 16,
"checksum": "f02d5a72cd2d57fa802840a76b44c6c6920a8b8e6b90b20e26c03876275069e0",
}
metadata = {}
temp_file_location = os.path.join(self.temp_dir, "test.iso")
with open(temp_file_location, "w") as temp_file:
temp_file.write(file_data)
sync_conduit = importer_mocks.get_sync_conduit(pkg_dir=pkg_dir)
config = importer_mocks.get_basic_config(**{importer_constants.KEY_VALIDATE: "true"})
# Run the upload. This should be successful
report = self.iso_importer.upload_unit(
repo, ids.TYPE_ID_ISO, unit_key, metadata, temp_file_location, sync_conduit, config
)
self.assertEqual(report["success_flag"], True)
self.assertEqual(report["summary"], None)
# The conduit's init_unit method should have been called
expected_rel_path = os.path.join(
unit_key["name"], unit_key["checksum"], str(unit_key["size"]), unit_key["name"]
)
modified_metadata = metadata.copy()
modified_metadata[server_constants.PULP_USER_METADATA_FIELDNAME] = {}
sync_conduit.init_unit.assert_called_once_with(ids.TYPE_ID_ISO, unit_key, modified_metadata, expected_rel_path)
# The file should have been moved to its final destination
self.assertFalse(os.path.exists(temp_file_location))
expected_destination = os.path.join(pkg_dir, expected_rel_path)
self.assertTrue(os.path.exists(expected_destination))
with open(expected_destination) as iso_file:
self.assertEqual(iso_file.read(), file_data)
# The conduit's save_unit method should have been called
self.assertEqual(sync_conduit.save_unit.call_count, 1)
saved_unit = sync_conduit.save_unit.mock_calls[0][1][0]
self.assertEqual(saved_unit.unit_key, unit_key)
def test_upload_unit_named_PULP_MANIFEST(self, remove):
"""
We had a bug[0] due to the ISOImporter allowing units to be uploaded named PULP_MANIFEST.
This test asserts that that is no longer allowed.
[0] https://bugzilla.redhat.com/show_bug.cgi?id=973678
"""
# Set up the test
file_data = 'This is a PULP_MANIFEST file. The upload should be rejected.\n'
working_dir = os.path.join(self.temp_dir, "working")
os.mkdir(working_dir)
pkg_dir = os.path.join(self.temp_dir, 'content')
os.mkdir(pkg_dir)
repo = mock.MagicMock(spec=Repository)
repo.working_dir = working_dir
# We'll set validation off so the checksum doesn't matter
unit_key = {'name': 'PULP_MANIFEST', 'size': len(file_data), 'checksum': "Doesn't matter"}
metadata = {}
temp_file_location = os.path.join(self.temp_dir, unit_key['name'])
with open(temp_file_location, 'w') as temp_file:
temp_file.write(file_data)
sync_conduit = importer_mocks.get_sync_conduit(pkg_dir=pkg_dir)
# Just so we don't have to care about the checksum
config = importer_mocks.get_basic_config(**{importer_constants.KEY_VALIDATE: 'false'})
report = self.iso_importer.upload_unit(repo, ids.TYPE_ID_ISO, unit_key, metadata,
temp_file_location, sync_conduit, config)
self.assertEqual(report['success_flag'], False)
self.assertEqual(report['summary'], 'An ISO may not be named PULP_MANIFEST, as it '
'conflicts with the name of the manifest during '
'publishing.')
# init_unit() should have been called
expected_rel_path = os.path.join(unit_key['name'], unit_key['checksum'],
str(unit_key['size']), unit_key['name'])
modified_metadata = metadata.copy()
modified_metadata[server_constants.PULP_USER_METADATA_FIELDNAME] = {}
sync_conduit.init_unit.assert_called_once_with(ids.TYPE_ID_ISO, unit_key, modified_metadata,
expected_rel_path)
# The file should have been deleted
self.assertFalse(os.path.exists(temp_file_location))
would_be_destination = os.path.join(pkg_dir, expected_rel_path)
self.assertFalse(os.path.exists(would_be_destination))
# The file should have been removed from there
remove.assert_called_once_with(would_be_destination)
# The conduit's save_unit method should not have been called
self.assertEqual(sync_conduit.save_unit.call_count, 0)
def test_upload_unit_validate_true_bad_checksum(self, remove, validate):
"""
Test behavior with a bad checksum.
"""
# Set up the test
file_data = "This is a file.\n"
error_message = "uh oh"
validate.side_effect = ValueError(error_message)
working_dir = os.path.join(self.temp_dir, "working")
os.mkdir(working_dir)
pkg_dir = os.path.join(self.temp_dir, "content")
os.mkdir(pkg_dir)
repo = mock.MagicMock(spec=Repository)
repo.working_dir = working_dir
# Set the checksum incorrect. The upload should fail.
unit_key = {"name": "test.iso", "size": 16, "checksum": "Wrong"}
metadata = {}
temp_file_location = os.path.join(self.temp_dir, "test.iso")
with open(temp_file_location, "w") as temp_file:
temp_file.write(file_data)
sync_conduit = importer_mocks.get_sync_conduit(pkg_dir=pkg_dir)
config = importer_mocks.get_basic_config(**{importer_constants.KEY_VALIDATE: "true"})
# Run the upload. This should fail due to the bad checksum
report = self.iso_importer.upload_unit(
repo, ids.TYPE_ID_ISO, unit_key, metadata, temp_file_location, sync_conduit, config
)
self.assertEqual(report["success_flag"], False)
self.assertEqual(report["summary"], error_message)
# The conduit's init_unit method should have been called
expected_rel_path = os.path.join(
unit_key["name"], unit_key["checksum"], str(unit_key["size"]), unit_key["name"]
)
modified_metadata = metadata.copy()
modified_metadata[server_constants.PULP_USER_METADATA_FIELDNAME] = {}
sync_conduit.init_unit.assert_called_once_with(ids.TYPE_ID_ISO, unit_key, modified_metadata, expected_rel_path)
# The file should have been deleted
self.assertFalse(os.path.exists(temp_file_location))
would_be_destination = os.path.join(pkg_dir, expected_rel_path)
self.assertFalse(os.path.exists(would_be_destination))
# The file should have been removed from there
remove.assert_called_once_with(would_be_destination)
# validate() should have been called with the full_validation=True flag
validate.assert_called_once_with(full_validation=True)
# The conduit's save_unit method should not have been called
self.assertEqual(sync_conduit.save_unit.call_count, 0)
def test_upload_unit_validate_true_bad_checksum(self, remove, validate):
"""
Test behavior with a bad checksum.
"""
# Set up the test
file_data = 'This is a file.\n'
error_message = 'uh oh'
validate.side_effect = ValueError(error_message)
working_dir = os.path.join(self.temp_dir, "working")
os.mkdir(working_dir)
pkg_dir = os.path.join(self.temp_dir, 'content')
os.mkdir(pkg_dir)
repo = mock.MagicMock(spec=Repository)
repo.working_dir = working_dir
# Set the checksum incorrect. The upload should fail.
unit_key = {'name': 'test.iso', 'size': 16, 'checksum': 'Wrong'}
metadata = {}
temp_file_location = os.path.join(self.temp_dir, 'test.iso')
with open(temp_file_location, 'w') as temp_file:
temp_file.write(file_data)
sync_conduit = importer_mocks.get_sync_conduit(type_id=ids.TYPE_ID_ISO, pkg_dir=pkg_dir)
config = importer_mocks.get_basic_config(**{importer_constants.KEY_VALIDATE: 'true'})
# Run the upload. This should fail due to the bad checksum
report = self.iso_importer.upload_unit(repo, ids.TYPE_ID_ISO, unit_key, metadata,
temp_file_location, sync_conduit, config)
self.assertEqual(report['success_flag'], False)
self.assertEqual(report['summary'], error_message)
# The conduit's init_unit method should have been called
expected_rel_path = os.path.join(unit_key['name'], unit_key['checksum'],
str(unit_key['size']), unit_key['name'])
sync_conduit.init_unit.assert_called_once_with(ids.TYPE_ID_ISO, unit_key, metadata,
expected_rel_path)
# The file should have been deleted
self.assertFalse(os.path.exists(temp_file_location))
would_be_destination = os.path.join(pkg_dir, expected_rel_path)
self.assertFalse(os.path.exists(would_be_destination))
# The file should have been removed from there
remove.assert_called_once_with(would_be_destination)
# validate() should have been called with the full_validation=True flag
validate.assert_called_once_with(full_validation=True)
# The conduit's save_unit method should not have been called
self.assertEqual(sync_conduit.save_unit.call_count, 0)
def test_upload_unit_validate_true_good_checksum(self):
"""
Test behavior with good arguments.
"""
# Set up the test
file_data = 'This is a file.\n'
working_dir = os.path.join(self.temp_dir, "working")
os.mkdir(working_dir)
pkg_dir = os.path.join(self.temp_dir, 'content')
os.mkdir(pkg_dir)
repo = mock.MagicMock(spec=Repository)
repo.working_dir = working_dir
unit_key = {'name': 'test.iso', 'size': 16,
'checksum': 'f02d5a72cd2d57fa802840a76b44c6c6920a8b8e6b90b20e26c03876275069e0'}
metadata = {}
temp_file_location = os.path.join(self.temp_dir, 'test.iso')
with open(temp_file_location, 'w') as temp_file:
temp_file.write(file_data)
sync_conduit = importer_mocks.get_sync_conduit(type_id=ids.TYPE_ID_ISO, pkg_dir=pkg_dir)
config = importer_mocks.get_basic_config(**{importer_constants.KEY_VALIDATE: 'true'})
# Run the upload. This should be successful
report = self.iso_importer.upload_unit(repo, ids.TYPE_ID_ISO, unit_key, metadata,
temp_file_location, sync_conduit, config)
self.assertEqual(report['success_flag'], True)
self.assertEqual(report['summary'], None)
# The conduit's init_unit method should have been called
expected_rel_path = os.path.join(unit_key['name'], unit_key['checksum'],
str(unit_key['size']), unit_key['name'])
sync_conduit.init_unit.assert_called_once_with(ids.TYPE_ID_ISO, unit_key, metadata,
expected_rel_path)
# The file should have been moved to its final destination
self.assertFalse(os.path.exists(temp_file_location))
expected_destination = os.path.join(pkg_dir, expected_rel_path)
self.assertTrue(os.path.exists(expected_destination))
with open(expected_destination) as iso_file:
self.assertEqual(iso_file.read(), file_data)
# The conduit's save_unit method should have been called
self.assertEqual(sync_conduit.save_unit.call_count, 1)
saved_unit = sync_conduit.save_unit.mock_calls[0][1][0]
self.assertEqual(saved_unit.unit_key, unit_key)
def setUp(self):
config = {
importer_constants.KEY_FEED: 'http://fake.com/iso_feed/',
importer_constants.KEY_MAX_SPEED: 500.0,
importer_constants.KEY_MAX_DOWNLOADS: 5,
importer_constants.KEY_SSL_VALIDATION: False,
importer_constants.KEY_SSL_CLIENT_CERT: "Trust me, I'm who I say I am.",
importer_constants.KEY_SSL_CLIENT_KEY: "Secret Key",
importer_constants.KEY_SSL_CA_CERT: "Uh, I guess that's the right server.",
importer_constants.KEY_PROXY_HOST: 'proxy.com',
importer_constants.KEY_PROXY_PORT: 1234,
importer_constants.KEY_PROXY_USER: "******",
importer_constants.KEY_PROXY_PASS: '******',
importer_constants.KEY_VALIDATE: False,
}
self.config = importer_mocks.get_basic_config(**config)
self.temp_dir = tempfile.mkdtemp()
self.pkg_dir = os.path.join(self.temp_dir, 'content')
os.mkdir(self.pkg_dir)
# These checksums correspond to the checksums of the files that our curl mocks will
# generate. Our curl mocks do not have a test4.iso, so that one is to test removal of
# old ISOs during sync
self.existing_units = [
Unit(TYPE_ID_ISO,
{'name': 'test.iso', 'size': 16,
'checksum': 'f02d5a72cd2d57fa802840a76b44c6c6920a8b8e6b90b20e26c03876275069e0'},
{}, '/path/test.iso'),
Unit(TYPE_ID_ISO,
{'name': 'test2.iso', 'size': 22,
'checksum': 'c7fbc0e821c0871805a99584c6a384533909f68a6bbe9a2a687d28d9f3b10c16'},
{}, '/path/test2.iso'),
Unit(TYPE_ID_ISO, {'name': 'test4.iso', 'size': 4, 'checksum': 'sum4'},
{}, '/path/test4.iso')]
self.sync_conduit = importer_mocks.get_sync_conduit(pkg_dir=self.pkg_dir,
existing_units=self.existing_units,
pulp_units=self.existing_units)
self.iso_sync_run = ISOSyncRun(self.sync_conduit, self.config)
def setUp(self):
self.conduit = importer_mocks.get_sync_conduit()
def setUp(self):
self.conduit = importer_mocks.get_sync_conduit()
