def sync_repo(self, repo, sync_conduit, config):
"""
Synchronizes content into the given repository. This call is responsible
for adding new content units to Pulp as well as associating them to the
given repository.
While this call may be implemented using multiple threads, its execution
from the Pulp server's standpoint should be synchronous. This call should
not return until the sync is complete.
It is not expected that this call be atomic. Should an error occur, it
is not the responsibility of the importer to rollback any unit additions
or associations that have been made.
The returned report object is used to communicate the results of the
sync back to the user. Care should be taken to i18n the free text "log"
attribute in the report if applicable.
:param repo: metadata describing the repository
:type repo: pulp.plugins.model.Repository
:param sync_conduit: provides access to relevant Pulp functionality
:type sync_conduit: pulp.plugins.conduits.repo_sync.RepoSyncConduit
:param config: plugin configuration
:type config: pulp.plugins.config.PluginCallConfiguration
:return: report of the details of the sync
:rtype: pulp.plugins.model.SyncReport
"""
self.sync_step = sync.SyncStep(repo=repo,
conduit=sync_conduit,
config=config)
return self.sync_step.process_lifecycle()
def test_generate_download_requests(self, _working_directory_path):
"""
Assert correct operation of the generate_download_requests() method.
"""
_working_directory_path.return_value = self.working_dir
repo = mock.MagicMock()
conduit = mock.MagicMock()
config = plugin_config.PluginCallConfiguration(
{}, {
'feed': 'https://registry.example.com',
'upstream_name': 'busybox',
importer_constants.KEY_MAX_DOWNLOADS: 25
})
step = sync.SyncStep(repo, conduit, config)
step.step_get_local_blobs.units_to_download = [
models.Blob(digest=i) for i in ['cool', 'stuff']
]
requests = step.generate_download_requests()
requests = list(requests)
self.assertEqual(len(requests), 2)
self.assertEqual(requests[0].url,
'https://registry.example.com/v2/busybox/blobs/cool')
self.assertEqual(requests[0].destination,
os.path.join(self.working_dir, 'cool'))
self.assertEqual(requests[0].data, None)
self.assertEqual(requests[0].headers, None)
self.assertEqual(
requests[1].url,
'https://registry.example.com/v2/busybox/blobs/stuff')
self.assertEqual(requests[1].destination,
os.path.join(self.working_dir, 'stuff'))
self.assertEqual(requests[1].data, None)
self.assertEqual(requests[1].headers, None)
def test_init_v1(self, mock_check_v1, mock_check_v2, mock_validate,
_working_directory_path):
_working_directory_path.return_value = self.working_dir
# re-run this with the mock in place
self.config.override_config[constants.CONFIG_KEY_ENABLE_V1] = True
step = sync.SyncStep(self.repo, self.conduit, self.config)
self.assertEqual(step.step_id, constants.SYNC_STEP_MAIN)
# make sure the children are present
step_ids = set([child.step_id for child in step.children])
self.assertTrue(constants.SYNC_STEP_METADATA_V1 in step_ids)
self.assertTrue(constants.SYNC_STEP_GET_LOCAL_V1 in step_ids)
self.assertTrue(constants.SYNC_STEP_DOWNLOAD_V1 in step_ids)
self.assertTrue(constants.SYNC_STEP_SAVE_V1 in step_ids)
# make sure it instantiated a Repository object
self.assertTrue(
isinstance(step.v1_index_repository, registry.V1Repository))
self.assertEqual(step.v1_index_repository.name, 'pulp/crane')
self.assertEqual(step.v1_index_repository.registry_url,
'http://pulpproject.org/')
# these are important because child steps will populate them with data
self.assertEqual(step.v1_available_units, [])
self.assertEqual(step.v1_tags, {})
mock_validate.assert_called_once_with(self.config)
def setUp(self):
super(TestSyncStep, self).setUp()
self.repo = RepositoryModel('repo1')
self.conduit = mock.MagicMock()
plugin_config = {
constants.CONFIG_KEY_UPSTREAM_NAME: 'pulp/crane',
importer_constants.KEY_FEED: 'http://pulpproject.org/',
}
self.config = PluginCallConfiguration({}, plugin_config)
self.step = sync.SyncStep(self.repo, self.conduit, self.config, '/a/b/c')
def test_generate_download_reqs_existing_dir(self, mock_working_dir,
mock_v1_check, mock_v2_check):
mock_working_dir.return_value = tempfile.mkdtemp()
self.config.override_config[constants.CONFIG_KEY_ENABLE_V1] = True
step = sync.SyncStep(self.repo, self.conduit, self.config)
step.v1_step_get_local_units.units_to_download.append(
models.Image(image_id='image1'))
os.makedirs(os.path.join(step.working_dir, 'image1'))
try:
# just make sure this doesn't complain
list(step.v1_generate_download_requests())
finally:
shutil.rmtree(step.working_dir)
def test_generate_download_reqs_perm_denied(self, mock_working_dir,
mock_v1_check, mock_v2_check):
mock_working_dir.return_value = tempfile.mkdtemp()
self.config.override_config[constants.CONFIG_KEY_ENABLE_V1] = True
try:
step = sync.SyncStep(self.repo, self.conduit, self.config)
step.v1_step_get_local_units.units_to_download.append(
models.Image(image_id='image1'))
step.working_dir = '/not/allowed'
# make sure the permission denies OSError bubbles up
self.assertRaises(OSError, list,
step.v1_generate_download_requests())
finally:
shutil.rmtree(mock_working_dir.return_value)
def test_generate_download_reqs_creates_dir(self, mock_working_dir,
mock_v1_check, mock_v2_check):
mock_working_dir.return_value = tempfile.mkdtemp()
self.config.override_config[constants.CONFIG_KEY_ENABLE_V1] = True
step = sync.SyncStep(self.repo, self.conduit, self.config)
step.v1_step_get_local_units.units_to_download.append(
models.Image(image_id='image1'))
try:
list(step.v1_generate_download_requests())
# make sure it created the destination directory
self.assertTrue(
os.path.isdir(os.path.join(step.working_dir, 'image1')))
finally:
shutil.rmtree(step.working_dir)
def test___init___without_v2_registry(self, mock_v2_check, mock_v1_check,
_validate, _working_directory_path):
"""
Test the __init__() method when the V2Repository raises a NotImplementedError with the
api_version_check() method, indicating that the feed URL is not a Docker v2 registry.
"""
_working_directory_path.return_value = self.working_dir
repo = mock.MagicMock()
conduit = mock.MagicMock()
with self.assertRaises(PulpCodedException) as error:
sync.SyncStep(repo, conduit, self.config)
self.assertEqual(error.exception.error_code, error_codes.DKR1008)
# The config should get validated
_validate.assert_called_once_with(self.config)
def test_generate_download_reqs_ancestry_exists(self, mock_working_dir,
mock_v1_check,
mock_v2_check):
mock_working_dir.return_value = tempfile.mkdtemp()
self.config.override_config[constants.CONFIG_KEY_ENABLE_V1] = True
step = sync.SyncStep(self.repo, self.conduit, self.config)
step.v1_step_get_local_units.units_to_download.append(
models.Image(image_id='image1'))
os.makedirs(os.path.join(step.working_dir, 'image1'))
# simulate the ancestry file already existing
open(os.path.join(step.working_dir, 'image1/ancestry'), 'w').close()
try:
# there should only be 2 reqs instead of 3, since the ancestry file already exists
reqs = list(step.v1_generate_download_requests())
self.assertEqual(len(reqs), 2)
finally:
shutil.rmtree(step.working_dir)
def test___init__nothing_enabled(self, v1_check, v2_check, validate,
working_dir_path):
"""
Test when both v1 and v2 are disabled, PulpCodedException is raised.
"""
working_dir_path.return_value = self.working_dir
repo = mock.MagicMock()
conduit = mock.MagicMock()
self.config.override_config[constants.CONFIG_KEY_ENABLE_V1] = False
self.config.override_config[constants.CONFIG_KEY_ENABLE_V2] = False
with self.assertRaises(PulpCodedException) as error:
sync.SyncStep(repo, conduit, self.config)
validate.assert_called_once_with(self.config)
self.assertEqual(error.exception.error_code, error_codes.DKR1008)
self.assertFalse(v1_check.called)
self.assertFalse(v2_check.called)
def test_v1_generate_download_requests(self, mock_working_dir,
mock_v1_check, mock_v2_check):
mock_working_dir.return_value = tempfile.mkdtemp()
self.config.override_config[constants.CONFIG_KEY_ENABLE_V1] = True
step = sync.SyncStep(self.repo, self.conduit, self.config)
step.v1_step_get_local_units.units_to_download.append(
models.Image(image_id='image1'))
try:
generator = step.v1_generate_download_requests()
self.assertTrue(inspect.isgenerator(generator))
download_reqs = list(generator)
self.assertEqual(len(download_reqs), 3)
for req in download_reqs:
self.assertTrue(isinstance(req, DownloadRequest))
finally:
shutil.rmtree(step.working_dir)
def test___init___v1_and_v2_enabled(self, v2_check, v1_check, validate,
working_dir_path, add_v2_steps,
add_v1_steps):
"""
Test both v1 and v2 enabled.
"""
repo = mock.MagicMock()
conduit = mock.MagicMock()
working_dir_path.return_value = self.working_dir
self.config.override_config[constants.CONFIG_KEY_ENABLE_V1] = True
self.config.override_config[constants.CONFIG_KEY_ENABLE_V2] = True
sync.SyncStep(repo, conduit, self.config)
validate.assert_called_once_with(self.config)
add_v1_steps.assert_called_once_with(repo, self.config)
add_v2_steps.assert_called_once_with(repo, conduit, self.config)
v1_check.assert_called_once_with()
v2_check.assert_called_once_with()
def test_generate_download_requests_correct_urls(self, mock_working_dir,
mock_v1_check,
mock_v2_check):
mock_working_dir.return_value = tempfile.mkdtemp()
self.config.override_config[constants.CONFIG_KEY_ENABLE_V1] = True
step = sync.SyncStep(self.repo, self.conduit, self.config)
step.v1_step_get_local_units.units_to_download.append(
models.Image(image_id='image1'))
try:
generator = step.v1_generate_download_requests()
# make sure the urls are correct
urls = [req.url for req in generator]
self.assertTrue(
'http://pulpproject.org/v1/images/image1/ancestry' in urls)
self.assertTrue(
'http://pulpproject.org/v1/images/image1/json' in urls)
self.assertTrue(
'http://pulpproject.org/v1/images/image1/layer' in urls)
finally:
shutil.rmtree(step.working_dir)
def test_init(self, mock_validate):
# re-run this with the mock in place
self.step = sync.SyncStep(self.repo, self.conduit, self.config, '/a/b/c')
self.assertEqual(self.step.step_id, constants.SYNC_STEP_MAIN)
# make sure the children are present
step_ids = set([child.step_id for child in self.step.children])
self.assertTrue(constants.SYNC_STEP_METADATA in step_ids)
self.assertTrue(reporting_constants.SYNC_STEP_GET_LOCAL in step_ids)
self.assertTrue(constants.SYNC_STEP_DOWNLOAD in step_ids)
self.assertTrue(constants.SYNC_STEP_SAVE in step_ids)
# make sure it instantiated a Repository object
self.assertTrue(isinstance(self.step.index_repository, registry.Repository))
self.assertEqual(self.step.index_repository.name, 'pulp/crane')
self.assertEqual(self.step.index_repository.registry_url, 'http://pulpproject.org/')
# these are important because child steps will populate them with data
self.assertEqual(self.step.available_units, [])
self.assertEqual(self.step.tags, {})
mock_validate.assert_called_once_with(self.config)
def test_generate_download_requests_correct_destinations(
self, mock_working_dir, mock_v1_check, mock_v2_check):
mock_working_dir.return_value = tempfile.mkdtemp()
self.config.override_config[constants.CONFIG_KEY_ENABLE_V1] = True
step = sync.SyncStep(self.repo, self.conduit, self.config)
step.v1_step_get_local_units.units_to_download.append(
models.Image(image_id='image1'))
try:
generator = step.v1_generate_download_requests()
# make sure the urls are correct
destinations = [req.destination for req in generator]
self.assertTrue(
os.path.join(step.working_dir, 'image1', 'ancestry') in
destinations)
self.assertTrue(
os.path.join(step.working_dir, 'image1', 'json') in
destinations)
self.assertTrue(
os.path.join(step.working_dir, 'image1', 'layer') in
destinations)
finally:
shutil.rmtree(step.working_dir)
def test___init___with_v2_registry(self, v1_api_check, api_version_check,
_validate, _working_directory_path):
"""
Test the __init__() method when the V2Repository does not raise a NotImplementedError with
the api_version_check() method, indicating that the feed URL is a Docker v2 registry.
"""
_working_directory_path.return_value = self.working_dir
repo = mock.MagicMock()
conduit = mock.MagicMock()
config = plugin_config.PluginCallConfiguration(
{}, {
'feed': 'https://registry.example.com',
'upstream_name': 'busybox',
importer_constants.KEY_MAX_DOWNLOADS: 25
})
step = sync.SyncStep(repo=repo, conduit=conduit, config=config)
self.assertEqual(step.description, _('Syncing Docker Repository'))
# The config should get validated
_validate.assert_called_once_with(config)
# available_blobs should have been initialized to an empty list
self.assertEqual(step.available_blobs, [])
self.assertEqual(step.available_manifests, [])
# Ensure that the index_repository was initialized correctly
self.assertEqual(type(step.index_repository), registry.V2Repository)
self.assertEqual(step.index_repository.name, 'busybox')
self.assertEqual(step.index_repository.download_config.max_concurrent,
25)
self.assertEqual(step.index_repository.registry_url,
'https://registry.example.com')
self.assertEqual(step.index_repository.working_dir, self.working_dir)
# The version check should have happened, and since we mocked it, it will not raise an error
api_version_check.assert_called_once_with()
# The correct children should be in place in the right order
self.assertEqual([type(child) for child in step.children], [
sync.DownloadManifestsStep, publish_step.GetLocalUnitsStep,
publish_step.GetLocalUnitsStep, sync.TokenAuthDownloadStep,
sync.SaveUnitsStep, sync.SaveTagsStep
])
# Ensure the first step was initialized correctly
self.assertEqual(step.children[0].repo, repo)
self.assertEqual(step.children[0].conduit, conduit)
self.assertEqual(step.children[0].config, config)
# And the second step
self.assertTrue(step.children[1] is step.step_get_local_manifests)
self.assertEqual(step.children[1].plugin_type,
constants.IMPORTER_TYPE_ID)
self.assertEqual(step.children[1].available_units,
step.available_manifests)
# And the third step
self.assertTrue(step.children[2] is step.step_get_local_blobs)
self.assertEqual(step.children[2].plugin_type,
constants.IMPORTER_TYPE_ID)
self.assertEqual(step.children[2].available_units,
step.available_blobs)
# And the fourth
self.assertEqual(step.children[3].step_type,
constants.SYNC_STEP_DOWNLOAD)
self.assertEqual(step.children[3].repo, repo)
self.assertEqual(step.children[3].config, config)
self.assertEqual(step.children[3].description,
_('Downloading remote files'))
