def sync_repo(self, repo, sync_conduit, config): """ Synchronizes content into the given repository. This call is responsible for adding new content units to Pulp as well as associating them to the given repository. While this call may be implemented using multiple threads, its execution from the Pulp server's standpoint should be synchronous. This call should not return until the sync is complete. It is not expected that this call be atomic. Should an error occur, it is not the responsibility of the importer to rollback any unit additions or associations that have been made. The returned report object is used to communicate the results of the sync back to the user. Care should be taken to i18n the free text "log" attribute in the report if applicable. :param repo: metadata describing the repository :type repo: pulp.plugins.model.Repository :param sync_conduit: provides access to relevant Pulp functionality :type sync_conduit: pulp.plugins.conduits.repo_sync.RepoSyncConduit :param config: plugin configuration :type config: pulp.plugins.config.PluginCallConfiguration :return: report of the details of the sync :rtype: pulp.plugins.model.SyncReport """ self.sync_step = sync.SyncStep(repo=repo, conduit=sync_conduit, config=config) return self.sync_step.process_lifecycle()
def test_generate_download_requests(self, _working_directory_path): """ Assert correct operation of the generate_download_requests() method. """ _working_directory_path.return_value = self.working_dir repo = mock.MagicMock() conduit = mock.MagicMock() config = plugin_config.PluginCallConfiguration( {}, { 'feed': 'https://registry.example.com', 'upstream_name': 'busybox', importer_constants.KEY_MAX_DOWNLOADS: 25 }) step = sync.SyncStep(repo, conduit, config) step.step_get_local_blobs.units_to_download = [ models.Blob(digest=i) for i in ['cool', 'stuff'] ] requests = step.generate_download_requests() requests = list(requests) self.assertEqual(len(requests), 2) self.assertEqual(requests[0].url, 'https://registry.example.com/v2/busybox/blobs/cool') self.assertEqual(requests[0].destination, os.path.join(self.working_dir, 'cool')) self.assertEqual(requests[0].data, None) self.assertEqual(requests[0].headers, None) self.assertEqual( requests[1].url, 'https://registry.example.com/v2/busybox/blobs/stuff') self.assertEqual(requests[1].destination, os.path.join(self.working_dir, 'stuff')) self.assertEqual(requests[1].data, None) self.assertEqual(requests[1].headers, None)
def test_init_v1(self, mock_check_v1, mock_check_v2, mock_validate, _working_directory_path): _working_directory_path.return_value = self.working_dir # re-run this with the mock in place self.config.override_config[constants.CONFIG_KEY_ENABLE_V1] = True step = sync.SyncStep(self.repo, self.conduit, self.config) self.assertEqual(step.step_id, constants.SYNC_STEP_MAIN) # make sure the children are present step_ids = set([child.step_id for child in step.children]) self.assertTrue(constants.SYNC_STEP_METADATA_V1 in step_ids) self.assertTrue(constants.SYNC_STEP_GET_LOCAL_V1 in step_ids) self.assertTrue(constants.SYNC_STEP_DOWNLOAD_V1 in step_ids) self.assertTrue(constants.SYNC_STEP_SAVE_V1 in step_ids) # make sure it instantiated a Repository object self.assertTrue( isinstance(step.v1_index_repository, registry.V1Repository)) self.assertEqual(step.v1_index_repository.name, 'pulp/crane') self.assertEqual(step.v1_index_repository.registry_url, 'http://pulpproject.org/') # these are important because child steps will populate them with data self.assertEqual(step.v1_available_units, []) self.assertEqual(step.v1_tags, {}) mock_validate.assert_called_once_with(self.config)
def setUp(self): super(TestSyncStep, self).setUp() self.repo = RepositoryModel('repo1') self.conduit = mock.MagicMock() plugin_config = { constants.CONFIG_KEY_UPSTREAM_NAME: 'pulp/crane', importer_constants.KEY_FEED: 'http://pulpproject.org/', } self.config = PluginCallConfiguration({}, plugin_config) self.step = sync.SyncStep(self.repo, self.conduit, self.config, '/a/b/c')
def test_generate_download_reqs_existing_dir(self, mock_working_dir, mock_v1_check, mock_v2_check): mock_working_dir.return_value = tempfile.mkdtemp() self.config.override_config[constants.CONFIG_KEY_ENABLE_V1] = True step = sync.SyncStep(self.repo, self.conduit, self.config) step.v1_step_get_local_units.units_to_download.append( models.Image(image_id='image1')) os.makedirs(os.path.join(step.working_dir, 'image1')) try: # just make sure this doesn't complain list(step.v1_generate_download_requests()) finally: shutil.rmtree(step.working_dir)
def test_generate_download_reqs_perm_denied(self, mock_working_dir, mock_v1_check, mock_v2_check): mock_working_dir.return_value = tempfile.mkdtemp() self.config.override_config[constants.CONFIG_KEY_ENABLE_V1] = True try: step = sync.SyncStep(self.repo, self.conduit, self.config) step.v1_step_get_local_units.units_to_download.append( models.Image(image_id='image1')) step.working_dir = '/not/allowed' # make sure the permission denies OSError bubbles up self.assertRaises(OSError, list, step.v1_generate_download_requests()) finally: shutil.rmtree(mock_working_dir.return_value)
def test_generate_download_reqs_creates_dir(self, mock_working_dir, mock_v1_check, mock_v2_check): mock_working_dir.return_value = tempfile.mkdtemp() self.config.override_config[constants.CONFIG_KEY_ENABLE_V1] = True step = sync.SyncStep(self.repo, self.conduit, self.config) step.v1_step_get_local_units.units_to_download.append( models.Image(image_id='image1')) try: list(step.v1_generate_download_requests()) # make sure it created the destination directory self.assertTrue( os.path.isdir(os.path.join(step.working_dir, 'image1'))) finally: shutil.rmtree(step.working_dir)
def test___init___without_v2_registry(self, mock_v2_check, mock_v1_check, _validate, _working_directory_path): """ Test the __init__() method when the V2Repository raises a NotImplementedError with the api_version_check() method, indicating that the feed URL is not a Docker v2 registry. """ _working_directory_path.return_value = self.working_dir repo = mock.MagicMock() conduit = mock.MagicMock() with self.assertRaises(PulpCodedException) as error: sync.SyncStep(repo, conduit, self.config) self.assertEqual(error.exception.error_code, error_codes.DKR1008) # The config should get validated _validate.assert_called_once_with(self.config)
def test_generate_download_reqs_ancestry_exists(self, mock_working_dir, mock_v1_check, mock_v2_check): mock_working_dir.return_value = tempfile.mkdtemp() self.config.override_config[constants.CONFIG_KEY_ENABLE_V1] = True step = sync.SyncStep(self.repo, self.conduit, self.config) step.v1_step_get_local_units.units_to_download.append( models.Image(image_id='image1')) os.makedirs(os.path.join(step.working_dir, 'image1')) # simulate the ancestry file already existing open(os.path.join(step.working_dir, 'image1/ancestry'), 'w').close() try: # there should only be 2 reqs instead of 3, since the ancestry file already exists reqs = list(step.v1_generate_download_requests()) self.assertEqual(len(reqs), 2) finally: shutil.rmtree(step.working_dir)
def test___init__nothing_enabled(self, v1_check, v2_check, validate, working_dir_path): """ Test when both v1 and v2 are disabled, PulpCodedException is raised. """ working_dir_path.return_value = self.working_dir repo = mock.MagicMock() conduit = mock.MagicMock() self.config.override_config[constants.CONFIG_KEY_ENABLE_V1] = False self.config.override_config[constants.CONFIG_KEY_ENABLE_V2] = False with self.assertRaises(PulpCodedException) as error: sync.SyncStep(repo, conduit, self.config) validate.assert_called_once_with(self.config) self.assertEqual(error.exception.error_code, error_codes.DKR1008) self.assertFalse(v1_check.called) self.assertFalse(v2_check.called)
def test_v1_generate_download_requests(self, mock_working_dir, mock_v1_check, mock_v2_check): mock_working_dir.return_value = tempfile.mkdtemp() self.config.override_config[constants.CONFIG_KEY_ENABLE_V1] = True step = sync.SyncStep(self.repo, self.conduit, self.config) step.v1_step_get_local_units.units_to_download.append( models.Image(image_id='image1')) try: generator = step.v1_generate_download_requests() self.assertTrue(inspect.isgenerator(generator)) download_reqs = list(generator) self.assertEqual(len(download_reqs), 3) for req in download_reqs: self.assertTrue(isinstance(req, DownloadRequest)) finally: shutil.rmtree(step.working_dir)
def test___init___v1_and_v2_enabled(self, v2_check, v1_check, validate, working_dir_path, add_v2_steps, add_v1_steps): """ Test both v1 and v2 enabled. """ repo = mock.MagicMock() conduit = mock.MagicMock() working_dir_path.return_value = self.working_dir self.config.override_config[constants.CONFIG_KEY_ENABLE_V1] = True self.config.override_config[constants.CONFIG_KEY_ENABLE_V2] = True sync.SyncStep(repo, conduit, self.config) validate.assert_called_once_with(self.config) add_v1_steps.assert_called_once_with(repo, self.config) add_v2_steps.assert_called_once_with(repo, conduit, self.config) v1_check.assert_called_once_with() v2_check.assert_called_once_with()
def test_generate_download_requests_correct_urls(self, mock_working_dir, mock_v1_check, mock_v2_check): mock_working_dir.return_value = tempfile.mkdtemp() self.config.override_config[constants.CONFIG_KEY_ENABLE_V1] = True step = sync.SyncStep(self.repo, self.conduit, self.config) step.v1_step_get_local_units.units_to_download.append( models.Image(image_id='image1')) try: generator = step.v1_generate_download_requests() # make sure the urls are correct urls = [req.url for req in generator] self.assertTrue( 'http://pulpproject.org/v1/images/image1/ancestry' in urls) self.assertTrue( 'http://pulpproject.org/v1/images/image1/json' in urls) self.assertTrue( 'http://pulpproject.org/v1/images/image1/layer' in urls) finally: shutil.rmtree(step.working_dir)
def test_init(self, mock_validate): # re-run this with the mock in place self.step = sync.SyncStep(self.repo, self.conduit, self.config, '/a/b/c') self.assertEqual(self.step.step_id, constants.SYNC_STEP_MAIN) # make sure the children are present step_ids = set([child.step_id for child in self.step.children]) self.assertTrue(constants.SYNC_STEP_METADATA in step_ids) self.assertTrue(reporting_constants.SYNC_STEP_GET_LOCAL in step_ids) self.assertTrue(constants.SYNC_STEP_DOWNLOAD in step_ids) self.assertTrue(constants.SYNC_STEP_SAVE in step_ids) # make sure it instantiated a Repository object self.assertTrue(isinstance(self.step.index_repository, registry.Repository)) self.assertEqual(self.step.index_repository.name, 'pulp/crane') self.assertEqual(self.step.index_repository.registry_url, 'http://pulpproject.org/') # these are important because child steps will populate them with data self.assertEqual(self.step.available_units, []) self.assertEqual(self.step.tags, {}) mock_validate.assert_called_once_with(self.config)
def test_generate_download_requests_correct_destinations( self, mock_working_dir, mock_v1_check, mock_v2_check): mock_working_dir.return_value = tempfile.mkdtemp() self.config.override_config[constants.CONFIG_KEY_ENABLE_V1] = True step = sync.SyncStep(self.repo, self.conduit, self.config) step.v1_step_get_local_units.units_to_download.append( models.Image(image_id='image1')) try: generator = step.v1_generate_download_requests() # make sure the urls are correct destinations = [req.destination for req in generator] self.assertTrue( os.path.join(step.working_dir, 'image1', 'ancestry') in destinations) self.assertTrue( os.path.join(step.working_dir, 'image1', 'json') in destinations) self.assertTrue( os.path.join(step.working_dir, 'image1', 'layer') in destinations) finally: shutil.rmtree(step.working_dir)
def test___init___with_v2_registry(self, v1_api_check, api_version_check, _validate, _working_directory_path): """ Test the __init__() method when the V2Repository does not raise a NotImplementedError with the api_version_check() method, indicating that the feed URL is a Docker v2 registry. """ _working_directory_path.return_value = self.working_dir repo = mock.MagicMock() conduit = mock.MagicMock() config = plugin_config.PluginCallConfiguration( {}, { 'feed': 'https://registry.example.com', 'upstream_name': 'busybox', importer_constants.KEY_MAX_DOWNLOADS: 25 }) step = sync.SyncStep(repo=repo, conduit=conduit, config=config) self.assertEqual(step.description, _('Syncing Docker Repository')) # The config should get validated _validate.assert_called_once_with(config) # available_blobs should have been initialized to an empty list self.assertEqual(step.available_blobs, []) self.assertEqual(step.available_manifests, []) # Ensure that the index_repository was initialized correctly self.assertEqual(type(step.index_repository), registry.V2Repository) self.assertEqual(step.index_repository.name, 'busybox') self.assertEqual(step.index_repository.download_config.max_concurrent, 25) self.assertEqual(step.index_repository.registry_url, 'https://registry.example.com') self.assertEqual(step.index_repository.working_dir, self.working_dir) # The version check should have happened, and since we mocked it, it will not raise an error api_version_check.assert_called_once_with() # The correct children should be in place in the right order self.assertEqual([type(child) for child in step.children], [ sync.DownloadManifestsStep, publish_step.GetLocalUnitsStep, publish_step.GetLocalUnitsStep, sync.TokenAuthDownloadStep, sync.SaveUnitsStep, sync.SaveTagsStep ]) # Ensure the first step was initialized correctly self.assertEqual(step.children[0].repo, repo) self.assertEqual(step.children[0].conduit, conduit) self.assertEqual(step.children[0].config, config) # And the second step self.assertTrue(step.children[1] is step.step_get_local_manifests) self.assertEqual(step.children[1].plugin_type, constants.IMPORTER_TYPE_ID) self.assertEqual(step.children[1].available_units, step.available_manifests) # And the third step self.assertTrue(step.children[2] is step.step_get_local_blobs) self.assertEqual(step.children[2].plugin_type, constants.IMPORTER_TYPE_ID) self.assertEqual(step.children[2].available_units, step.available_blobs) # And the fourth self.assertEqual(step.children[3].step_type, constants.SYNC_STEP_DOWNLOAD) self.assertEqual(step.children[3].repo, repo) self.assertEqual(step.children[3].config, config) self.assertEqual(step.children[3].description, _('Downloading remote files'))