def publish_repo(self, transfer_repo, publish_conduit, config): """ Export a yum repository to a given directory, or to ISO :param transfer_repo: metadata describing the repository :type transfer_repo: pulp.plugins.model.Repository :param publish_conduit: provides access to relevant Pulp functionality :type publish_conduit: pulp.plugins.conduits.repo_publish.RepoPublishConduit :param config: plugin configuration :type config: pulp.plugins.config.PluginConfiguration :return: report describing the publish run :rtype: pulp.plugins.model.PublishReport """ # First, validate the configuration because there may be override config options, and # currently, validate_config is not called prior to publishing by the manager. valid_config, msg = export_utils.validate_export_config(config) if not valid_config: raise PulpDataException(msg) _logger.info('Starting export of [%s]' % transfer_repo.id) self._publisher = ExportRepoPublisher(transfer_repo, publish_conduit, config, ids.TYPE_ID_DISTRIBUTOR_EXPORT) return self._publisher.process_lifecycle()
def validate_config(self, repo_group, config, config_conduit): """ Allows the distributor to check the contents of a potential configuration for the given repository. This call is made both for the addition of this distributor to a new repository group, as well as updating the configuration for this distributor on a previously configured repository. The return is a tuple of the result of the validation (True for success, False for failure) and a message. The message may be None and is unused in the success case. If the message is not None, i18n is taken into consideration when generating the message. The related_repo_groups parameter contains a list of other repository groups that have a configured distributor of this type. The distributor configurations is found in each repository group in the "plugin_configs" field. :param repo_group: metadata describing the repository to which the configuration applies :type repo_group: pulp.plugins.model.Repository :param config: plugin configuration instance :type config: pulp.plugins.config.PluginCallConfiguration :param config_conduit: Configuration Conduit; :type config_conduit: pulp.plugins.conduits.repo_validate.RepoConfigConduit :return: tuple of (bool, str) to describe the result :rtype: tuple """ return export_utils.validate_export_config(config)
def test_bad_skip_config(self): # Setup self.repo_config[constants.SKIP_KEYWORD] = 'not a list' # Test that a skip list that isn't a list fails to validate result = export_utils.validate_export_config(PluginCallConfiguration({}, self.repo_config)) self.assertFalse(result[0])
def test_bad_start_date(self): # Setup self.repo_config[constants.START_DATE_KEYWORD] = 'malformed date' # Test result = export_utils.validate_export_config(PluginCallConfiguration({}, self.repo_config)) self.assertFalse(result[0])
def test_bad_skip_config(self): # Setup self.repo_config[constants.SKIP_KEYWORD] = 'not a list' # Test that a skip list that isn't a list fails to validate result = export_utils.validate_export_config(PluginCallConfiguration({}, self.repo_config)) self.assertFalse(result[0])
def test_bad_iso_size_config(self): # Setup self.repo_config[constants.ISO_SIZE_KEYWORD] = -55 # Test that a prefix with invalid characters fails validation result = export_utils.validate_export_config(PluginCallConfiguration({}, self.repo_config)) self.assertFalse(result[0])
def test_none_end_date(self): # Setup self.repo_config[constants.END_DATE_KEYWORD] = None # Test result = export_utils.validate_export_config(PluginCallConfiguration({}, self.repo_config)) self.assertFalse(result[0])
def test_none_iso_prefix(self): # Setup self.repo_config[constants.ISO_PREFIX_KEYWORD] = None # Test result = export_utils.validate_export_config(PluginCallConfiguration({}, self.repo_config)) self.assertFalse(result[0])
def publish_repo(self, transfer_repo, publish_conduit, config): """ Export a yum repository to a given directory, or to ISO :param transfer_repo: metadata describing the repository :type transfer_repo: pulp.plugins.model.Repository :param publish_conduit: provides access to relevant Pulp functionality :type publish_conduit: pulp.plugins.conduits.repo_publish.RepoPublishConduit :param config: plugin configuration :type config: pulp.plugins.config.PluginConfiguration :return: report describing the publish run :rtype: pulp.plugins.model.PublishReport """ # First, validate the configuration because there may be override config options, and # currently, validate_config is not called prior to publishing by the manager. valid_config, msg = export_utils.validate_export_config(config) if not valid_config: raise PulpDataException(msg) # raises a PulpCodedException if all units are not downloaded self.ensure_all_units_downloaded(transfer_repo.id) _logger.info('Starting export of [%s]' % transfer_repo.id) self._publisher = ExportRepoPublisher(transfer_repo, publish_conduit, config, ids.TYPE_ID_DISTRIBUTOR_EXPORT) return self._publisher.process_lifecycle()
def publish_group(self, repo_group, publish_conduit, config): """ Publishes the given repository group. :param repo_group: metadata describing the repository group :type repo_group: pulp.plugins.model.RepositoryGroup :param publish_conduit: provides access to relevant Pulp functionality :type publish_conduit: pulp.plugins.conduits.repo_publish.RepoGroupPublishConduit :param config: plugin configuration :type config: pulp.plugins.config.PluginConfiguration :return: report describing the publish run :rtype: pulp.plugins.model.PublishReport """ # First, validate the configuration because there may be override config options, # and currently, # validate_config is not called prior to publishing by the manager. valid_config, msg = export_utils.validate_export_config(config) if not valid_config: raise PulpDataException(msg) # raises a PulpCodedException if all units are not downloaded self.ensure_all_units_downloaded(repo_group) _logger.info('Beginning export of the following repository group: [%s]' % repo_group.id) self._publisher = ExportRepoGroupPublisher(repo_group, publish_conduit, config, ids.TYPE_ID_DISTRIBUTOR_GROUP_EXPORT) return self._publisher.process_lifecycle()
def test_none_https(self): # Setup self.repo_config[constants.PUBLISH_HTTPS_KEYWORD] = None # Test result = export_utils.validate_export_config(PluginCallConfiguration({}, self.repo_config)) self.assertFalse(result[0])
def test_bad_iso_size_config(self): # Setup self.repo_config[constants.ISO_SIZE_KEYWORD] = -55 # Test that a prefix with invalid characters fails validation result = export_utils.validate_export_config(PluginCallConfiguration({}, self.repo_config)) self.assertFalse(result[0])
def test_none_iso_prefix(self): # Setup self.repo_config[constants.ISO_PREFIX_KEYWORD] = None # Test result = export_utils.validate_export_config(PluginCallConfiguration({}, self.repo_config)) self.assertFalse(result[0])
def test_none_end_date(self): # Setup self.repo_config[constants.END_DATE_KEYWORD] = None # Test result = export_utils.validate_export_config(PluginCallConfiguration({}, self.repo_config)) self.assertFalse(result[0])
def test_none_path(self): # Setup self.repo_config[constants.EXPORT_DIRECTORY_KEYWORD] = None # Test result = export_utils.validate_export_config(PluginCallConfiguration({}, self.repo_config)) self.assertFalse(result[0])
def test_non_absolute_path(self): # Setup self.repo_config[constants.EXPORT_DIRECTORY_KEYWORD] = 'non/absolute/path' # Test that if the export directory isn't found, validation fails result = export_utils.validate_export_config(PluginCallConfiguration({}, self.repo_config)) self.assertFalse(result[0])
def test_bad_start_date(self): # Setup self.repo_config[constants.START_DATE_KEYWORD] = 'malformed date' # Test result = export_utils.validate_export_config(PluginCallConfiguration({}, self.repo_config)) self.assertFalse(result[0])
def test_none_path(self): # Setup self.repo_config[constants.EXPORT_DIRECTORY_KEYWORD] = None # Test result = export_utils.validate_export_config(PluginCallConfiguration({}, self.repo_config)) self.assertFalse(result[0])
def test_non_absolute_path(self): # Setup self.repo_config[constants.EXPORT_DIRECTORY_KEYWORD] = 'non/absolute/path' # Test that if the export directory isn't found, validation fails result = export_utils.validate_export_config(PluginCallConfiguration({}, self.repo_config)) self.assertFalse(result[0])
def publish_group(self, repo_group, publish_conduit, config): """ Publishes the given repository group. :param repo_group: metadata describing the repository group :type repo_group: pulp.plugins.model.RepositoryGroup :param publish_conduit: provides access to relevant Pulp functionality :type publish_conduit: pulp.plugins.conduits.repo_publish.RepoGroupPublishConduit :param config: plugin configuration :type config: pulp.plugins.config.PluginConfiguration :return: report describing the publish run :rtype: pulp.plugins.model.PublishReport """ # First, validate the configuration because there may be override config options, # and currently, # validate_config is not called prior to publishing by the manager. valid_config, msg = export_utils.validate_export_config(config) if not valid_config: raise PulpDataException(msg) # raises a PulpCodedException if all units are not downloaded self.ensure_all_units_downloaded(repo_group) _logger.info( 'Beginning export of the following repository group: [%s]' % repo_group.id) self._publisher = ExportRepoGroupPublisher( repo_group, publish_conduit, config, ids.TYPE_ID_DISTRIBUTOR_GROUP_EXPORT) return self._publisher.process_lifecycle()
def test_none_https(self): # Setup self.repo_config[constants.PUBLISH_HTTPS_KEYWORD] = None # Test result = export_utils.validate_export_config(PluginCallConfiguration({}, self.repo_config)) self.assertFalse(result[0])
def test_missing_export_dir(self, mock_isdir): # Setup self.repo_config[constants.EXPORT_DIRECTORY_KEYWORD] = '/directory/not/found' mock_isdir.return_value = False # Test that if the export directory isn't found, validation fails result = export_utils.validate_export_config(PluginCallConfiguration({}, self.repo_config)) self.assertFalse(result[0])
def test_unwritable_export_dir(self, mock_isdir, mock_access): # Setup self.repo_config[constants.EXPORT_DIRECTORY_KEYWORD] = '/some/dir' mock_isdir.return_value = True mock_access.return_value = False # Test that if the export directory isn't writable, validation fails result = export_utils.validate_export_config(PluginCallConfiguration({}, self.repo_config)) self.assertFalse(result[0])
def test_full_config(self): self.repo_config[constants.SKIP_KEYWORD] = [] self.repo_config[constants.ISO_PREFIX_KEYWORD] = 'prefix' self.repo_config[constants.ISO_SIZE_KEYWORD] = 630 self.repo_config[constants.EXPORT_DIRECTORY_KEYWORD] = export_dir = '/path/to/dir' self.repo_config[constants.START_DATE_KEYWORD] = '2013-07-18T11:22:00' self.repo_config[constants.END_DATE_KEYWORD] = '2013-07-18T11:23:00' result, msg = export_utils.validate_export_config(PluginCallConfiguration({}, self.repo_config)) self.assertTrue(result)
def test_full_config(self): self.repo_config[constants.SKIP_KEYWORD] = [] self.repo_config[constants.ISO_PREFIX_KEYWORD] = "prefix" self.repo_config[constants.ISO_SIZE_KEYWORD] = 630 self.repo_config[constants.EXPORT_DIRECTORY_KEYWORD] = "/path/to/dir" self.repo_config[constants.START_DATE_KEYWORD] = "2013-07-18T11:22:00" self.repo_config[constants.END_DATE_KEYWORD] = "2013-07-18T11:23:00" self.repo_config[constants.CREATE_PULP_MANIFEST] = True result, msg = export_utils.validate_export_config(PluginCallConfiguration({}, self.repo_config)) self.assertTrue(result)
def test_full_config(self): self.repo_config[constants.SKIP_KEYWORD] = [] self.repo_config[constants.ISO_PREFIX_KEYWORD] = 'prefix' self.repo_config[constants.ISO_SIZE_KEYWORD] = 630 self.repo_config[constants.EXPORT_DIRECTORY_KEYWORD] = '/path/to/dir' self.repo_config[constants.START_DATE_KEYWORD] = '2013-07-18T11:22:00' self.repo_config[constants.END_DATE_KEYWORD] = '2013-07-18T11:23:00' result, msg = export_utils.validate_export_config( PluginCallConfiguration({}, self.repo_config)) self.assertTrue(result)
def test_full_config(self, mock_access, mock_isdir): self.repo_config[constants.SKIP_KEYWORD] = [] self.repo_config[constants.ISO_PREFIX_KEYWORD] = 'prefix' self.repo_config[constants.ISO_SIZE_KEYWORD] = 630 self.repo_config[constants.EXPORT_DIRECTORY_KEYWORD] = export_dir = '/path/to/dir' self.repo_config[constants.START_DATE_KEYWORD] = '2013-07-18T11:22:00' self.repo_config[constants.END_DATE_KEYWORD] = '2013-07-18T11:23:00' result, msg = export_utils.validate_export_config(PluginCallConfiguration({}, self.repo_config)) self.assertTrue(result) mock_isdir.assert_called_once_with(export_dir) self.assertEqual(2, mock_access.call_count) self.assertEqual((export_dir, os.R_OK), mock_access.call_args_list[0][0]) self.assertEqual((export_dir, os.W_OK), mock_access.call_args_list[1][0])
def test_bad_manifest_flag(self): self.repo_config[constants.CREATE_PULP_MANIFEST] = 'true' # Test result = export_utils.validate_export_config(PluginCallConfiguration({}, self.repo_config)) self.assertFalse(result[0])
def test_invalid_key(self): self.repo_config['leek'] = 'garlic' result, msg = export_utils.validate_export_config( PluginCallConfiguration({}, self.repo_config)) self.assertFalse(result)
def test_missing_required_key(self): # Confirm missing required keys causes validation to fail result, msg = export_utils.validate_export_config(PluginCallConfiguration({}, {})) self.assertFalse(result)
def test_required_keys_only(self): # Confirm providing only required keys causes a successful validation return_value = export_utils.validate_export_config(self.valid_config) self.assertEqual(return_value, (True, None))
def test_non_bool_https_key(self): # Confirm including a non-boolean for the publish https keyword fails validation self.repo_config[constants.PUBLISH_HTTPS_KEYWORD] = 'potato' result, msg = export_utils.validate_export_config( PluginCallConfiguration({}, self.repo_config)) self.assertFalse(result)
def test_bad_manifest_flag(self): self.repo_config[constants.CREATE_PULP_MANIFEST] = 'true' # Test result = export_utils.validate_export_config(PluginCallConfiguration({}, self.repo_config)) self.assertFalse(result[0])
def test_missing_required_key(self): # Confirm missing required keys causes validation to fail result, msg = export_utils.validate_export_config(PluginCallConfiguration({}, {})) self.assertFalse(result)
def test_none_relative_url(self): self.repo_config[constants.RELATIVE_URL_KEYWORD] = None result = export_utils.validate_export_config(PluginCallConfiguration({}, self.repo_config)) self.assertFalse(result[0])
def test_required_keys_only(self): # Confirm providing only required keys causes a successful validation return_value = export_utils.validate_export_config(self.valid_config) self.assertEqual(return_value, (True, None))
def test_non_bool_https_key(self): # Confirm including a non-boolean for the publish https keyword fails validation self.repo_config[constants.PUBLISH_HTTPS_KEYWORD] = 'potato' result, msg = export_utils.validate_export_config( PluginCallConfiguration({}, self.repo_config)) self.assertFalse(result)
def test_invalid_key(self): self.repo_config['leek'] = 'garlic' result, msg = export_utils.validate_export_config( PluginCallConfiguration({}, self.repo_config)) self.assertFalse(result)
def publish_repo(self, repo, publish_conduit, config): """ Export a yum repository to a given directory, or to ISO :param repo: metadata describing the repository :type repo: pulp.plugins.model.Repository :param publish_conduit: provides access to relevant Pulp functionality :type publish_conduit: pulp.plugins.conduits.repo_publish.RepoPublishConduit :param config: plugin configuration :type config: pulp.plugins.config.PluginConfiguration :return: report describing the publish run :rtype: pulp.plugins.model.PublishReport """ # First, validate the configuration because there may be override config options, and # currently, validate_config is not called prior to publishing by the manager. valid_config, msg = export_utils.validate_export_config(config) if not valid_config: raise PulpDataException(msg) _logger.info('Starting export of [%s]' % repo.id) progress_status = { ids.TYPE_ID_RPM: {'state': constants.STATE_NOT_STARTED}, ids.TYPE_ID_ERRATA: {'state': constants.STATE_NOT_STARTED}, ids.TYPE_ID_DISTRO: {'state': constants.STATE_NOT_STARTED}, ids.TYPE_ID_PKG_CATEGORY: {'state': constants.STATE_NOT_STARTED}, ids.TYPE_ID_PKG_GROUP: {'state': constants.STATE_NOT_STARTED}, 'metadata': {'state': constants.STATE_NOT_STARTED}, 'isos': {'state': constants.STATE_NOT_STARTED}, 'publish_http': {'state': constants.STATE_NOT_STARTED}, 'publish_https': {'state': constants.STATE_NOT_STARTED}, } def progress_callback(type_id, status): progress_status[type_id] = status publish_conduit.set_progress(progress_status) # Retrieve a config tuple and unpack it for use config_settings = export_utils.retrieve_repo_config(repo, config) self.working_dir, self.date_filter = config_settings # Before starting, clean out the working directory. Done to remove last published ISOs shutil.rmtree(repo.working_dir, ignore_errors=True) os.makedirs(repo.working_dir) # If a date filter is not present, do a complete export. If it is, do an incremental export. if self.date_filter: result = export_utils.export_incremental_content(self.working_dir, publish_conduit, self.date_filter, progress_callback) else: result = export_utils.export_complete_repo(repo.id, self.working_dir, publish_conduit, config, progress_callback) self.summary = result[0] self.details = result[1] if not config.get(constants.EXPORT_DIRECTORY_KEYWORD): util.generate_listing_files(repo.working_dir, self.working_dir) # build iso and publish via HTTPS self._publish_isos(repo, config, progress_callback) else: export_dir = config.get(constants.EXPORT_DIRECTORY_KEYWORD) util.generate_listing_files(export_dir, self.working_dir) if len(self.details['errors']) != 0: return publish_conduit.build_failure_report(self.summary, self.details) return publish_conduit.build_success_report(self.summary, self.details)