def publish_repo(self, transfer_repo, publish_conduit, config):
"""
Export a yum repository to a given directory, or to ISO
:param transfer_repo: metadata describing the repository
:type transfer_repo: pulp.plugins.model.Repository
:param publish_conduit: provides access to relevant Pulp functionality
:type publish_conduit: pulp.plugins.conduits.repo_publish.RepoPublishConduit
:param config: plugin configuration
:type config: pulp.plugins.config.PluginConfiguration
:return: report describing the publish run
:rtype: pulp.plugins.model.PublishReport
"""
# First, validate the configuration because there may be override config options, and
# currently, validate_config is not called prior to publishing by the manager.
valid_config, msg = export_utils.validate_export_config(config)
if not valid_config:
raise PulpDataException(msg)
_logger.info('Starting export of [%s]' % transfer_repo.id)
self._publisher = ExportRepoPublisher(transfer_repo, publish_conduit,
config,
ids.TYPE_ID_DISTRIBUTOR_EXPORT)
return self._publisher.process_lifecycle()
def validate_config(self, repo_group, config, config_conduit):
"""
Allows the distributor to check the contents of a potential configuration
for the given repository. This call is made both for the addition of
this distributor to a new repository group, as well as updating the configuration
for this distributor on a previously configured repository.
The return is a tuple of the result of the validation (True for success,
False for failure) and a message. The message may be None and is unused
in the success case. If the message is not None, i18n is taken into
consideration when generating the message.
The related_repo_groups parameter contains a list of other repository groups that
have a configured distributor of this type. The distributor configurations
is found in each repository group in the "plugin_configs" field.
:param repo_group: metadata describing the repository to which the configuration
applies
:type repo_group: pulp.plugins.model.Repository
:param config: plugin configuration instance
:type config: pulp.plugins.config.PluginCallConfiguration
:param config_conduit: Configuration Conduit;
:type config_conduit: pulp.plugins.conduits.repo_validate.RepoConfigConduit
:return: tuple of (bool, str) to describe the result
:rtype: tuple
"""
return export_utils.validate_export_config(config)
def test_bad_skip_config(self):
# Setup
self.repo_config[constants.SKIP_KEYWORD] = 'not a list'
# Test that a skip list that isn't a list fails to validate
result = export_utils.validate_export_config(PluginCallConfiguration({}, self.repo_config))
self.assertFalse(result[0])
def test_bad_start_date(self):
# Setup
self.repo_config[constants.START_DATE_KEYWORD] = 'malformed date'
# Test
result = export_utils.validate_export_config(PluginCallConfiguration({}, self.repo_config))
self.assertFalse(result[0])
def test_bad_skip_config(self):
# Setup
self.repo_config[constants.SKIP_KEYWORD] = 'not a list'
# Test that a skip list that isn't a list fails to validate
result = export_utils.validate_export_config(PluginCallConfiguration({}, self.repo_config))
self.assertFalse(result[0])
def test_bad_iso_size_config(self):
# Setup
self.repo_config[constants.ISO_SIZE_KEYWORD] = -55
# Test that a prefix with invalid characters fails validation
result = export_utils.validate_export_config(PluginCallConfiguration({}, self.repo_config))
self.assertFalse(result[0])
def test_none_end_date(self):
# Setup
self.repo_config[constants.END_DATE_KEYWORD] = None
# Test
result = export_utils.validate_export_config(PluginCallConfiguration({}, self.repo_config))
self.assertFalse(result[0])
def test_none_iso_prefix(self):
# Setup
self.repo_config[constants.ISO_PREFIX_KEYWORD] = None
# Test
result = export_utils.validate_export_config(PluginCallConfiguration({}, self.repo_config))
self.assertFalse(result[0])
def publish_repo(self, transfer_repo, publish_conduit, config):
"""
Export a yum repository to a given directory, or to ISO
:param transfer_repo: metadata describing the repository
:type transfer_repo: pulp.plugins.model.Repository
:param publish_conduit: provides access to relevant Pulp functionality
:type publish_conduit: pulp.plugins.conduits.repo_publish.RepoPublishConduit
:param config: plugin configuration
:type config: pulp.plugins.config.PluginConfiguration
:return: report describing the publish run
:rtype: pulp.plugins.model.PublishReport
"""
# First, validate the configuration because there may be override config options, and
# currently, validate_config is not called prior to publishing by the manager.
valid_config, msg = export_utils.validate_export_config(config)
if not valid_config:
raise PulpDataException(msg)
# raises a PulpCodedException if all units are not downloaded
self.ensure_all_units_downloaded(transfer_repo.id)
_logger.info('Starting export of [%s]' % transfer_repo.id)
self._publisher = ExportRepoPublisher(transfer_repo, publish_conduit, config,
ids.TYPE_ID_DISTRIBUTOR_EXPORT)
return self._publisher.process_lifecycle()
def publish_group(self, repo_group, publish_conduit, config):
"""
Publishes the given repository group.
:param repo_group: metadata describing the repository group
:type repo_group: pulp.plugins.model.RepositoryGroup
:param publish_conduit: provides access to relevant Pulp functionality
:type publish_conduit: pulp.plugins.conduits.repo_publish.RepoGroupPublishConduit
:param config: plugin configuration
:type config: pulp.plugins.config.PluginConfiguration
:return: report describing the publish run
:rtype: pulp.plugins.model.PublishReport
"""
# First, validate the configuration because there may be override config options,
# and currently,
# validate_config is not called prior to publishing by the manager.
valid_config, msg = export_utils.validate_export_config(config)
if not valid_config:
raise PulpDataException(msg)
# raises a PulpCodedException if all units are not downloaded
self.ensure_all_units_downloaded(repo_group)
_logger.info('Beginning export of the following repository group: [%s]' % repo_group.id)
self._publisher = ExportRepoGroupPublisher(repo_group, publish_conduit, config,
ids.TYPE_ID_DISTRIBUTOR_GROUP_EXPORT)
return self._publisher.process_lifecycle()
def test_none_https(self):
# Setup
self.repo_config[constants.PUBLISH_HTTPS_KEYWORD] = None
# Test
result = export_utils.validate_export_config(PluginCallConfiguration({}, self.repo_config))
self.assertFalse(result[0])
def test_bad_iso_size_config(self):
# Setup
self.repo_config[constants.ISO_SIZE_KEYWORD] = -55
# Test that a prefix with invalid characters fails validation
result = export_utils.validate_export_config(PluginCallConfiguration({}, self.repo_config))
self.assertFalse(result[0])
def test_none_iso_prefix(self):
# Setup
self.repo_config[constants.ISO_PREFIX_KEYWORD] = None
# Test
result = export_utils.validate_export_config(PluginCallConfiguration({}, self.repo_config))
self.assertFalse(result[0])
def test_none_end_date(self):
# Setup
self.repo_config[constants.END_DATE_KEYWORD] = None
# Test
result = export_utils.validate_export_config(PluginCallConfiguration({}, self.repo_config))
self.assertFalse(result[0])
def test_none_path(self):
# Setup
self.repo_config[constants.EXPORT_DIRECTORY_KEYWORD] = None
# Test
result = export_utils.validate_export_config(PluginCallConfiguration({}, self.repo_config))
self.assertFalse(result[0])
def test_non_absolute_path(self):
# Setup
self.repo_config[constants.EXPORT_DIRECTORY_KEYWORD] = 'non/absolute/path'
# Test that if the export directory isn't found, validation fails
result = export_utils.validate_export_config(PluginCallConfiguration({}, self.repo_config))
self.assertFalse(result[0])
def test_bad_start_date(self):
# Setup
self.repo_config[constants.START_DATE_KEYWORD] = 'malformed date'
# Test
result = export_utils.validate_export_config(PluginCallConfiguration({}, self.repo_config))
self.assertFalse(result[0])
def test_none_path(self):
# Setup
self.repo_config[constants.EXPORT_DIRECTORY_KEYWORD] = None
# Test
result = export_utils.validate_export_config(PluginCallConfiguration({}, self.repo_config))
self.assertFalse(result[0])
def test_non_absolute_path(self):
# Setup
self.repo_config[constants.EXPORT_DIRECTORY_KEYWORD] = 'non/absolute/path'
# Test that if the export directory isn't found, validation fails
result = export_utils.validate_export_config(PluginCallConfiguration({}, self.repo_config))
self.assertFalse(result[0])
def publish_group(self, repo_group, publish_conduit, config):
"""
Publishes the given repository group.
:param repo_group: metadata describing the repository group
:type repo_group: pulp.plugins.model.RepositoryGroup
:param publish_conduit: provides access to relevant Pulp functionality
:type publish_conduit: pulp.plugins.conduits.repo_publish.RepoGroupPublishConduit
:param config: plugin configuration
:type config: pulp.plugins.config.PluginConfiguration
:return: report describing the publish run
:rtype: pulp.plugins.model.PublishReport
"""
# First, validate the configuration because there may be override config options,
# and currently,
# validate_config is not called prior to publishing by the manager.
valid_config, msg = export_utils.validate_export_config(config)
if not valid_config:
raise PulpDataException(msg)
# raises a PulpCodedException if all units are not downloaded
self.ensure_all_units_downloaded(repo_group)
_logger.info(
'Beginning export of the following repository group: [%s]' %
repo_group.id)
self._publisher = ExportRepoGroupPublisher(
repo_group, publish_conduit, config,
ids.TYPE_ID_DISTRIBUTOR_GROUP_EXPORT)
return self._publisher.process_lifecycle()
def test_none_https(self):
# Setup
self.repo_config[constants.PUBLISH_HTTPS_KEYWORD] = None
# Test
result = export_utils.validate_export_config(PluginCallConfiguration({}, self.repo_config))
self.assertFalse(result[0])
def test_missing_export_dir(self, mock_isdir):
# Setup
self.repo_config[constants.EXPORT_DIRECTORY_KEYWORD] = '/directory/not/found'
mock_isdir.return_value = False
# Test that if the export directory isn't found, validation fails
result = export_utils.validate_export_config(PluginCallConfiguration({}, self.repo_config))
self.assertFalse(result[0])
def test_unwritable_export_dir(self, mock_isdir, mock_access):
# Setup
self.repo_config[constants.EXPORT_DIRECTORY_KEYWORD] = '/some/dir'
mock_isdir.return_value = True
mock_access.return_value = False
# Test that if the export directory isn't writable, validation fails
result = export_utils.validate_export_config(PluginCallConfiguration({}, self.repo_config))
self.assertFalse(result[0])
def test_full_config(self):
self.repo_config[constants.SKIP_KEYWORD] = []
self.repo_config[constants.ISO_PREFIX_KEYWORD] = 'prefix'
self.repo_config[constants.ISO_SIZE_KEYWORD] = 630
self.repo_config[constants.EXPORT_DIRECTORY_KEYWORD] = export_dir = '/path/to/dir'
self.repo_config[constants.START_DATE_KEYWORD] = '2013-07-18T11:22:00'
self.repo_config[constants.END_DATE_KEYWORD] = '2013-07-18T11:23:00'
result, msg = export_utils.validate_export_config(PluginCallConfiguration({}, self.repo_config))
self.assertTrue(result)
def test_full_config(self):
self.repo_config[constants.SKIP_KEYWORD] = []
self.repo_config[constants.ISO_PREFIX_KEYWORD] = "prefix"
self.repo_config[constants.ISO_SIZE_KEYWORD] = 630
self.repo_config[constants.EXPORT_DIRECTORY_KEYWORD] = "/path/to/dir"
self.repo_config[constants.START_DATE_KEYWORD] = "2013-07-18T11:22:00"
self.repo_config[constants.END_DATE_KEYWORD] = "2013-07-18T11:23:00"
self.repo_config[constants.CREATE_PULP_MANIFEST] = True
result, msg = export_utils.validate_export_config(PluginCallConfiguration({}, self.repo_config))
self.assertTrue(result)
def test_full_config(self):
self.repo_config[constants.SKIP_KEYWORD] = []
self.repo_config[constants.ISO_PREFIX_KEYWORD] = 'prefix'
self.repo_config[constants.ISO_SIZE_KEYWORD] = 630
self.repo_config[constants.EXPORT_DIRECTORY_KEYWORD] = '/path/to/dir'
self.repo_config[constants.START_DATE_KEYWORD] = '2013-07-18T11:22:00'
self.repo_config[constants.END_DATE_KEYWORD] = '2013-07-18T11:23:00'
result, msg = export_utils.validate_export_config(
PluginCallConfiguration({}, self.repo_config))
self.assertTrue(result)
def test_full_config(self, mock_access, mock_isdir):
self.repo_config[constants.SKIP_KEYWORD] = []
self.repo_config[constants.ISO_PREFIX_KEYWORD] = 'prefix'
self.repo_config[constants.ISO_SIZE_KEYWORD] = 630
self.repo_config[constants.EXPORT_DIRECTORY_KEYWORD] = export_dir = '/path/to/dir'
self.repo_config[constants.START_DATE_KEYWORD] = '2013-07-18T11:22:00'
self.repo_config[constants.END_DATE_KEYWORD] = '2013-07-18T11:23:00'
result, msg = export_utils.validate_export_config(PluginCallConfiguration({}, self.repo_config))
self.assertTrue(result)
mock_isdir.assert_called_once_with(export_dir)
self.assertEqual(2, mock_access.call_count)
self.assertEqual((export_dir, os.R_OK), mock_access.call_args_list[0][0])
self.assertEqual((export_dir, os.W_OK), mock_access.call_args_list[1][0])
def test_bad_manifest_flag(self):
self.repo_config[constants.CREATE_PULP_MANIFEST] = 'true'
# Test
result = export_utils.validate_export_config(PluginCallConfiguration({}, self.repo_config))
self.assertFalse(result[0])
def test_invalid_key(self):
self.repo_config['leek'] = 'garlic'
result, msg = export_utils.validate_export_config(
PluginCallConfiguration({}, self.repo_config))
self.assertFalse(result)
def test_missing_required_key(self):
# Confirm missing required keys causes validation to fail
result, msg = export_utils.validate_export_config(PluginCallConfiguration({}, {}))
self.assertFalse(result)
def test_required_keys_only(self):
# Confirm providing only required keys causes a successful validation
return_value = export_utils.validate_export_config(self.valid_config)
self.assertEqual(return_value, (True, None))
def test_non_bool_https_key(self):
# Confirm including a non-boolean for the publish https keyword fails validation
self.repo_config[constants.PUBLISH_HTTPS_KEYWORD] = 'potato'
result, msg = export_utils.validate_export_config(
PluginCallConfiguration({}, self.repo_config))
self.assertFalse(result)
def test_bad_manifest_flag(self):
self.repo_config[constants.CREATE_PULP_MANIFEST] = 'true'
# Test
result = export_utils.validate_export_config(PluginCallConfiguration({}, self.repo_config))
self.assertFalse(result[0])
def test_missing_required_key(self):
# Confirm missing required keys causes validation to fail
result, msg = export_utils.validate_export_config(PluginCallConfiguration({}, {}))
self.assertFalse(result)
def test_none_relative_url(self):
self.repo_config[constants.RELATIVE_URL_KEYWORD] = None
result = export_utils.validate_export_config(PluginCallConfiguration({}, self.repo_config))
self.assertFalse(result[0])
def test_required_keys_only(self):
# Confirm providing only required keys causes a successful validation
return_value = export_utils.validate_export_config(self.valid_config)
self.assertEqual(return_value, (True, None))
def test_non_bool_https_key(self):
# Confirm including a non-boolean for the publish https keyword fails validation
self.repo_config[constants.PUBLISH_HTTPS_KEYWORD] = 'potato'
result, msg = export_utils.validate_export_config(
PluginCallConfiguration({}, self.repo_config))
self.assertFalse(result)
def test_invalid_key(self):
self.repo_config['leek'] = 'garlic'
result, msg = export_utils.validate_export_config(
PluginCallConfiguration({}, self.repo_config))
self.assertFalse(result)
def publish_repo(self, repo, publish_conduit, config):
"""
Export a yum repository to a given directory, or to ISO
:param repo: metadata describing the repository
:type repo: pulp.plugins.model.Repository
:param publish_conduit: provides access to relevant Pulp functionality
:type publish_conduit: pulp.plugins.conduits.repo_publish.RepoPublishConduit
:param config: plugin configuration
:type config: pulp.plugins.config.PluginConfiguration
:return: report describing the publish run
:rtype: pulp.plugins.model.PublishReport
"""
# First, validate the configuration because there may be override config options, and
# currently, validate_config is not called prior to publishing by the manager.
valid_config, msg = export_utils.validate_export_config(config)
if not valid_config:
raise PulpDataException(msg)
_logger.info('Starting export of [%s]' % repo.id)
progress_status = {
ids.TYPE_ID_RPM: {'state': constants.STATE_NOT_STARTED},
ids.TYPE_ID_ERRATA: {'state': constants.STATE_NOT_STARTED},
ids.TYPE_ID_DISTRO: {'state': constants.STATE_NOT_STARTED},
ids.TYPE_ID_PKG_CATEGORY: {'state': constants.STATE_NOT_STARTED},
ids.TYPE_ID_PKG_GROUP: {'state': constants.STATE_NOT_STARTED},
'metadata': {'state': constants.STATE_NOT_STARTED},
'isos': {'state': constants.STATE_NOT_STARTED},
'publish_http': {'state': constants.STATE_NOT_STARTED},
'publish_https': {'state': constants.STATE_NOT_STARTED},
}
def progress_callback(type_id, status):
progress_status[type_id] = status
publish_conduit.set_progress(progress_status)
# Retrieve a config tuple and unpack it for use
config_settings = export_utils.retrieve_repo_config(repo, config)
self.working_dir, self.date_filter = config_settings
# Before starting, clean out the working directory. Done to remove last published ISOs
shutil.rmtree(repo.working_dir, ignore_errors=True)
os.makedirs(repo.working_dir)
# If a date filter is not present, do a complete export. If it is, do an incremental export.
if self.date_filter:
result = export_utils.export_incremental_content(self.working_dir, publish_conduit,
self.date_filter, progress_callback)
else:
result = export_utils.export_complete_repo(repo.id, self.working_dir, publish_conduit,
config, progress_callback)
self.summary = result[0]
self.details = result[1]
if not config.get(constants.EXPORT_DIRECTORY_KEYWORD):
util.generate_listing_files(repo.working_dir, self.working_dir)
# build iso and publish via HTTPS
self._publish_isos(repo, config, progress_callback)
else:
export_dir = config.get(constants.EXPORT_DIRECTORY_KEYWORD)
util.generate_listing_files(export_dir, self.working_dir)
if len(self.details['errors']) != 0:
return publish_conduit.build_failure_report(self.summary, self.details)
return publish_conduit.build_success_report(self.summary, self.details)
