def publish_repo(self, repo, publish_conduit, config):
publish_start_time = time.time()
_LOG.info("Start publish time %s" % publish_start_time)
source_repo_id = config.get('source_repo_id')
destination_dist_config = self.find_yum_distributor(repo.id)['config']
source_working_dir = self.source_working_dir(source_repo_id)
working_dir = self.full_working_dir(repo.id, publish_start_time)
os.makedirs(working_dir)
#copy contents from source's working directory to destinations
if not self.copy_directory(source_working_dir, working_dir):
publish_conduit.set_progress(self.summary)
raise Exception("Failed to copy metadata. See errors for more details.")
#symlink the destination's publish directories
if destination_dist_config['http']:
http_publish_dir = os.path.join(HTTP_PUBLISH_DIR, destination_dist_config["relative_url"]).rstrip('/')
self.link_directory(working_dir, http_publish_dir)
util.generate_listing_files(HTTP_PUBLISH_DIR, http_publish_dir)
self.update_repomd(http_publish_dir)
if destination_dist_config['https']:
https_publish_dir = os.path.join(HTTPS_PUBLISH_DIR, destination_dist_config["relative_url"]).rstrip('/')
self.link_directory(working_dir, https_publish_dir)
util.generate_listing_files(HTTPS_PUBLISH_DIR, https_publish_dir)
self.update_repomd(https_publish_dir)
self.clean_path(self.base_working_dir(repo.id), str(publish_start_time))
publish_conduit.set_progress(self.summary)
if len(self.summary["errors"]) > 0:
raise Exception("Failed to link metadata. See errors for more details.")
else:
return publish_conduit.build_success_report(self.summary, self.details)
def clean_simple_hosting_directories(self, start_location, containing_dir):
"""
Remove any orphaned directory structure left by the removal of a repo.
:param start_location: path of the symlink of the repo that was removed.
:type start_location: str
:param containing_dir: path of the location of the base dir containing hosted repos
:type containing_dir: str
"""
up_dir = os.path.dirname(start_location)
# This function is potentially dangerous so it is important to restrict it to the
# publish directories.
if containing_dir not in os.path.dirname(up_dir):
return
# If the only file is the listing file, it is safe to delete the file and containing dir.
if os.listdir(up_dir) == ['listing']:
os.remove(os.path.join(up_dir, 'listing'))
try:
os.rmdir(up_dir)
except OSError:
# If this happened, there was a concurrency issue and it is no longer safe to
# remove the directory. It is possible that the concurrent operation created the
# listing file before this operation deleted it, so to be safe, we need to
# regenerate the listing file.
util.generate_listing_files(up_dir, up_dir)
return
self.clean_simple_hosting_directories(up_dir, containing_dir)
def clean_simple_hosting_directories(self, start_location, containing_dir):
"""
Remove any orphaned directory structure left by the removal of a repo.
:param start_location: path of the symlink of the repo that was removed.
:type start_location: str
:param containing_dir: path of the location of the base dir containing hosted repos
:type containing_dir: str
"""
up_dir = os.path.dirname(start_location)
# This function is potentially dangerous so it is important to restrict it to the
# publish directories.
if containing_dir not in os.path.dirname(up_dir):
return
# If the only file is the listing file, it is safe to delete the file and containing dir.
if os.listdir(up_dir) == ['listing']:
os.remove(os.path.join(up_dir, 'listing'))
try:
os.rmdir(up_dir)
except OSError:
# If this happened, there was a concurrency issue and it is no longer safe to
# remove the directory. It is possible that the concurrent operation created the
# listing file before this operation deleted it, so to be safe, we need to
# regenerate the listing file.
util.generate_listing_files(up_dir, up_dir)
return
self.clean_simple_hosting_directories(up_dir, containing_dir)
def test_all(self):
tmp_dir = tempfile.mkdtemp()
try:
# setup a directory structure and define the expected listing file values
publish_dir = os.path.join(tmp_dir, 'a/b/c')
os.makedirs(publish_dir)
os.makedirs(os.path.join(tmp_dir, 'a/d'))
os.makedirs(os.path.join(tmp_dir, 'a/b/e'))
expected = ['a', 'b\nd', 'c\ne']
# run it
util.generate_listing_files(tmp_dir, publish_dir)
# ensure that each listing file exists and has the correct contents
current_path = tmp_dir
for next_dir, expected_listing in zip(['a', 'b', 'c'], expected):
file_path = os.path.join(current_path, 'listing')
with open(file_path) as open_file:
self.assertEqual(open_file.read(), expected_listing)
current_path = os.path.join(current_path, next_dir)
# make sure there is not a listing file inside the repo's publish dir
self.assertFalse(
os.path.exists(os.path.join(publish_dir, 'listing')))
finally:
shutil.rmtree(tmp_dir, ignore_errors=True)
def process_unit(self, root_publish_dir):
"""
Publish a directory from the repo to a target directory.
"""
# Find the location of the master repository tree structure
master_publish_dir = os.path.join(configuration.get_master_publish_dir(self.parent.repo),
self.parent.timestamp)
# Find the location of the published repository tree structure
repo_relative_dir = configuration.get_repo_relative_path(self.parent.repo, self.parent.config)
repo_publish_dir = os.path.join(root_publish_dir, repo_relative_dir)
# Without the trailing '/'
if repo_publish_dir.endswith('/'):
repo_publish_dir = repo_publish_dir[:-1]
# Create the parent directory of the published repository tree, if needed
repo_publish_dir_parent = repo_publish_dir.rsplit('/', 1)[0]
if not os.path.exists(repo_publish_dir_parent):
os.makedirs(repo_publish_dir_parent, 0750)
# Create a temporary symlink in the parent of the published directory tree
tmp_link_name = os.path.join(repo_publish_dir_parent, self.parent.timestamp)
os.symlink(master_publish_dir, tmp_link_name)
# Rename the symlink to the official published repository directory name.
# This has two desirable effects:
# 1. it will overwrite an existing link, if it's there
# 2. the operation is atomic, instantly changing the published directory
# NOTE: it's not easy (possible?) to directly edit the target of a symlink
os.rename(tmp_link_name, repo_publish_dir)
# (Re)generate the listing files
util.generate_listing_files(root_publish_dir, repo_publish_dir)
def test_all(self):
tmp_dir = tempfile.mkdtemp()
try:
# setup a directory structure and define the expected listing file values
publish_dir = os.path.join(tmp_dir, 'a/b/c')
os.makedirs(publish_dir)
os.makedirs(os.path.join(tmp_dir, 'a/d'))
os.makedirs(os.path.join(tmp_dir, 'a/b/e'))
expected = ['a', 'b\nd', 'c\ne']
# run it
util.generate_listing_files(tmp_dir, publish_dir)
# ensure that each listing file exists and has the correct contents
current_path = tmp_dir
for next_dir, expected_listing in zip(['a', 'b', 'c'], expected):
file_path = os.path.join(current_path, 'listing')
with open(file_path) as open_file:
self.assertEqual(open_file.read(), expected_listing)
current_path = os.path.join(current_path, next_dir)
# make sure there is not a listing file inside the repo's publish dir
self.assertFalse(os.path.exists(os.path.join(publish_dir, 'listing')))
finally:
shutil.rmtree(tmp_dir, ignore_errors=True)
def publish_repo(self, repo, publish_conduit, config):
publish_start_time = time.time()
_LOG.info("Start publish time %s" % publish_start_time)
source_repo_id = config.get('source_repo_id')
destination_dist_config = self.find_yum_distributor(repo.id)['config']
source_working_dir = self.source_working_dir(source_repo_id)
working_dir = self.full_working_dir(repo.id, publish_start_time)
self.safe_makedirs(working_dir)
#copy contents from source's working directory to destinations
if not self.copy_directory(source_working_dir, working_dir):
publish_conduit.set_progress(self.summary)
raise Exception(
"Failed to copy metadata. See errors for more details.")
#symlink the destination's publish directories
if destination_dist_config['http']:
http_publish_dir = os.path.join(
HTTP_PUBLISH_DIR,
destination_dist_config["relative_url"]).rstrip('/')
self.link_directory(working_dir, http_publish_dir)
util.generate_listing_files(HTTP_PUBLISH_DIR, http_publish_dir)
self.update_repomd(http_publish_dir)
if destination_dist_config['https']:
https_publish_dir = os.path.join(
HTTPS_PUBLISH_DIR,
destination_dist_config["relative_url"]).rstrip('/')
self.link_directory(working_dir, https_publish_dir)
util.generate_listing_files(HTTPS_PUBLISH_DIR, https_publish_dir)
self.update_repomd(https_publish_dir)
self.clean_path(self.base_working_dir(repo.id),
str(publish_start_time))
publish_conduit.set_progress(self.summary)
if len(self.summary["errors"]) > 0:
raise Exception(
"Failed to link metadata. See errors for more details.")
else:
return publish_conduit.build_success_report(
self.summary, self.details)
def process_main(self, item=None):
util.generate_listing_files(self.root_dir, self.target_dir)
def process_main(self):
util.generate_listing_files(self.root_dir, self.target_dir)
def publish_group(self, repo_group, publish_conduit, config):
"""
Publishes the given repository group.
:param repo_group: metadata describing the repository group
:type repo_group: pulp.plugins.model.RepositoryGroup
:param publish_conduit: provides access to relevant Pulp functionality
:type publish_conduit: pulp.plugins.conduits.repo_publish.RepoGroupPublishConduit
:param config: plugin configuration
:type config: pulp.plugins.config.PluginConfiguration
:return: report describing the publish run
:rtype: pulp.plugins.model.PublishReport
"""
# First, validate the configuration because there may be override config options, and currently,
# validate_config is not called prior to publishing by the manager.
valid_config, msg = export_utils.validate_export_config(config)
if not valid_config:
raise PulpDataException(msg)
_logger.info("Beginning export of the following repository group: [%s]" % repo_group.id)
# The progress report for a group publish
progress_status = {
constants.PROGRESS_REPOS_KEYWORD: {constants.PROGRESS_STATE_KEY: constants.STATE_NOT_STARTED},
constants.PROGRESS_ISOS_KEYWORD: {constants.PROGRESS_STATE_KEY: constants.STATE_NOT_STARTED},
constants.PROGRESS_PUBLISH_HTTP: {constants.PROGRESS_STATE_KEY: constants.STATE_NOT_STARTED},
constants.PROGRESS_PUBLISH_HTTPS: {constants.PROGRESS_STATE_KEY: constants.STATE_NOT_STARTED},
}
def progress_callback(progress_keyword, status):
"""
Progress callback used to update the progress report for the publish conduit
:param progress_keyword: The keyword to assign the status to in the progress report dict
:type progress_keyword: str
:param status: The status to assign to the keyword.
:type status: dict
"""
progress_status[progress_keyword] = status
publish_conduit.set_progress(progress_status)
# Before starting, clean out the working directory. Done to remove last published ISOs
shutil.rmtree(repo_group.working_dir, ignore_errors=True)
os.makedirs(repo_group.working_dir)
# Retrieve the configuration for each repository, the skip types, and the date filter
packed_config = export_utils.retrieve_group_export_config(repo_group, config)
rpm_repositories, self.date_filter = packed_config
# Update the progress for the repositories section
repos_progress = export_utils.init_progress_report(len(rpm_repositories))
progress_callback(constants.PROGRESS_REPOS_KEYWORD, repos_progress)
# For every repository, extract the requested types to the working directory
for repo_id, working_dir in rpm_repositories:
# Create a repo conduit, which makes sharing code with the export and yum distributors easier
repo_conduit = RepoPublishConduit(repo_id, publish_conduit.distributor_id)
# If there is a date filter perform an incremental export, otherwise do everything
if self.date_filter:
result = export_utils.export_incremental_content(working_dir, repo_conduit, self.date_filter)
else:
result = export_utils.export_complete_repo(repo_id, working_dir, repo_conduit, config)
if not config.get(constants.EXPORT_DIRECTORY_KEYWORD):
util.generate_listing_files(repo_group.working_dir, working_dir)
else:
export_dir = config.get(constants.EXPORT_DIRECTORY_KEYWORD)
util.generate_listing_files(export_dir, working_dir)
self.summary[repo_id] = result[0]
self.details[repo_id] = result[1]
repos_progress[constants.PROGRESS_ITEMS_LEFT_KEY] -= 1
repos_progress[constants.PROGRESS_NUM_SUCCESS_KEY] += 1
progress_callback(constants.PROGRESS_REPOS_KEYWORD, repos_progress)
repos_progress[constants.PROGRESS_STATE_KEY] = constants.STATE_COMPLETE
progress_callback(constants.PROGRESS_REPOS_KEYWORD, repos_progress)
# If there was no export directory, publish via ISOs
if not config.get(constants.EXPORT_DIRECTORY_KEYWORD):
self._publish_isos(repo_group, config, progress_callback)
for repo_id, repo_dir in rpm_repositories:
if repo_id in self.details and len(self.details[repo_id]["errors"]) != 0:
return publish_conduit.build_failure_report(self.summary, self.details)
self.summary["repositories_exported"] = len(rpm_repositories)
self.summary["repositories_skipped"] = len(repo_group.repo_ids) - len(rpm_repositories)
return publish_conduit.build_success_report(self.summary, self.details)
def publish_repo(self, repo, publish_conduit, config):
summary = {}
details = {}
progress_status = {
"packages": {"state": "NOT_STARTED"},
"distribution": {"state": "NOT_STARTED"},
"metadata": {"state": "NOT_STARTED"},
"packagegroups": {"state": "NOT_STARTED"},
"publish_http": {"state": "NOT_STARTED"},
"publish_https": {"state": "NOT_STARTED"},
}
def progress_callback(type_id, status):
progress_status[type_id] = status
publish_conduit.set_progress(progress_status)
self.repo_working_dir = repo.working_dir
if self.canceled:
return publish_conduit.build_cancel_report(summary, details)
skip_list = config.get('skip') or []
# Determine Content in this repo
pkg_units = []
pkg_errors = []
if 'rpm' not in skip_list:
for type_id in [TYPE_ID_RPM, TYPE_ID_SRPM]:
criteria = UnitAssociationCriteria(type_ids=type_id,
unit_fields=['id', 'name', 'version', 'release', 'arch', 'epoch', '_storage_path', "checksum", "checksumtype" ])
pkg_units += publish_conduit.get_units(criteria=criteria)
drpm_units = []
if 'drpm' not in skip_list:
criteria = UnitAssociationCriteria(type_ids=TYPE_ID_DRPM)
drpm_units = publish_conduit.get_units(criteria=criteria)
pkg_units += drpm_units
# Create symlinks under repo.working_dir
pkg_status, pkg_errors = self.handle_symlinks(pkg_units, repo.working_dir, progress_callback)
if not pkg_status:
_LOG.error("Unable to publish %s items" % (len(pkg_errors)))
distro_errors = []
distro_units = []
if 'distribution' not in skip_list:
criteria = UnitAssociationCriteria(type_ids=TYPE_ID_DISTRO)
distro_units = publish_conduit.get_units(criteria=criteria)
# symlink distribution files if any under repo.working_dir
distro_status, distro_errors = self.symlink_distribution_unit_files(distro_units, repo.working_dir, publish_conduit, progress_callback)
if not distro_status:
_LOG.error("Unable to publish distribution tree %s items" % (len(distro_errors)))
updateinfo_xml_path = None
if 'erratum' not in skip_list:
criteria = UnitAssociationCriteria(type_ids=TYPE_ID_ERRATA)
errata_units = publish_conduit.get_units(criteria=criteria)
updateinfo_xml_path = updateinfo.updateinfo(errata_units, repo.working_dir)
if self.canceled:
return publish_conduit.build_cancel_report(summary, details)
groups_xml_path = None
existing_cats = []
existing_groups = []
if 'packagegroup' not in skip_list:
criteria = UnitAssociationCriteria(type_ids=[TYPE_ID_PKG_GROUP, TYPE_ID_PKG_CATEGORY])
existing_units = publish_conduit.get_units(criteria)
existing_groups = filter(lambda u : u.type_id in [TYPE_ID_PKG_GROUP], existing_units)
existing_cats = filter(lambda u : u.type_id in [TYPE_ID_PKG_CATEGORY], existing_units)
groups_xml_path = comps_util.write_comps_xml(repo.working_dir, existing_groups, existing_cats)
metadata_start_time = time.time()
# update/generate metadata for the published repo
self.use_createrepo = config.get('use_createrepo')
if self.use_createrepo:
metadata_status, metadata_errors = metadata.generate_metadata(
repo.working_dir, publish_conduit, config, progress_callback, groups_xml_path)
else:
metadata_status, metadata_errors = metadata.generate_yum_metadata(repo.id, repo.working_dir, publish_conduit, config,
progress_callback, is_cancelled=self.canceled, group_xml_path=groups_xml_path, updateinfo_xml_path=updateinfo_xml_path,
repo_scratchpad=publish_conduit.get_repo_scratchpad())
metadata_end_time = time.time()
relpath = self.get_repo_relative_path(repo, config)
if relpath.startswith("/"):
relpath = relpath[1:]
# Build the https and http publishing paths
https_publish_dir = self.get_https_publish_dir(config)
https_repo_publish_dir = os.path.join(https_publish_dir, relpath).rstrip('/')
http_publish_dir = self.get_http_publish_dir(config)
http_repo_publish_dir = os.path.join(http_publish_dir, relpath).rstrip('/')
# Clean up the old publish directories, if they exist.
scratchpad = publish_conduit.get_repo_scratchpad()
if OLD_REL_PATH_KEYWORD in scratchpad:
old_relative_path = scratchpad[OLD_REL_PATH_KEYWORD]
old_https_repo_publish_dir = os.path.join(https_publish_dir, old_relative_path)
if os.path.exists(old_https_repo_publish_dir):
util.remove_repo_publish_dir(https_publish_dir, old_https_repo_publish_dir)
old_http_repo_publish_dir = os.path.join(http_publish_dir, old_relative_path)
if os.path.exists(old_http_repo_publish_dir):
util.remove_repo_publish_dir(http_publish_dir, old_http_repo_publish_dir)
# Now write the current publish relative path to the scratch pad. This way, if the relative path
# changes before the next publish, we can clean up the old path.
scratchpad[OLD_REL_PATH_KEYWORD] = relpath
publish_conduit.set_repo_scratchpad(scratchpad)
# Handle publish link for HTTPS
if config.get("https"):
# Publish for HTTPS
self.set_progress("publish_https", {"state" : "IN_PROGRESS"}, progress_callback)
try:
_LOG.info("HTTPS Publishing repo <%s> to <%s>" % (repo.id, https_repo_publish_dir))
util.create_symlink(repo.working_dir, https_repo_publish_dir)
util.generate_listing_files(https_publish_dir, https_repo_publish_dir)
summary["https_publish_dir"] = https_repo_publish_dir
self.set_progress("publish_https", {"state" : "FINISHED"}, progress_callback)
except:
self.set_progress("publish_https", {"state" : "FAILED"}, progress_callback)
else:
self.set_progress("publish_https", {"state" : "SKIPPED"}, progress_callback)
if os.path.lexists(https_repo_publish_dir):
_LOG.debug("Removing link for %s since https is not set" % https_repo_publish_dir)
util.remove_repo_publish_dir(https_publish_dir, https_repo_publish_dir)
# Handle publish link for HTTP
if config.get("http"):
# Publish for HTTP
self.set_progress("publish_http", {"state" : "IN_PROGRESS"}, progress_callback)
try:
_LOG.info("HTTP Publishing repo <%s> to <%s>" % (repo.id, http_repo_publish_dir))
util.create_symlink(repo.working_dir, http_repo_publish_dir)
util.generate_listing_files(http_publish_dir, http_repo_publish_dir)
summary["http_publish_dir"] = http_repo_publish_dir
self.set_progress("publish_http", {"state" : "FINISHED"}, progress_callback)
except:
self.set_progress("publish_http", {"state" : "FAILED"}, progress_callback)
else:
self.set_progress("publish_http", {"state" : "SKIPPED"}, progress_callback)
if os.path.lexists(http_repo_publish_dir):
_LOG.debug("Removing link for %s since http is not set" % http_repo_publish_dir)
util.remove_repo_publish_dir(http_publish_dir, http_repo_publish_dir)
summary["num_package_units_attempted"] = len(pkg_units)
summary["num_package_units_published"] = len(pkg_units) - len(pkg_errors)
summary["num_package_units_errors"] = len(pkg_errors)
summary["num_distribution_units_attempted"] = len(distro_units)
summary["num_distribution_units_published"] = len(distro_units) - len(distro_errors)
summary["num_distribution_units_errors"] = len(distro_errors)
summary["num_package_groups_published"] = len(existing_groups)
summary["num_package_categories_published"] = len(existing_cats)
summary["relative_path"] = relpath
if metadata_status is False and not len(metadata_errors):
summary["skip_metadata_update"] = True
else:
summary["skip_metadata_update"] = False
details["errors"] = pkg_errors + distro_errors # metadata_errors
details['time_metadata_sec'] = metadata_end_time - metadata_start_time
# metadata generate skipped vs run
_LOG.info("Publish complete: summary = <%s>, details = <%s>" % (summary, details))
if details["errors"]:
return publish_conduit.build_failure_report(summary, details)
return publish_conduit.build_success_report(summary, details)
def publish_repo(self, repo, publish_conduit, config):
"""
Export a yum repository to a given directory, or to ISO
:param repo: metadata describing the repository
:type repo: pulp.plugins.model.Repository
:param publish_conduit: provides access to relevant Pulp functionality
:type publish_conduit: pulp.plugins.conduits.repo_publish.RepoPublishConduit
:param config: plugin configuration
:type config: pulp.plugins.config.PluginConfiguration
:return: report describing the publish run
:rtype: pulp.plugins.model.PublishReport
"""
# First, validate the configuration because there may be override config options, and
# currently, validate_config is not called prior to publishing by the manager.
valid_config, msg = export_utils.validate_export_config(config)
if not valid_config:
raise PulpDataException(msg)
_logger.info('Starting export of [%s]' % repo.id)
progress_status = {
ids.TYPE_ID_RPM: {'state': constants.STATE_NOT_STARTED},
ids.TYPE_ID_ERRATA: {'state': constants.STATE_NOT_STARTED},
ids.TYPE_ID_DISTRO: {'state': constants.STATE_NOT_STARTED},
ids.TYPE_ID_PKG_CATEGORY: {'state': constants.STATE_NOT_STARTED},
ids.TYPE_ID_PKG_GROUP: {'state': constants.STATE_NOT_STARTED},
'metadata': {'state': constants.STATE_NOT_STARTED},
'isos': {'state': constants.STATE_NOT_STARTED},
'publish_http': {'state': constants.STATE_NOT_STARTED},
'publish_https': {'state': constants.STATE_NOT_STARTED},
}
def progress_callback(type_id, status):
progress_status[type_id] = status
publish_conduit.set_progress(progress_status)
# Retrieve a config tuple and unpack it for use
config_settings = export_utils.retrieve_repo_config(repo, config)
self.working_dir, self.date_filter = config_settings
# Before starting, clean out the working directory. Done to remove last published ISOs
shutil.rmtree(repo.working_dir, ignore_errors=True)
os.makedirs(repo.working_dir)
# If a date filter is not present, do a complete export. If it is, do an incremental export.
if self.date_filter:
result = export_utils.export_incremental_content(self.working_dir, publish_conduit,
self.date_filter, progress_callback)
else:
result = export_utils.export_complete_repo(repo.id, self.working_dir, publish_conduit,
config, progress_callback)
self.summary = result[0]
self.details = result[1]
if not config.get(constants.EXPORT_DIRECTORY_KEYWORD):
util.generate_listing_files(repo.working_dir, self.working_dir)
# build iso and publish via HTTPS
self._publish_isos(repo, config, progress_callback)
else:
export_dir = config.get(constants.EXPORT_DIRECTORY_KEYWORD)
util.generate_listing_files(export_dir, self.working_dir)
if len(self.details['errors']) != 0:
return publish_conduit.build_failure_report(self.summary, self.details)
return publish_conduit.build_success_report(self.summary, self.details)