def _publish_isos(self, repo_group, config, progress_callback=None):
"""
Generate the iso images on the exported directory of repos and publish
them to the publish directory. Supports http/https publish. This does not
support repo_auth.
@param repo_group: metadata describing the repository group to which the
configuration applies
@type repo_group: pulp.plugins.model.RepositoryGroup
@param config: plugin configuration instance; the proposed repo
configuration is found within
@type config: pulp.plugins.config.PluginCallConfiguration
@param progress_callback: callback to report progress info to publish_conduit
@type progress_callback: function
"""
# build iso and publish via HTTPS
https_publish_dir = iso_util.get_https_publish_iso_dir(config)
https_repo_publish_dir = os.path.join(https_publish_dir, repo_group.id).rstrip('/')
prefix = config.get('iso_prefix') or repo_group.id
if config.get("https"):
# Publish for HTTPS
self.set_progress("publish_https", {"state" : "IN_PROGRESS"}, progress_callback)
try:
_LOG.info("HTTPS Publishing repo <%s> to <%s>" % (repo_group.id, https_repo_publish_dir))
# generate iso images and write them to the publish directory
isogen = GenerateIsos(self.group_working_dir, https_repo_publish_dir, prefix=prefix, progress=self.init_progress(), is_cancelled=self.canceled)
isogen.run(progress_callback=progress_callback)
self.group_summary["https_publish_dir"] = https_repo_publish_dir
self.set_progress("publish_https", {"state" : "FINISHED"}, progress_callback)
except Exception,e:
_LOG.debug(" publish operaation failed due to exception: %s" % e)
self.set_progress("publish_https", {"state" : "FAILED"}, progress_callback)
def generate_isos(self, repo_working_dir, publish_dir, prefix, progress_callback=None):
"""
generate iso images for the exported directory
"""
iso_progress_status = self.init_progress()
iso_progress_status['state'] = "IN_PROGRESS"
self.set_progress("isos", iso_progress_status, progress_callback)
isogen = GenerateIsos(repo_working_dir, publish_dir, prefix=prefix, progress=progress_callback)
# get size and filelists of the target directory
filelist, total_dir_size = isogen.list_dir_with_size(repo_working_dir)
_LOG.debug("Total target directory size to create isos %s" % total_dir_size)
# media size
img_size = isogen.get_image_type_size(total_dir_size)
# compute no.of images it takes per media image size
imgcount = int(math.ceil(total_dir_size/float(img_size)))
# get the filelists per image by size
imgs = isogen.compute_image_files(filelist, imgcount, img_size)
iso_progress_status['items_total'] = imgcount
iso_progress_status['items_left'] = imgcount
iso_progress_status["size_total"] = total_dir_size
iso_progress_status["size_left"] = total_dir_size
for i in range(imgcount):
self.set_progress("isos", iso_progress_status, progress_callback)
msg = "Generating iso images for exported content (%s/%s)" % (i+1, imgcount)
_LOG.info(msg)
grafts = isogen.get_grafts(imgs[i])
pathfiles_fd, pathfiles = isogen.get_pathspecs(grafts)
filename = isogen.get_iso_filename(publish_dir, prefix, i+1)
cmd = isogen.get_mkisofs_template() % (string.join([pathfiles]), filename)
status, out = isogen.run_command(cmd)
if status != 0:
_LOG.error("Error creating iso %s" % filename)
_LOG.info("successfully created iso %s" % filename)
_LOG.debug("status code: %s; output: %s" % (status, out))
os.unlink(pathfiles)
iso_progress_status['items_left'] -= 1
iso_progress_status['num_success'] += 1
if iso_progress_status["size_left"] > img_size:
iso_progress_status["size_left"] -= img_size
else:
iso_progress_status["size_left"] = 0
iso_progress_status["state"] = "FINISHED"
self.set_progress("isos", iso_progress_status, progress_callback)
return True, []
def _publish_isos(self, repo, config, progress_callback=None):
"""
Generate the iso images on the exported repo and publish
them to the publish directory. Supports http/https publish. This does not
support repo_auth.
@param repo: metadata describing the repository to which the
configuration applies
@type repo: pulp.plugins.model.Repository
@param config: plugin configuration instance; the proposed repo
configuration is found within
@type config: pulp.plugins.config.PluginCallConfiguration
@param progress_callback: callback to report progress info to publish_conduit
@type progress_callback: function
"""
# build iso and publish via HTTPS
https_publish_dir = iso_util.get_https_publish_iso_dir(config)
https_repo_publish_dir = os.path.join(https_publish_dir, repo.id).rstrip('/')
prefix = config.get('iso_prefix') or repo.id
if config.get("https"):
# Publish for HTTPS
self.set_progress("publish_https", {"state" : "IN_PROGRESS"}, progress_callback)
try:
_LOG.info("HTTPS Publishing repo <%s> to <%s>" % (repo.id, https_repo_publish_dir))
isogen = GenerateIsos(self.repo_working_dir, https_repo_publish_dir, prefix=prefix, progress=self.init_progress())
isogen.run(progress_callback=progress_callback)
self.summary["https_publish_dir"] = https_repo_publish_dir
self.set_progress("publish_https", {"state" : "FINISHED"}, progress_callback)
except:
self.set_progress("publish_https", {"state" : "FAILED"}, progress_callback)
else:
self.set_progress("publish_https", {"state" : "SKIPPED"}, progress_callback)
if os.path.lexists(https_repo_publish_dir):
_LOG.debug("Removing link for %s since https is not set" % https_repo_publish_dir)
shutil.rmtree(https_repo_publish_dir)
# build iso and publish via HTTP
http_publish_dir = iso_util.get_http_publish_iso_dir(config)
http_repo_publish_dir = os.path.join(http_publish_dir, repo.id).rstrip('/')
if config.get("http"):
# Publish for HTTP
self.set_progress("publish_http", {"state" : "IN_PROGRESS"}, progress_callback)
try:
_LOG.info("HTTP Publishing repo <%s> to <%s>" % (repo.id, http_repo_publish_dir))
isogen = GenerateIsos(self.repo_working_dir, http_repo_publish_dir, prefix=prefix, progress=self.init_progress())
isogen.run(progress_callback=progress_callback)
self.summary["http_publish_dir"] = http_repo_publish_dir
self.set_progress("publish_http", {"state" : "FINISHED"}, progress_callback)
except:
self.set_progress("publish_http", {"state" : "FAILED"}, progress_callback)
else:
self.set_progress("publish_http", {"state" : "SKIPPED"}, progress_callback)
if os.path.lexists(http_repo_publish_dir):
_LOG.debug("Removing link for %s since http is not set" % http_repo_publish_dir)
shutil.rmtree(http_repo_publish_dir)
except Exception,e:
_LOG.debug(" publish operaation failed due to exception: %s" % e)
self.set_progress("publish_https", {"state" : "FAILED"}, progress_callback)
else:
self.set_progress("publish_https", {"state" : "SKIPPED"}, progress_callback)
if os.path.lexists(https_repo_publish_dir):
_LOG.debug("Removing link for %s since https is not set" % https_repo_publish_dir)
shutil.rmtree(https_repo_publish_dir)
# build iso and publish via HTTP
http_publish_dir = iso_util.get_http_publish_iso_dir(config)
http_repo_publish_dir = os.path.join(http_publish_dir, repo_group.id).rstrip('/')
if config.get("http"):
# Publish for HTTP
self.set_progress("publish_http", {"state" : "IN_PROGRESS"}, progress_callback)
try:
_LOG.info("HTTP Publishing repo <%s> to <%s>" % (repo_group.id, http_repo_publish_dir))
# generate iso images and write them to the publish directory
isogen = GenerateIsos(self.group_working_dir, http_repo_publish_dir, prefix=prefix, progress=self.init_progress(), is_cancelled=self.canceled)
isogen.run(progress_callback=progress_callback)
self.group_summary["http_publish_dir"] = http_repo_publish_dir
self.set_progress("publish_http", {"state" : "FINISHED"}, progress_callback)
except Exception,e:
_LOG.debug(" publish operaation failed due to exception: %s" % e)
self.set_progress("publish_http", {"state" : "FAILED"}, progress_callback)
else:
self.set_progress("publish_http", {"state" : "SKIPPED"}, progress_callback)
if os.path.lexists(http_repo_publish_dir):
_LOG.debug("Removing link for %s since http is not set" % http_repo_publish_dir)
shutil.rmtree(http_repo_publish_dir)
def publish_repo(self, repo, publish_conduit, config):
publish_start_time = time.time()
_LOG.info("Start publish time %s" % publish_start_time)
summary = {}
details = {}
progress_status = {
"rpms": {"state": "NOT_STARTED"},
"errata": {"state": "NOT_STARTED"},
"distribution": {"state": "NOT_STARTED"},
"packagegroups": {"state": "NOT_STARTED"},
"isos": {"state": "NOT_STARTED"},
"publish_http": {"state": "NOT_STARTED"},
}
def progress_callback(type_id, status):
progress_status[type_id] = status
publish_conduit.set_progress(progress_status)
self.repo_working_dir = repo_working_dir = repo.working_dir
if self.canceled:
return publish_conduit.build_failure_report(summary, details)
date_filter = self.create_date_range_filter(config)
if date_filter:
# export errata by date and associated rpm units
progress_status["errata"]["state"] = "STARTED"
criteria = UnitAssociationCriteria(type_ids=[TYPE_ID_ERRATA], unit_filters=date_filter)
errata_units = publish_conduit.get_units(criteria)
criteria = UnitAssociationCriteria(type_ids=[TYPE_ID_RPM, TYPE_ID_SRPM, TYPE_ID_DRPM])
rpm_units = publish_conduit.get_units(criteria)
rpm_units = self._get_errata_rpms(errata_units, rpm_units)
rpm_status, rpm_errors = self._export_rpms(rpm_units, repo_working_dir, progress_callback=progress_callback)
progress_status["rpms"]["state"] = "FINISHED"
if self.canceled:
return publish_conduit.build_failure_report(summary, details)
# generate metadata
metadata_status, metadata_errors = metadata.generate_metadata(
repo, publish_conduit, config, progress_callback)
_LOG.info("metadata generation complete at target location %s" % repo_working_dir)
errata_status, errata_errors = self._export_errata(errata_units, repo_working_dir, progress_callback=progress_callback)
progress_status["errata"]["state"] = "FINISHED"
summary["num_package_units_attempted"] = len(rpm_units)
summary["num_package_units_exported"] = len(rpm_units) - len(rpm_errors)
summary["num_package_units_errors"] = len(rpm_errors)
details["errors"] = rpm_errors + errata_errors
else:
# export everything
progress_status["rpms"]["state"] = "STARTED"
criteria = UnitAssociationCriteria(type_ids=[TYPE_ID_RPM, TYPE_ID_SRPM, TYPE_ID_DRPM])
rpm_units = publish_conduit.get_units(criteria)
rpm_status, rpm_errors = self._export_rpms(rpm_units, repo_working_dir, progress_callback=progress_callback)
progress_status["rpms"]["state"] = "FINISHED"
# package groups
progress_status["packagegroups"]["state"] = "STARTED"
criteria = UnitAssociationCriteria(type_ids=[TYPE_ID_PKG_GROUP, TYPE_ID_PKG_CATEGORY])
existing_units = publish_conduit.get_units(criteria)
existing_groups = filter(lambda u : u.type_id in [TYPE_ID_PKG_GROUP], existing_units)
existing_cats = filter(lambda u : u.type_id in [TYPE_ID_PKG_CATEGORY], existing_units)
groups_xml_path = comps_util.write_comps_xml(repo, existing_groups, existing_cats)
if self.canceled:
return publish_conduit.build_failure_report(summary, details)
# generate metadata
metadata_status, metadata_errors = metadata.generate_metadata(
repo, publish_conduit, config, progress_callback, groups_xml_path)
_LOG.info("metadata generation complete at target location %s" % repo_working_dir)
progress_status["packagegroups"]["state"] = "FINISHED"
progress_status["errata"]["state"] = "STARTED"
criteria = UnitAssociationCriteria(type_ids=[TYPE_ID_ERRATA])
errata_units = publish_conduit.get_units(criteria)
rpm_units = self._get_errata_rpms(errata_units, rpm_units)
self._export_rpms(rpm_units, repo_working_dir, progress_callback=progress_callback)
errata_status, errata_errors = self._export_errata(errata_units, repo_working_dir, progress_callback=progress_callback)
progress_status["errata"]["state"] = "FINISHED"
# distro units
progress_status["distribution"]["state"] = "STARTED"
criteria = UnitAssociationCriteria(type_ids=[TYPE_ID_DISTRO])
distro_units = publish_conduit.get_units(criteria)
distro_status, distro_errors = self._export_distributions(distro_units, repo_working_dir, progress_callback=progress_callback)
progress_status["distribution"]["state"] = "FINISHED"
summary["num_distribution_units_attempted"] = len(distro_units)
summary["num_distribution_units_exported"] = len(distro_units) - len(distro_errors)
summary["num_distribution_units_errors"] = len(distro_errors)
summary["num_package_groups_exported"] = len(existing_groups)
summary["num_package_categories_exported"] = len(existing_cats)
summary["num_package_units_attempted"] = len(rpm_units)
summary["num_package_units_exported"] = len(rpm_units) - len(rpm_errors)
summary["num_package_units_errors"] = len(rpm_errors)
details["errors"] = rpm_errors + distro_errors + errata_errors
# build iso and publish via HTTPS
https_publish_dir = self.get_https_publish_iso_dir(config)
https_repo_publish_dir = os.path.join(https_publish_dir, repo.id).rstrip('/')
prefix = config.get('iso_prefix') or repo.id
if config.get("https"):
# Publish for HTTPS
self.set_progress("publish_https", {"state" : "IN_PROGRESS"}, progress_callback)
try:
_LOG.info("HTTPS Publishing repo <%s> to <%s>" % (repo.id, https_repo_publish_dir))
isogen = GenerateIsos(repo_working_dir, https_repo_publish_dir, prefix=prefix, progress=progress_status)
progress_status = isogen.run()
summary["https_publish_dir"] = https_repo_publish_dir
self.set_progress("publish_https", {"state" : "FINISHED"}, progress_callback)
progress_status["isos"]["state"] = "FINISHED"
except:
self.set_progress("publish_https", {"state" : "FAILED"}, progress_callback)
else:
self.set_progress("publish_https", {"state" : "SKIPPED"}, progress_callback)
if os.path.lexists(https_repo_publish_dir):
_LOG.debug("Removing link for %s since https is not set" % https_repo_publish_dir)
shutil.rmtree(https_repo_publish_dir)
# Handle publish link for HTTP
# build iso and publish via HTTP
http_publish_dir = self.get_http_publish_iso_dir(config)
http_repo_publish_dir = os.path.join(http_publish_dir, repo.id).rstrip('/')
if config.get("http"):
# Publish for HTTP
self.set_progress("publish_http", {"state" : "IN_PROGRESS"}, progress_callback)
try:
_LOG.info("HTTP Publishing repo <%s> to <%s>" % (repo.id, http_repo_publish_dir))
isogen = GenerateIsos(repo_working_dir, http_repo_publish_dir, prefix=prefix, progress=progress_status)
progress_status = isogen.run()
summary["http_publish_dir"] = http_repo_publish_dir
self.set_progress("publish_http", {"state" : "FINISHED"}, progress_callback)
progress_status["isos"]["state"] = "FINISHED"
except:
self.set_progress("publish_http", {"state" : "FAILED"}, progress_callback)
else:
self.set_progress("publish_http", {"state" : "SKIPPED"}, progress_callback)
if os.path.lexists(http_repo_publish_dir):
_LOG.debug("Removing link for %s since http is not set" % http_repo_publish_dir)
shutil.rmtree(http_repo_publish_dir)
details["errors"] += metadata_errors
# metadata generate skipped vs run
_LOG.info("Publish complete: summary = <%s>, details = <%s>" % (summary, details))
# remove exported content from working dirctory
self.cleanup()
if details["errors"]:
return publish_conduit.build_failure_report(summary, details)
return publish_conduit.build_success_report(summary, details)
