def _publish_isos(self, repo_group, config, progress_callback=None): """ Generate the iso images on the exported directory of repos and publish them to the publish directory. Supports http/https publish. This does not support repo_auth. @param repo_group: metadata describing the repository group to which the configuration applies @type repo_group: pulp.plugins.model.RepositoryGroup @param config: plugin configuration instance; the proposed repo configuration is found within @type config: pulp.plugins.config.PluginCallConfiguration @param progress_callback: callback to report progress info to publish_conduit @type progress_callback: function """ # build iso and publish via HTTPS https_publish_dir = iso_util.get_https_publish_iso_dir(config) https_repo_publish_dir = os.path.join(https_publish_dir, repo_group.id).rstrip('/') prefix = config.get('iso_prefix') or repo_group.id if config.get("https"): # Publish for HTTPS self.set_progress("publish_https", {"state" : "IN_PROGRESS"}, progress_callback) try: _LOG.info("HTTPS Publishing repo <%s> to <%s>" % (repo_group.id, https_repo_publish_dir)) # generate iso images and write them to the publish directory isogen = GenerateIsos(self.group_working_dir, https_repo_publish_dir, prefix=prefix, progress=self.init_progress(), is_cancelled=self.canceled) isogen.run(progress_callback=progress_callback) self.group_summary["https_publish_dir"] = https_repo_publish_dir self.set_progress("publish_https", {"state" : "FINISHED"}, progress_callback) except Exception,e: _LOG.debug(" publish operaation failed due to exception: %s" % e) self.set_progress("publish_https", {"state" : "FAILED"}, progress_callback)
def _publish_isos(self, repo, config, progress_callback=None): """ Generate the iso images on the exported repo and publish them to the publish directory. Supports http/https publish. This does not support repo_auth. @param repo: metadata describing the repository to which the configuration applies @type repo: pulp.plugins.model.Repository @param config: plugin configuration instance; the proposed repo configuration is found within @type config: pulp.plugins.config.PluginCallConfiguration @param progress_callback: callback to report progress info to publish_conduit @type progress_callback: function """ # build iso and publish via HTTPS https_publish_dir = iso_util.get_https_publish_iso_dir(config) https_repo_publish_dir = os.path.join(https_publish_dir, repo.id).rstrip('/') prefix = config.get('iso_prefix') or repo.id if config.get("https"): # Publish for HTTPS self.set_progress("publish_https", {"state" : "IN_PROGRESS"}, progress_callback) try: _LOG.info("HTTPS Publishing repo <%s> to <%s>" % (repo.id, https_repo_publish_dir)) isogen = GenerateIsos(self.repo_working_dir, https_repo_publish_dir, prefix=prefix, progress=self.init_progress()) isogen.run(progress_callback=progress_callback) self.summary["https_publish_dir"] = https_repo_publish_dir self.set_progress("publish_https", {"state" : "FINISHED"}, progress_callback) except: self.set_progress("publish_https", {"state" : "FAILED"}, progress_callback) else: self.set_progress("publish_https", {"state" : "SKIPPED"}, progress_callback) if os.path.lexists(https_repo_publish_dir): _LOG.debug("Removing link for %s since https is not set" % https_repo_publish_dir) shutil.rmtree(https_repo_publish_dir) # build iso and publish via HTTP http_publish_dir = iso_util.get_http_publish_iso_dir(config) http_repo_publish_dir = os.path.join(http_publish_dir, repo.id).rstrip('/') if config.get("http"): # Publish for HTTP self.set_progress("publish_http", {"state" : "IN_PROGRESS"}, progress_callback) try: _LOG.info("HTTP Publishing repo <%s> to <%s>" % (repo.id, http_repo_publish_dir)) isogen = GenerateIsos(self.repo_working_dir, http_repo_publish_dir, prefix=prefix, progress=self.init_progress()) isogen.run(progress_callback=progress_callback) self.summary["http_publish_dir"] = http_repo_publish_dir self.set_progress("publish_http", {"state" : "FINISHED"}, progress_callback) except: self.set_progress("publish_http", {"state" : "FAILED"}, progress_callback) else: self.set_progress("publish_http", {"state" : "SKIPPED"}, progress_callback) if os.path.lexists(http_repo_publish_dir): _LOG.debug("Removing link for %s since http is not set" % http_repo_publish_dir) shutil.rmtree(http_repo_publish_dir)
except Exception,e: _LOG.debug(" publish operaation failed due to exception: %s" % e) self.set_progress("publish_https", {"state" : "FAILED"}, progress_callback) else: self.set_progress("publish_https", {"state" : "SKIPPED"}, progress_callback) if os.path.lexists(https_repo_publish_dir): _LOG.debug("Removing link for %s since https is not set" % https_repo_publish_dir) shutil.rmtree(https_repo_publish_dir) # build iso and publish via HTTP http_publish_dir = iso_util.get_http_publish_iso_dir(config) http_repo_publish_dir = os.path.join(http_publish_dir, repo_group.id).rstrip('/') if config.get("http"): # Publish for HTTP self.set_progress("publish_http", {"state" : "IN_PROGRESS"}, progress_callback) try: _LOG.info("HTTP Publishing repo <%s> to <%s>" % (repo_group.id, http_repo_publish_dir)) # generate iso images and write them to the publish directory isogen = GenerateIsos(self.group_working_dir, http_repo_publish_dir, prefix=prefix, progress=self.init_progress(), is_cancelled=self.canceled) isogen.run(progress_callback=progress_callback) self.group_summary["http_publish_dir"] = http_repo_publish_dir self.set_progress("publish_http", {"state" : "FINISHED"}, progress_callback) except Exception,e: _LOG.debug(" publish operaation failed due to exception: %s" % e) self.set_progress("publish_http", {"state" : "FAILED"}, progress_callback) else: self.set_progress("publish_http", {"state" : "SKIPPED"}, progress_callback) if os.path.lexists(http_repo_publish_dir): _LOG.debug("Removing link for %s since http is not set" % http_repo_publish_dir) shutil.rmtree(http_repo_publish_dir)
def publish_repo(self, repo, publish_conduit, config): publish_start_time = time.time() _LOG.info("Start publish time %s" % publish_start_time) summary = {} details = {} progress_status = { "rpms": {"state": "NOT_STARTED"}, "errata": {"state": "NOT_STARTED"}, "distribution": {"state": "NOT_STARTED"}, "packagegroups": {"state": "NOT_STARTED"}, "isos": {"state": "NOT_STARTED"}, "publish_http": {"state": "NOT_STARTED"}, } def progress_callback(type_id, status): progress_status[type_id] = status publish_conduit.set_progress(progress_status) self.repo_working_dir = repo_working_dir = repo.working_dir if self.canceled: return publish_conduit.build_failure_report(summary, details) date_filter = self.create_date_range_filter(config) if date_filter: # export errata by date and associated rpm units progress_status["errata"]["state"] = "STARTED" criteria = UnitAssociationCriteria(type_ids=[TYPE_ID_ERRATA], unit_filters=date_filter) errata_units = publish_conduit.get_units(criteria) criteria = UnitAssociationCriteria(type_ids=[TYPE_ID_RPM, TYPE_ID_SRPM, TYPE_ID_DRPM]) rpm_units = publish_conduit.get_units(criteria) rpm_units = self._get_errata_rpms(errata_units, rpm_units) rpm_status, rpm_errors = self._export_rpms(rpm_units, repo_working_dir, progress_callback=progress_callback) progress_status["rpms"]["state"] = "FINISHED" if self.canceled: return publish_conduit.build_failure_report(summary, details) # generate metadata metadata_status, metadata_errors = metadata.generate_metadata( repo, publish_conduit, config, progress_callback) _LOG.info("metadata generation complete at target location %s" % repo_working_dir) errata_status, errata_errors = self._export_errata(errata_units, repo_working_dir, progress_callback=progress_callback) progress_status["errata"]["state"] = "FINISHED" summary["num_package_units_attempted"] = len(rpm_units) summary["num_package_units_exported"] = len(rpm_units) - len(rpm_errors) summary["num_package_units_errors"] = len(rpm_errors) details["errors"] = rpm_errors + errata_errors else: # export everything progress_status["rpms"]["state"] = "STARTED" criteria = UnitAssociationCriteria(type_ids=[TYPE_ID_RPM, TYPE_ID_SRPM, TYPE_ID_DRPM]) rpm_units = publish_conduit.get_units(criteria) rpm_status, rpm_errors = self._export_rpms(rpm_units, repo_working_dir, progress_callback=progress_callback) progress_status["rpms"]["state"] = "FINISHED" # package groups progress_status["packagegroups"]["state"] = "STARTED" criteria = UnitAssociationCriteria(type_ids=[TYPE_ID_PKG_GROUP, TYPE_ID_PKG_CATEGORY]) existing_units = publish_conduit.get_units(criteria) existing_groups = filter(lambda u : u.type_id in [TYPE_ID_PKG_GROUP], existing_units) existing_cats = filter(lambda u : u.type_id in [TYPE_ID_PKG_CATEGORY], existing_units) groups_xml_path = comps_util.write_comps_xml(repo, existing_groups, existing_cats) if self.canceled: return publish_conduit.build_failure_report(summary, details) # generate metadata metadata_status, metadata_errors = metadata.generate_metadata( repo, publish_conduit, config, progress_callback, groups_xml_path) _LOG.info("metadata generation complete at target location %s" % repo_working_dir) progress_status["packagegroups"]["state"] = "FINISHED" progress_status["errata"]["state"] = "STARTED" criteria = UnitAssociationCriteria(type_ids=[TYPE_ID_ERRATA]) errata_units = publish_conduit.get_units(criteria) rpm_units = self._get_errata_rpms(errata_units, rpm_units) self._export_rpms(rpm_units, repo_working_dir, progress_callback=progress_callback) errata_status, errata_errors = self._export_errata(errata_units, repo_working_dir, progress_callback=progress_callback) progress_status["errata"]["state"] = "FINISHED" # distro units progress_status["distribution"]["state"] = "STARTED" criteria = UnitAssociationCriteria(type_ids=[TYPE_ID_DISTRO]) distro_units = publish_conduit.get_units(criteria) distro_status, distro_errors = self._export_distributions(distro_units, repo_working_dir, progress_callback=progress_callback) progress_status["distribution"]["state"] = "FINISHED" summary["num_distribution_units_attempted"] = len(distro_units) summary["num_distribution_units_exported"] = len(distro_units) - len(distro_errors) summary["num_distribution_units_errors"] = len(distro_errors) summary["num_package_groups_exported"] = len(existing_groups) summary["num_package_categories_exported"] = len(existing_cats) summary["num_package_units_attempted"] = len(rpm_units) summary["num_package_units_exported"] = len(rpm_units) - len(rpm_errors) summary["num_package_units_errors"] = len(rpm_errors) details["errors"] = rpm_errors + distro_errors + errata_errors # build iso and publish via HTTPS https_publish_dir = self.get_https_publish_iso_dir(config) https_repo_publish_dir = os.path.join(https_publish_dir, repo.id).rstrip('/') prefix = config.get('iso_prefix') or repo.id if config.get("https"): # Publish for HTTPS self.set_progress("publish_https", {"state" : "IN_PROGRESS"}, progress_callback) try: _LOG.info("HTTPS Publishing repo <%s> to <%s>" % (repo.id, https_repo_publish_dir)) isogen = GenerateIsos(repo_working_dir, https_repo_publish_dir, prefix=prefix, progress=progress_status) progress_status = isogen.run() summary["https_publish_dir"] = https_repo_publish_dir self.set_progress("publish_https", {"state" : "FINISHED"}, progress_callback) progress_status["isos"]["state"] = "FINISHED" except: self.set_progress("publish_https", {"state" : "FAILED"}, progress_callback) else: self.set_progress("publish_https", {"state" : "SKIPPED"}, progress_callback) if os.path.lexists(https_repo_publish_dir): _LOG.debug("Removing link for %s since https is not set" % https_repo_publish_dir) shutil.rmtree(https_repo_publish_dir) # Handle publish link for HTTP # build iso and publish via HTTP http_publish_dir = self.get_http_publish_iso_dir(config) http_repo_publish_dir = os.path.join(http_publish_dir, repo.id).rstrip('/') if config.get("http"): # Publish for HTTP self.set_progress("publish_http", {"state" : "IN_PROGRESS"}, progress_callback) try: _LOG.info("HTTP Publishing repo <%s> to <%s>" % (repo.id, http_repo_publish_dir)) isogen = GenerateIsos(repo_working_dir, http_repo_publish_dir, prefix=prefix, progress=progress_status) progress_status = isogen.run() summary["http_publish_dir"] = http_repo_publish_dir self.set_progress("publish_http", {"state" : "FINISHED"}, progress_callback) progress_status["isos"]["state"] = "FINISHED" except: self.set_progress("publish_http", {"state" : "FAILED"}, progress_callback) else: self.set_progress("publish_http", {"state" : "SKIPPED"}, progress_callback) if os.path.lexists(http_repo_publish_dir): _LOG.debug("Removing link for %s since http is not set" % http_repo_publish_dir) shutil.rmtree(http_repo_publish_dir) details["errors"] += metadata_errors # metadata generate skipped vs run _LOG.info("Publish complete: summary = <%s>, details = <%s>" % (summary, details)) # remove exported content from working dirctory self.cleanup() if details["errors"]: return publish_conduit.build_failure_report(summary, details) return publish_conduit.build_success_report(summary, details)