def _publish_isos(self, repo_group, config, progress_callback=None): """ Generate the iso images on the exported directory of repos and publish them to the publish directory. Supports http/https publish. This does not support repo_auth. @param repo_group: metadata describing the repository group to which the configuration applies @type repo_group: pulp.plugins.model.RepositoryGroup @param config: plugin configuration instance; the proposed repo configuration is found within @type config: pulp.plugins.config.PluginCallConfiguration @param progress_callback: callback to report progress info to publish_conduit @type progress_callback: function """ # build iso and publish via HTTPS https_publish_dir = iso_util.get_https_publish_iso_dir(config) https_repo_publish_dir = os.path.join(https_publish_dir, repo_group.id).rstrip('/') prefix = config.get('iso_prefix') or repo_group.id if config.get("https"): # Publish for HTTPS self.set_progress("publish_https", {"state" : "IN_PROGRESS"}, progress_callback) try: _LOG.info("HTTPS Publishing repo <%s> to <%s>" % (repo_group.id, https_repo_publish_dir)) # generate iso images and write them to the publish directory isogen = GenerateIsos(self.group_working_dir, https_repo_publish_dir, prefix=prefix, progress=self.init_progress(), is_cancelled=self.canceled) isogen.run(progress_callback=progress_callback) self.group_summary["https_publish_dir"] = https_repo_publish_dir self.set_progress("publish_https", {"state" : "FINISHED"}, progress_callback) except Exception,e: _LOG.debug(" publish operaation failed due to exception: %s" % e) self.set_progress("publish_https", {"state" : "FAILED"}, progress_callback)
def generate_isos(self, repo_working_dir, publish_dir, prefix, progress_callback=None): """ generate iso images for the exported directory """ iso_progress_status = self.init_progress() iso_progress_status['state'] = "IN_PROGRESS" self.set_progress("isos", iso_progress_status, progress_callback) isogen = GenerateIsos(repo_working_dir, publish_dir, prefix=prefix, progress=progress_callback) # get size and filelists of the target directory filelist, total_dir_size = isogen.list_dir_with_size(repo_working_dir) _LOG.debug("Total target directory size to create isos %s" % total_dir_size) # media size img_size = isogen.get_image_type_size(total_dir_size) # compute no.of images it takes per media image size imgcount = int(math.ceil(total_dir_size/float(img_size))) # get the filelists per image by size imgs = isogen.compute_image_files(filelist, imgcount, img_size) iso_progress_status['items_total'] = imgcount iso_progress_status['items_left'] = imgcount iso_progress_status["size_total"] = total_dir_size iso_progress_status["size_left"] = total_dir_size for i in range(imgcount): self.set_progress("isos", iso_progress_status, progress_callback) msg = "Generating iso images for exported content (%s/%s)" % (i+1, imgcount) _LOG.info(msg) grafts = isogen.get_grafts(imgs[i]) pathfiles_fd, pathfiles = isogen.get_pathspecs(grafts) filename = isogen.get_iso_filename(publish_dir, prefix, i+1) cmd = isogen.get_mkisofs_template() % (string.join([pathfiles]), filename) status, out = isogen.run_command(cmd) if status != 0: _LOG.error("Error creating iso %s" % filename) _LOG.info("successfully created iso %s" % filename) _LOG.debug("status code: %s; output: %s" % (status, out)) os.unlink(pathfiles) iso_progress_status['items_left'] -= 1 iso_progress_status['num_success'] += 1 if iso_progress_status["size_left"] > img_size: iso_progress_status["size_left"] -= img_size else: iso_progress_status["size_left"] = 0 iso_progress_status["state"] = "FINISHED" self.set_progress("isos", iso_progress_status, progress_callback) return True, []
def _publish_isos(self, repo, config, progress_callback=None): """ Generate the iso images on the exported repo and publish them to the publish directory. Supports http/https publish. This does not support repo_auth. @param repo: metadata describing the repository to which the configuration applies @type repo: pulp.plugins.model.Repository @param config: plugin configuration instance; the proposed repo configuration is found within @type config: pulp.plugins.config.PluginCallConfiguration @param progress_callback: callback to report progress info to publish_conduit @type progress_callback: function """ # build iso and publish via HTTPS https_publish_dir = iso_util.get_https_publish_iso_dir(config) https_repo_publish_dir = os.path.join(https_publish_dir, repo.id).rstrip('/') prefix = config.get('iso_prefix') or repo.id if config.get("https"): # Publish for HTTPS self.set_progress("publish_https", {"state" : "IN_PROGRESS"}, progress_callback) try: _LOG.info("HTTPS Publishing repo <%s> to <%s>" % (repo.id, https_repo_publish_dir)) isogen = GenerateIsos(self.repo_working_dir, https_repo_publish_dir, prefix=prefix, progress=self.init_progress()) isogen.run(progress_callback=progress_callback) self.summary["https_publish_dir"] = https_repo_publish_dir self.set_progress("publish_https", {"state" : "FINISHED"}, progress_callback) except: self.set_progress("publish_https", {"state" : "FAILED"}, progress_callback) else: self.set_progress("publish_https", {"state" : "SKIPPED"}, progress_callback) if os.path.lexists(https_repo_publish_dir): _LOG.debug("Removing link for %s since https is not set" % https_repo_publish_dir) shutil.rmtree(https_repo_publish_dir) # build iso and publish via HTTP http_publish_dir = iso_util.get_http_publish_iso_dir(config) http_repo_publish_dir = os.path.join(http_publish_dir, repo.id).rstrip('/') if config.get("http"): # Publish for HTTP self.set_progress("publish_http", {"state" : "IN_PROGRESS"}, progress_callback) try: _LOG.info("HTTP Publishing repo <%s> to <%s>" % (repo.id, http_repo_publish_dir)) isogen = GenerateIsos(self.repo_working_dir, http_repo_publish_dir, prefix=prefix, progress=self.init_progress()) isogen.run(progress_callback=progress_callback) self.summary["http_publish_dir"] = http_repo_publish_dir self.set_progress("publish_http", {"state" : "FINISHED"}, progress_callback) except: self.set_progress("publish_http", {"state" : "FAILED"}, progress_callback) else: self.set_progress("publish_http", {"state" : "SKIPPED"}, progress_callback) if os.path.lexists(http_repo_publish_dir): _LOG.debug("Removing link for %s since http is not set" % http_repo_publish_dir) shutil.rmtree(http_repo_publish_dir)
except Exception,e: _LOG.debug(" publish operaation failed due to exception: %s" % e) self.set_progress("publish_https", {"state" : "FAILED"}, progress_callback) else: self.set_progress("publish_https", {"state" : "SKIPPED"}, progress_callback) if os.path.lexists(https_repo_publish_dir): _LOG.debug("Removing link for %s since https is not set" % https_repo_publish_dir) shutil.rmtree(https_repo_publish_dir) # build iso and publish via HTTP http_publish_dir = iso_util.get_http_publish_iso_dir(config) http_repo_publish_dir = os.path.join(http_publish_dir, repo_group.id).rstrip('/') if config.get("http"): # Publish for HTTP self.set_progress("publish_http", {"state" : "IN_PROGRESS"}, progress_callback) try: _LOG.info("HTTP Publishing repo <%s> to <%s>" % (repo_group.id, http_repo_publish_dir)) # generate iso images and write them to the publish directory isogen = GenerateIsos(self.group_working_dir, http_repo_publish_dir, prefix=prefix, progress=self.init_progress(), is_cancelled=self.canceled) isogen.run(progress_callback=progress_callback) self.group_summary["http_publish_dir"] = http_repo_publish_dir self.set_progress("publish_http", {"state" : "FINISHED"}, progress_callback) except Exception,e: _LOG.debug(" publish operaation failed due to exception: %s" % e) self.set_progress("publish_http", {"state" : "FAILED"}, progress_callback) else: self.set_progress("publish_http", {"state" : "SKIPPED"}, progress_callback) if os.path.lexists(http_repo_publish_dir): _LOG.debug("Removing link for %s since http is not set" % http_repo_publish_dir) shutil.rmtree(http_repo_publish_dir)
def publish_repo(self, repo, publish_conduit, config): publish_start_time = time.time() _LOG.info("Start publish time %s" % publish_start_time) summary = {} details = {} progress_status = { "rpms": {"state": "NOT_STARTED"}, "errata": {"state": "NOT_STARTED"}, "distribution": {"state": "NOT_STARTED"}, "packagegroups": {"state": "NOT_STARTED"}, "isos": {"state": "NOT_STARTED"}, "publish_http": {"state": "NOT_STARTED"}, } def progress_callback(type_id, status): progress_status[type_id] = status publish_conduit.set_progress(progress_status) self.repo_working_dir = repo_working_dir = repo.working_dir if self.canceled: return publish_conduit.build_failure_report(summary, details) date_filter = self.create_date_range_filter(config) if date_filter: # export errata by date and associated rpm units progress_status["errata"]["state"] = "STARTED" criteria = UnitAssociationCriteria(type_ids=[TYPE_ID_ERRATA], unit_filters=date_filter) errata_units = publish_conduit.get_units(criteria) criteria = UnitAssociationCriteria(type_ids=[TYPE_ID_RPM, TYPE_ID_SRPM, TYPE_ID_DRPM]) rpm_units = publish_conduit.get_units(criteria) rpm_units = self._get_errata_rpms(errata_units, rpm_units) rpm_status, rpm_errors = self._export_rpms(rpm_units, repo_working_dir, progress_callback=progress_callback) progress_status["rpms"]["state"] = "FINISHED" if self.canceled: return publish_conduit.build_failure_report(summary, details) # generate metadata metadata_status, metadata_errors = metadata.generate_metadata( repo, publish_conduit, config, progress_callback) _LOG.info("metadata generation complete at target location %s" % repo_working_dir) errata_status, errata_errors = self._export_errata(errata_units, repo_working_dir, progress_callback=progress_callback) progress_status["errata"]["state"] = "FINISHED" summary["num_package_units_attempted"] = len(rpm_units) summary["num_package_units_exported"] = len(rpm_units) - len(rpm_errors) summary["num_package_units_errors"] = len(rpm_errors) details["errors"] = rpm_errors + errata_errors else: # export everything progress_status["rpms"]["state"] = "STARTED" criteria = UnitAssociationCriteria(type_ids=[TYPE_ID_RPM, TYPE_ID_SRPM, TYPE_ID_DRPM]) rpm_units = publish_conduit.get_units(criteria) rpm_status, rpm_errors = self._export_rpms(rpm_units, repo_working_dir, progress_callback=progress_callback) progress_status["rpms"]["state"] = "FINISHED" # package groups progress_status["packagegroups"]["state"] = "STARTED" criteria = UnitAssociationCriteria(type_ids=[TYPE_ID_PKG_GROUP, TYPE_ID_PKG_CATEGORY]) existing_units = publish_conduit.get_units(criteria) existing_groups = filter(lambda u : u.type_id in [TYPE_ID_PKG_GROUP], existing_units) existing_cats = filter(lambda u : u.type_id in [TYPE_ID_PKG_CATEGORY], existing_units) groups_xml_path = comps_util.write_comps_xml(repo, existing_groups, existing_cats) if self.canceled: return publish_conduit.build_failure_report(summary, details) # generate metadata metadata_status, metadata_errors = metadata.generate_metadata( repo, publish_conduit, config, progress_callback, groups_xml_path) _LOG.info("metadata generation complete at target location %s" % repo_working_dir) progress_status["packagegroups"]["state"] = "FINISHED" progress_status["errata"]["state"] = "STARTED" criteria = UnitAssociationCriteria(type_ids=[TYPE_ID_ERRATA]) errata_units = publish_conduit.get_units(criteria) rpm_units = self._get_errata_rpms(errata_units, rpm_units) self._export_rpms(rpm_units, repo_working_dir, progress_callback=progress_callback) errata_status, errata_errors = self._export_errata(errata_units, repo_working_dir, progress_callback=progress_callback) progress_status["errata"]["state"] = "FINISHED" # distro units progress_status["distribution"]["state"] = "STARTED" criteria = UnitAssociationCriteria(type_ids=[TYPE_ID_DISTRO]) distro_units = publish_conduit.get_units(criteria) distro_status, distro_errors = self._export_distributions(distro_units, repo_working_dir, progress_callback=progress_callback) progress_status["distribution"]["state"] = "FINISHED" summary["num_distribution_units_attempted"] = len(distro_units) summary["num_distribution_units_exported"] = len(distro_units) - len(distro_errors) summary["num_distribution_units_errors"] = len(distro_errors) summary["num_package_groups_exported"] = len(existing_groups) summary["num_package_categories_exported"] = len(existing_cats) summary["num_package_units_attempted"] = len(rpm_units) summary["num_package_units_exported"] = len(rpm_units) - len(rpm_errors) summary["num_package_units_errors"] = len(rpm_errors) details["errors"] = rpm_errors + distro_errors + errata_errors # build iso and publish via HTTPS https_publish_dir = self.get_https_publish_iso_dir(config) https_repo_publish_dir = os.path.join(https_publish_dir, repo.id).rstrip('/') prefix = config.get('iso_prefix') or repo.id if config.get("https"): # Publish for HTTPS self.set_progress("publish_https", {"state" : "IN_PROGRESS"}, progress_callback) try: _LOG.info("HTTPS Publishing repo <%s> to <%s>" % (repo.id, https_repo_publish_dir)) isogen = GenerateIsos(repo_working_dir, https_repo_publish_dir, prefix=prefix, progress=progress_status) progress_status = isogen.run() summary["https_publish_dir"] = https_repo_publish_dir self.set_progress("publish_https", {"state" : "FINISHED"}, progress_callback) progress_status["isos"]["state"] = "FINISHED" except: self.set_progress("publish_https", {"state" : "FAILED"}, progress_callback) else: self.set_progress("publish_https", {"state" : "SKIPPED"}, progress_callback) if os.path.lexists(https_repo_publish_dir): _LOG.debug("Removing link for %s since https is not set" % https_repo_publish_dir) shutil.rmtree(https_repo_publish_dir) # Handle publish link for HTTP # build iso and publish via HTTP http_publish_dir = self.get_http_publish_iso_dir(config) http_repo_publish_dir = os.path.join(http_publish_dir, repo.id).rstrip('/') if config.get("http"): # Publish for HTTP self.set_progress("publish_http", {"state" : "IN_PROGRESS"}, progress_callback) try: _LOG.info("HTTP Publishing repo <%s> to <%s>" % (repo.id, http_repo_publish_dir)) isogen = GenerateIsos(repo_working_dir, http_repo_publish_dir, prefix=prefix, progress=progress_status) progress_status = isogen.run() summary["http_publish_dir"] = http_repo_publish_dir self.set_progress("publish_http", {"state" : "FINISHED"}, progress_callback) progress_status["isos"]["state"] = "FINISHED" except: self.set_progress("publish_http", {"state" : "FAILED"}, progress_callback) else: self.set_progress("publish_http", {"state" : "SKIPPED"}, progress_callback) if os.path.lexists(http_repo_publish_dir): _LOG.debug("Removing link for %s since http is not set" % http_repo_publish_dir) shutil.rmtree(http_repo_publish_dir) details["errors"] += metadata_errors # metadata generate skipped vs run _LOG.info("Publish complete: summary = <%s>, details = <%s>" % (summary, details)) # remove exported content from working dirctory self.cleanup() if details["errors"]: return publish_conduit.build_failure_report(summary, details) return publish_conduit.build_success_report(summary, details)