def test_generate_metadata(self): mock_repo = mock.Mock(spec=Repository) mock_repo.id = "test_repo" mock_repo.scratchpad = {"checksum_type": "sha"} mock_repo.working_dir = os.path.join(self.data_dir, "test_repo_metadata") # Confirm required and optional are successful optional_kwargs = {"generate_metadata": 1} config = distributor_mocks.get_basic_config(**optional_kwargs) mock_publish_conduit = distributor_mocks.get_publish_conduit() status, errors = metadata.generate_metadata(mock_repo, mock_publish_conduit, config) self.assertEquals(status, True) optional_kwargs = {"generate_metadata": 0} config = distributor_mocks.get_basic_config(**optional_kwargs) mock_publish_conduit = distributor_mocks.get_publish_conduit() status, errors = metadata.generate_metadata(mock_repo, mock_publish_conduit, config) self.assertEquals(status, False)
def run(self): self.status, self.errors = metadata.generate_metadata(self.mock_repo.working_dir, self.mock_publish_conduit, self.config, set_progress) self.finished = True
def publish_repo(self, repo, publish_conduit, config): summary = {} details = {} progress_status = { "packages": {"state": "NOT_STARTED"}, "distribution": {"state": "NOT_STARTED"}, "metadata": {"state": "NOT_STARTED"}, "packagegroups": {"state": "NOT_STARTED"}, "publish_http": {"state": "NOT_STARTED"}, "publish_https": {"state": "NOT_STARTED"}, } def progress_callback(type_id, status): progress_status[type_id] = status publish_conduit.set_progress(progress_status) self.repo_working_dir = repo.working_dir if self.canceled: return publish_conduit.build_cancel_report(summary, details) skip_list = config.get('skip') or [] # Determine Content in this repo pkg_units = [] pkg_errors = [] if 'rpm' not in skip_list: for type_id in [TYPE_ID_RPM, TYPE_ID_SRPM]: criteria = UnitAssociationCriteria(type_ids=type_id, unit_fields=['id', 'name', 'version', 'release', 'arch', 'epoch', '_storage_path', "checksum", "checksumtype" ]) pkg_units += publish_conduit.get_units(criteria=criteria) drpm_units = [] if 'drpm' not in skip_list: criteria = UnitAssociationCriteria(type_ids=TYPE_ID_DRPM) drpm_units = publish_conduit.get_units(criteria=criteria) pkg_units += drpm_units # Create symlinks under repo.working_dir pkg_status, pkg_errors = self.handle_symlinks(pkg_units, repo.working_dir, progress_callback) if not pkg_status: _LOG.error("Unable to publish %s items" % (len(pkg_errors))) distro_errors = [] distro_units = [] if 'distribution' not in skip_list: criteria = UnitAssociationCriteria(type_ids=TYPE_ID_DISTRO) distro_units = publish_conduit.get_units(criteria=criteria) # symlink distribution files if any under repo.working_dir distro_status, distro_errors = self.symlink_distribution_unit_files(distro_units, repo.working_dir, publish_conduit, progress_callback) if not distro_status: _LOG.error("Unable to publish distribution tree %s items" % (len(distro_errors))) updateinfo_xml_path = None if 'erratum' not in skip_list: criteria = UnitAssociationCriteria(type_ids=TYPE_ID_ERRATA) errata_units = publish_conduit.get_units(criteria=criteria) updateinfo_xml_path = updateinfo.updateinfo(errata_units, repo.working_dir) if self.canceled: return publish_conduit.build_cancel_report(summary, details) groups_xml_path = None existing_cats = [] existing_groups = [] if 'packagegroup' not in skip_list: criteria = UnitAssociationCriteria(type_ids=[TYPE_ID_PKG_GROUP, TYPE_ID_PKG_CATEGORY]) existing_units = publish_conduit.get_units(criteria) existing_groups = filter(lambda u : u.type_id in [TYPE_ID_PKG_GROUP], existing_units) existing_cats = filter(lambda u : u.type_id in [TYPE_ID_PKG_CATEGORY], existing_units) groups_xml_path = comps_util.write_comps_xml(repo.working_dir, existing_groups, existing_cats) metadata_start_time = time.time() # update/generate metadata for the published repo self.use_createrepo = config.get('use_createrepo') if self.use_createrepo: metadata_status, metadata_errors = metadata.generate_metadata( repo.working_dir, publish_conduit, config, progress_callback, groups_xml_path) else: metadata_status, metadata_errors = metadata.generate_yum_metadata(repo.id, repo.working_dir, publish_conduit, config, progress_callback, is_cancelled=self.canceled, group_xml_path=groups_xml_path, updateinfo_xml_path=updateinfo_xml_path, repo_scratchpad=publish_conduit.get_repo_scratchpad()) metadata_end_time = time.time() relpath = self.get_repo_relative_path(repo, config) if relpath.startswith("/"): relpath = relpath[1:] # Build the https and http publishing paths https_publish_dir = self.get_https_publish_dir(config) https_repo_publish_dir = os.path.join(https_publish_dir, relpath).rstrip('/') http_publish_dir = self.get_http_publish_dir(config) http_repo_publish_dir = os.path.join(http_publish_dir, relpath).rstrip('/') # Clean up the old publish directories, if they exist. scratchpad = publish_conduit.get_repo_scratchpad() if OLD_REL_PATH_KEYWORD in scratchpad: old_relative_path = scratchpad[OLD_REL_PATH_KEYWORD] old_https_repo_publish_dir = os.path.join(https_publish_dir, old_relative_path) if os.path.exists(old_https_repo_publish_dir): util.remove_repo_publish_dir(https_publish_dir, old_https_repo_publish_dir) old_http_repo_publish_dir = os.path.join(http_publish_dir, old_relative_path) if os.path.exists(old_http_repo_publish_dir): util.remove_repo_publish_dir(http_publish_dir, old_http_repo_publish_dir) # Now write the current publish relative path to the scratch pad. This way, if the relative path # changes before the next publish, we can clean up the old path. scratchpad[OLD_REL_PATH_KEYWORD] = relpath publish_conduit.set_repo_scratchpad(scratchpad) # Handle publish link for HTTPS if config.get("https"): # Publish for HTTPS self.set_progress("publish_https", {"state" : "IN_PROGRESS"}, progress_callback) try: _LOG.info("HTTPS Publishing repo <%s> to <%s>" % (repo.id, https_repo_publish_dir)) util.create_symlink(repo.working_dir, https_repo_publish_dir) util.generate_listing_files(https_publish_dir, https_repo_publish_dir) summary["https_publish_dir"] = https_repo_publish_dir self.set_progress("publish_https", {"state" : "FINISHED"}, progress_callback) except: self.set_progress("publish_https", {"state" : "FAILED"}, progress_callback) else: self.set_progress("publish_https", {"state" : "SKIPPED"}, progress_callback) if os.path.lexists(https_repo_publish_dir): _LOG.debug("Removing link for %s since https is not set" % https_repo_publish_dir) util.remove_repo_publish_dir(https_publish_dir, https_repo_publish_dir) # Handle publish link for HTTP if config.get("http"): # Publish for HTTP self.set_progress("publish_http", {"state" : "IN_PROGRESS"}, progress_callback) try: _LOG.info("HTTP Publishing repo <%s> to <%s>" % (repo.id, http_repo_publish_dir)) util.create_symlink(repo.working_dir, http_repo_publish_dir) util.generate_listing_files(http_publish_dir, http_repo_publish_dir) summary["http_publish_dir"] = http_repo_publish_dir self.set_progress("publish_http", {"state" : "FINISHED"}, progress_callback) except: self.set_progress("publish_http", {"state" : "FAILED"}, progress_callback) else: self.set_progress("publish_http", {"state" : "SKIPPED"}, progress_callback) if os.path.lexists(http_repo_publish_dir): _LOG.debug("Removing link for %s since http is not set" % http_repo_publish_dir) util.remove_repo_publish_dir(http_publish_dir, http_repo_publish_dir) summary["num_package_units_attempted"] = len(pkg_units) summary["num_package_units_published"] = len(pkg_units) - len(pkg_errors) summary["num_package_units_errors"] = len(pkg_errors) summary["num_distribution_units_attempted"] = len(distro_units) summary["num_distribution_units_published"] = len(distro_units) - len(distro_errors) summary["num_distribution_units_errors"] = len(distro_errors) summary["num_package_groups_published"] = len(existing_groups) summary["num_package_categories_published"] = len(existing_cats) summary["relative_path"] = relpath if metadata_status is False and not len(metadata_errors): summary["skip_metadata_update"] = True else: summary["skip_metadata_update"] = False details["errors"] = pkg_errors + distro_errors # metadata_errors details['time_metadata_sec'] = metadata_end_time - metadata_start_time # metadata generate skipped vs run _LOG.info("Publish complete: summary = <%s>, details = <%s>" % (summary, details)) if details["errors"]: return publish_conduit.build_failure_report(summary, details) return publish_conduit.build_success_report(summary, details)
def publish_repo(self, repo, publish_conduit, config): summary = {} details = {} progress_status = { "packages": {"state": "NOT_STARTED"}, "distribution": {"state": "NOT_STARTED"}, "metadata": {"state": "NOT_STARTED"}, "packagegroups": {"state": "NOT_STARTED"}, "publish_http": {"state": "NOT_STARTED"}, "publish_https": {"state": "NOT_STARTED"}, } def progress_callback(type_id, status): progress_status[type_id] = status publish_conduit.set_progress(progress_status) self.repo_working_dir = repo.working_dir if self.canceled: return publish_conduit.build_failure_report(summary, details) skip_list = config.get('skip') or [] # Determine Content in this repo unfiltered_units = publish_conduit.get_units() # filter compatible units rpm_units = filter(lambda u : u.type_id in [TYPE_ID_RPM, TYPE_ID_SRPM], unfiltered_units) drpm_units = filter(lambda u : u.type_id == TYPE_ID_DRPM, unfiltered_units) rpm_errors = [] if 'rpm' not in skip_list: _LOG.info("Publish on %s invoked. %s existing units, %s of which are supported to be published." \ % (repo.id, len(unfiltered_units), len(rpm_units))) # Create symlinks under repo.working_dir rpm_status, rpm_errors = self.handle_symlinks(rpm_units, repo.working_dir, progress_callback) if not rpm_status: _LOG.error("Unable to publish %s items" % (len(rpm_errors))) drpm_errors = [] if 'drpm' not in skip_list: _LOG.info("Publish on %s invoked. %s existing units, %s of which are supported to be published." \ % (repo.id, len(unfiltered_units), len(drpm_units))) # Create symlinks under repo.working_dir drpm_status, drpm_errors = self.handle_symlinks(drpm_units, repo.working_dir, progress_callback) if not drpm_status: _LOG.error("Unable to publish %s items" % (len(drpm_errors))) pkg_errors = rpm_errors + drpm_errors pkg_units = rpm_units + drpm_units distro_errors = [] distro_units = filter(lambda u: u.type_id == TYPE_ID_DISTRO, unfiltered_units) if 'distribution' not in skip_list: # symlink distribution files if any under repo.working_dir distro_status, distro_errors = self.symlink_distribution_unit_files(distro_units, repo.working_dir, progress_callback) if not distro_status: _LOG.error("Unable to publish distribution tree %s items" % (len(distro_errors))) # update/generate metadata for the published repo repo_scratchpad = publish_conduit.get_repo_scratchpad() src_working_dir = '' if repo_scratchpad.has_key("importer_working_dir"): src_working_dir = repo_scratchpad['importer_working_dir'] if self.canceled: return publish_conduit.build_failure_report(summary, details) groups_xml_path = None existing_cats = [] existing_groups = [] if 'packagegroup' not in skip_list: criteria = UnitAssociationCriteria(type_ids=[TYPE_ID_PKG_GROUP, TYPE_ID_PKG_CATEGORY]) existing_units = publish_conduit.get_units(criteria) existing_groups = filter(lambda u : u.type_id in [TYPE_ID_PKG_GROUP], existing_units) existing_cats = filter(lambda u : u.type_id in [TYPE_ID_PKG_CATEGORY], existing_units) groups_xml_path = comps_util.write_comps_xml(repo, existing_groups, existing_cats) metadata_start_time = time.time() self.copy_importer_repodata(src_working_dir, repo.working_dir) metadata_status, metadata_errors = metadata.generate_metadata( repo, publish_conduit, config, progress_callback, groups_xml_path) metadata_end_time = time.time() relpath = self.get_repo_relative_path(repo, config) if relpath.startswith("/"): relpath = relpath[1:] # # Handle publish link for HTTPS # https_publish_dir = self.get_https_publish_dir(config) https_repo_publish_dir = os.path.join(https_publish_dir, relpath).rstrip('/') if config.get("https"): # Publish for HTTPS self.set_progress("publish_https", {"state" : "IN_PROGRESS"}, progress_callback) try: _LOG.info("HTTPS Publishing repo <%s> to <%s>" % (repo.id, https_repo_publish_dir)) util.create_symlink(repo.working_dir, https_repo_publish_dir) summary["https_publish_dir"] = https_repo_publish_dir self.set_progress("publish_https", {"state" : "FINISHED"}, progress_callback) except: self.set_progress("publish_https", {"state" : "FAILED"}, progress_callback) else: self.set_progress("publish_https", {"state" : "SKIPPED"}, progress_callback) if os.path.lexists(https_repo_publish_dir): _LOG.debug("Removing link for %s since https is not set" % https_repo_publish_dir) util.remove_symlink(https_publish_dir, https_repo_publish_dir) # # Handle publish link for HTTP # http_publish_dir = self.get_http_publish_dir(config) http_repo_publish_dir = os.path.join(http_publish_dir, relpath).rstrip('/') if config.get("http"): # Publish for HTTP self.set_progress("publish_http", {"state" : "IN_PROGRESS"}, progress_callback) try: _LOG.info("HTTP Publishing repo <%s> to <%s>" % (repo.id, http_repo_publish_dir)) util.create_symlink(repo.working_dir, http_repo_publish_dir) summary["http_publish_dir"] = http_repo_publish_dir self.set_progress("publish_http", {"state" : "FINISHED"}, progress_callback) except: self.set_progress("publish_http", {"state" : "FAILED"}, progress_callback) else: self.set_progress("publish_http", {"state" : "SKIPPED"}, progress_callback) if os.path.lexists(http_repo_publish_dir): _LOG.debug("Removing link for %s since http is not set" % http_repo_publish_dir) util.remove_symlink(http_publish_dir, http_repo_publish_dir) summary["num_package_units_attempted"] = len(pkg_units) summary["num_package_units_published"] = len(pkg_units) - len(pkg_errors) summary["num_package_units_errors"] = len(pkg_errors) summary["num_distribution_units_attempted"] = len(distro_units) summary["num_distribution_units_published"] = len(distro_units) - len(distro_errors) summary["num_distribution_units_errors"] = len(distro_errors) summary["num_package_groups_published"] = len(existing_groups) summary["num_package_categories_published"] = len(existing_cats) summary["relative_path"] = relpath if metadata_status is False and not len(metadata_errors): summary["skip_metadata_update"] = True else: summary["skip_metadata_update"] = False details["errors"] = pkg_errors + distro_errors + metadata_errors details['time_metadata_sec'] = metadata_end_time - metadata_start_time # metadata generate skipped vs run _LOG.info("Publish complete: summary = <%s>, details = <%s>" % (summary, details)) if details["errors"]: return publish_conduit.build_failure_report(summary, details) return publish_conduit.build_success_report(summary, details)
def publish_repo(self, repo, publish_conduit, config): publish_start_time = time.time() _LOG.info("Start publish time %s" % publish_start_time) summary = {} details = {} progress_status = { "rpms": {"state": "NOT_STARTED"}, "errata": {"state": "NOT_STARTED"}, "distribution": {"state": "NOT_STARTED"}, "packagegroups": {"state": "NOT_STARTED"}, "isos": {"state": "NOT_STARTED"}, "publish_http": {"state": "NOT_STARTED"}, } def progress_callback(type_id, status): progress_status[type_id] = status publish_conduit.set_progress(progress_status) self.repo_working_dir = repo_working_dir = repo.working_dir if self.canceled: return publish_conduit.build_failure_report(summary, details) skip_types = config.get("skip") or [] date_filter = self.create_date_range_filter(config) if date_filter: # export errata by date and associated rpm units if "errata" not in skip_types: progress_status["errata"]["state"] = "STARTED" criteria = UnitAssociationCriteria(type_ids=[TYPE_ID_ERRATA], unit_filters=date_filter) errata_units = publish_conduit.get_units(criteria) criteria = UnitAssociationCriteria(type_ids=[TYPE_ID_RPM, TYPE_ID_SRPM, TYPE_ID_DRPM]) rpm_units = publish_conduit.get_units(criteria) rpm_units = self._get_errata_rpms(errata_units, rpm_units) rpm_status, rpm_errors = self._export_rpms(rpm_units, repo_working_dir, progress_callback=progress_callback) progress_status["rpms"]["state"] = "FINISHED" if self.canceled: return publish_conduit.build_failure_report(summary, details) # generate metadata metadata_status, metadata_errors = metadata.generate_metadata( repo, publish_conduit, config, progress_callback) _LOG.info("metadata generation complete at target location %s" % repo_working_dir) errata_status, errata_errors = self._export_errata(errata_units, repo_working_dir, progress_callback=progress_callback) progress_status["errata"]["state"] = "FINISHED" summary["num_package_units_attempted"] = len(rpm_units) summary["num_package_units_exported"] = len(rpm_units) - len(rpm_errors) summary["num_package_units_errors"] = len(rpm_errors) details["errors"] = rpm_errors + errata_errors + metadata_errors else: progress_status["errata"]["state"] = "SKIPPED" _LOG.info("erratum unit type in skip list [%s]; skipping export" % skip_types) else: # export everything rpm_units = [] rpm_errors = [] if "rpm" not in skip_types: progress_status["rpms"]["state"] = "STARTED" criteria = UnitAssociationCriteria(type_ids=[TYPE_ID_RPM, TYPE_ID_SRPM, TYPE_ID_DRPM]) rpm_units = publish_conduit.get_units(criteria) rpm_status, rpm_errors = self._export_rpms(rpm_units, repo_working_dir, progress_callback=progress_callback) progress_status["rpms"]["state"] = "FINISHED" summary["num_package_units_attempted"] = len(rpm_units) summary["num_package_units_exported"] = len(rpm_units) - len(rpm_errors) summary["num_package_units_errors"] = len(rpm_errors) else: progress_status["rpms"]["state"] = "SKIPPED" _LOG.info("rpm unit type in skip list [%s]; skipping export" % skip_types) # package groups groups_xml_path = None if "packagegroup" not in skip_types: progress_status["packagegroups"]["state"] = "STARTED" criteria = UnitAssociationCriteria(type_ids=[TYPE_ID_PKG_GROUP, TYPE_ID_PKG_CATEGORY]) existing_units = publish_conduit.get_units(criteria) existing_groups = filter(lambda u : u.type_id in [TYPE_ID_PKG_GROUP], existing_units) existing_cats = filter(lambda u : u.type_id in [TYPE_ID_PKG_CATEGORY], existing_units) groups_xml_path = comps_util.write_comps_xml(repo, existing_groups, existing_cats) summary["num_package_groups_exported"] = len(existing_groups) summary["num_package_categories_exported"] = len(existing_cats) progress_status["packagegroups"]["state"] = "FINISHED" else: progress_status["packagegroups"]["state"] = "SKIPPED" _LOG.info("packagegroup unit type in skip list [%s]; skipping export" % skip_types) if self.canceled: return publish_conduit.build_failure_report(summary, details) # generate metadata metadata_status, metadata_errors = metadata.generate_metadata( repo, publish_conduit, config, progress_callback, groups_xml_path) _LOG.info("metadata generation complete at target location %s" % repo_working_dir) errata_errors = [] if "errata" not in skip_types: progress_status["errata"]["state"] = "STARTED" criteria = UnitAssociationCriteria(type_ids=[TYPE_ID_ERRATA]) errata_units = publish_conduit.get_units(criteria) rpm_units = self._get_errata_rpms(errata_units, rpm_units) self._export_rpms(rpm_units, repo_working_dir, progress_callback=progress_callback) errata_status, errata_errors = self._export_errata(errata_units, repo_working_dir, progress_callback=progress_callback) summary["num_errata_units_exported"] = len(errata_units) progress_status["errata"]["state"] = "FINISHED" else: progress_status["errata"]["state"] = "SKIPPED" _LOG.info("erratum unit type in skip list [%s]; skipping export" % skip_types) distro_errors = [] if "distribution" not in skip_types: # distro units progress_status["distribution"]["state"] = "STARTED" criteria = UnitAssociationCriteria(type_ids=[TYPE_ID_DISTRO]) distro_units = publish_conduit.get_units(criteria) distro_status, distro_errors = self._export_distributions(distro_units, repo_working_dir, progress_callback=progress_callback) progress_status["distribution"]["state"] = "FINISHED" summary["num_distribution_units_attempted"] = len(distro_units) summary["num_distribution_units_exported"] = len(distro_units) - len(distro_errors) summary["num_distribution_units_errors"] = len(distro_errors) else: progress_status["distribution"]["state"] = "SKIPPED" _LOG.info("distribution unit type in skip list [%s]; skipping export" % skip_types) details["errors"] = rpm_errors + distro_errors + errata_errors + metadata_errors # build iso and publish via HTTPS https_publish_dir = self.get_https_publish_iso_dir(config) https_repo_publish_dir = os.path.join(https_publish_dir, repo.id).rstrip('/') prefix = config.get('iso_prefix') or repo.id if config.get("https"): # Publish for HTTPS self.set_progress("publish_https", {"state" : "IN_PROGRESS"}, progress_callback) try: _LOG.info("HTTPS Publishing repo <%s> to <%s>" % (repo.id, https_repo_publish_dir)) iso_status, iso_errors = self.generate_isos(repo_working_dir, https_repo_publish_dir, prefix=prefix, progress_callback=progress_callback) summary["https_publish_dir"] = https_repo_publish_dir self.set_progress("publish_https", {"state" : "FINISHED"}, progress_callback) progress_status["isos"]["state"] = "FINISHED" except: self.set_progress("publish_https", {"state" : "FAILED"}, progress_callback) else: self.set_progress("publish_https", {"state" : "SKIPPED"}, progress_callback) if os.path.lexists(https_repo_publish_dir): _LOG.debug("Removing link for %s since https is not set" % https_repo_publish_dir) shutil.rmtree(https_repo_publish_dir) # Handle publish link for HTTP # build iso and publish via HTTP http_publish_dir = self.get_http_publish_iso_dir(config) http_repo_publish_dir = os.path.join(http_publish_dir, repo.id).rstrip('/') if config.get("http"): # Publish for HTTP self.set_progress("publish_http", {"state" : "IN_PROGRESS"}, progress_callback) try: _LOG.info("HTTP Publishing repo <%s> to <%s>" % (repo.id, http_repo_publish_dir)) iso_status, iso_errors = self.generate_isos(repo_working_dir, http_repo_publish_dir, prefix=prefix, progress_callback=progress_callback) print iso_status, iso_errors summary["http_publish_dir"] = http_repo_publish_dir self.set_progress("publish_http", {"state" : "FINISHED"}, progress_callback) progress_status["isos"]["state"] = "FINISHED" except: self.set_progress("publish_http", {"state" : "FAILED"}, progress_callback) else: self.set_progress("publish_http", {"state" : "SKIPPED"}, progress_callback) if os.path.lexists(http_repo_publish_dir): _LOG.debug("Removing link for %s since http is not set" % http_repo_publish_dir) shutil.rmtree(http_repo_publish_dir) # metadata generate skipped vs run _LOG.info("Publish complete: summary = <%s>, details = <%s>" % (summary, details)) # remove exported content from working dirctory self.cleanup() if details["errors"]: return publish_conduit.build_failure_report(summary, details) return publish_conduit.build_success_report(summary, details)
def publish_group(self, repo_group, publish_conduit, config): """ see parent class for doc string """ self.group_working_dir = group_working_dir = repo_group.working_dir skip_types = config.get("skip") or [] self.init_group_progress() self.group_progress_status["group-id"] = repo_group.id # progress callback for group status def group_progress_callback(type_id, status): self.group_progress_status[type_id] = status publish_conduit.set_progress(self.group_progress_status) # loop through each repo in the group and perform exports for repoid in repo_group.repo_ids: _LOG.info("Exporting repo %s " % repoid) summary = {} details = {} progress_status = { "rpms": {"state": "NOT_STARTED"}, "errata": {"state": "NOT_STARTED"}, "distribution": {"state": "NOT_STARTED"}, "packagegroups": {"state": "NOT_STARTED"},} def progress_callback(type_id, status): progress_status[type_id] = status publish_conduit.set_progress(progress_status) repo_working_dir = "%s/%s" % (group_working_dir, repoid) repo_exporter = RepoExporter(repo_working_dir, skip=skip_types) # check if any datefilter is set on the distributor date_filter = repo_exporter.create_date_range_filter(config) _LOG.info("repo working dir %s" % repo_working_dir) if date_filter: # If a date range is specified, we only export the errata within that range # and associated rpm units. This might change once we have dates associated # to other units. criteria = UnitAssociationCriteria(type_ids=[TYPE_ID_ERRATA], unit_filters=date_filter) errata_units = publish_conduit.get_units(repoid, criteria=criteria) # we only include binary and source; drpms are not associated to errata criteria = UnitAssociationCriteria(type_ids=[TYPE_ID_RPM, TYPE_ID_SRPM]) rpm_units = publish_conduit.get_units(repoid, criteria=criteria) rpm_units = repo_exporter.get_errata_rpms(errata_units, rpm_units) rpm_status, rpm_errors = repo_exporter.export_rpms(rpm_units, progress_callback=progress_callback) if self.canceled: return publish_conduit.build_failure_report(summary, details) # generate metadata metadata_status, metadata_errors = metadata.generate_metadata( repo_working_dir, publish_conduit, config, progress_callback) _LOG.info("metadata generation complete at target location %s" % repo_working_dir) # export errata and generate updateinfo xml errata_status, errata_errors = repo_exporter.export_errata(errata_units, progress_callback=progress_callback) summary["num_package_units_attempted"] = len(rpm_units) summary["num_package_units_exported"] = len(rpm_units) - len(rpm_errors) summary["num_package_units_errors"] = len(rpm_errors) details["errors"] = rpm_errors + errata_errors + metadata_errors else: # export rpm units(this includes binary, source and delta) criteria = UnitAssociationCriteria(type_ids=[TYPE_ID_RPM, TYPE_ID_SRPM, TYPE_ID_DRPM]) rpm_units = publish_conduit.get_units(repoid, criteria) rpm_status, rpm_errors = repo_exporter.export_rpms(rpm_units, progress_callback=progress_callback) summary["num_package_units_attempted"] = len(rpm_units) summary["num_package_units_exported"] = len(rpm_units) - len(rpm_errors) summary["num_package_units_errors"] = len(rpm_errors) # export package groups information and generate comps.xml groups_xml_path = None if "packagegroup" not in skip_types: progress_status["packagegroups"]["state"] = "STARTED" criteria = UnitAssociationCriteria(type_ids=[TYPE_ID_PKG_GROUP, TYPE_ID_PKG_CATEGORY]) existing_units = publish_conduit.get_units(repoid, criteria) existing_groups = filter(lambda u : u.type_id in [TYPE_ID_PKG_GROUP], existing_units) existing_cats = filter(lambda u : u.type_id in [TYPE_ID_PKG_CATEGORY], existing_units) groups_xml_path = comps_util.write_comps_xml(repo_working_dir, existing_groups, existing_cats) summary["num_package_groups_exported"] = len(existing_groups) summary["num_package_categories_exported"] = len(existing_cats) progress_status["packagegroups"]["state"] = "FINISHED" else: progress_status["packagegroups"]["state"] = "SKIPPED" _LOG.info("packagegroup unit type in skip list [%s]; skipping export" % skip_types) if self.canceled: return publish_conduit.build_failure_report(summary, details) # generate metadata metadata_status, metadata_errors = metadata.generate_metadata( repo_working_dir, publish_conduit, config, progress_callback, groups_xml_path) _LOG.info("metadata generation complete at target location %s" % repo_working_dir) # export errata units and associated rpms criteria = UnitAssociationCriteria(type_ids=[TYPE_ID_ERRATA]) errata_units = publish_conduit.get_units(repoid, criteria) errata_status, errata_errors = repo_exporter.export_errata(errata_units, progress_callback=progress_callback) summary["num_errata_units_exported"] = len(errata_units) # export distributions criteria = UnitAssociationCriteria(type_ids=[TYPE_ID_DISTRO]) distro_units = publish_conduit.get_units(repoid, criteria) distro_status, distro_errors = repo_exporter.export_distributions(distro_units, progress_callback=progress_callback) summary["num_distribution_units_attempted"] = len(distro_units) summary["num_distribution_units_exported"] = len(distro_units) - len(distro_errors) summary["num_distribution_units_errors"] = len(distro_errors) self.group_progress_status["repositories"][repoid] = progress_status self.set_progress("repositories", self.group_progress_status["repositories"], group_progress_callback) details["errors"] = rpm_errors + distro_errors + errata_errors + metadata_errors self.group_summary[repoid] = summary self.group_details[repoid] = details # generate and publish isos self._publish_isos(repo_group, config, progress_callback=group_progress_callback) _LOG.info("Publish complete: summary = <%s>, details = <%s>" % (self.group_summary, self.group_details)) # remove exported content from working dirctory iso_util.cleanup_working_dir(self.group_working_dir) # check for any errors if not len([self.group_details[repoid]["errors"] for repoid in self.group_details.keys()]): return publish_conduit.build_failure_report(self.group_summary, self.group_details) return publish_conduit.build_success_report(self.group_summary, self.group_details)
def publish_repo(self, repo, publish_conduit, config): publish_start_time = time.time() _LOG.info("Start publish time %s" % publish_start_time) progress_status = { "rpms": {"state": "NOT_STARTED"}, "errata": {"state": "NOT_STARTED"}, "distribution": {"state": "NOT_STARTED"}, "packagegroups": {"state": "NOT_STARTED"}, "isos": {"state": "NOT_STARTED"}, "publish_http": {"state": "NOT_STARTED"}, } def progress_callback(type_id, status): progress_status[type_id] = status publish_conduit.set_progress(progress_status) self.repo_working_dir = repo_working_dir = repo.working_dir if self.canceled: return publish_conduit.build_failure_report(self.summary, self.details) skip_types = config.get("skip") or [] repo_exporter = RepoExporter(repo_working_dir, skip=skip_types) date_filter = repo_exporter.create_date_range_filter(config) if date_filter: # export errata by date and associated rpm units criteria = UnitAssociationCriteria(type_ids=[TYPE_ID_ERRATA], unit_filters=date_filter) errata_units = publish_conduit.get_units(criteria=criteria) criteria = UnitAssociationCriteria(type_ids=[TYPE_ID_RPM, TYPE_ID_SRPM, TYPE_ID_DRPM]) rpm_units = publish_conduit.get_units(criteria=criteria) rpm_units = repo_exporter.get_errata_rpms(errata_units, rpm_units) rpm_summary, rpm_errors = repo_exporter.export_rpms(rpm_units, progress_callback=progress_callback) if self.canceled: return publish_conduit.build_failure_report(self.summary, self.details) # generate metadata metadata_status, metadata_errors = metadata.generate_metadata( repo_working_dir, publish_conduit, config, progress_callback) _LOG.info("metadata generation complete at target location %s" % repo_working_dir) errata_summary, errata_errors = repo_exporter.export_errata(errata_units, progress_callback=progress_callback) self.summary = dict(self.summary.items() + rpm_summary.items() + errata_summary.items()) self.details["errors"] = rpm_errors + errata_errors + metadata_errors else: # export everything # export rpms criteria = UnitAssociationCriteria(type_ids=[TYPE_ID_RPM, TYPE_ID_SRPM, TYPE_ID_DRPM]) rpm_units = publish_conduit.get_units(criteria=criteria) rpm_summary, rpm_errors = repo_exporter.export_rpms(rpm_units, progress_callback=progress_callback) # export package groups groups_xml_path = None if "packagegroup" not in skip_types: progress_status["packagegroups"]["state"] = "STARTED" criteria = UnitAssociationCriteria(type_ids=[TYPE_ID_PKG_GROUP, TYPE_ID_PKG_CATEGORY]) existing_units = publish_conduit.get_units(criteria=criteria) existing_groups = filter(lambda u : u.type_id in [TYPE_ID_PKG_GROUP], existing_units) existing_cats = filter(lambda u : u.type_id in [TYPE_ID_PKG_CATEGORY], existing_units) groups_xml_path = comps_util.write_comps_xml(repo_working_dir, existing_groups, existing_cats) self.summary["num_package_groups_exported"] = len(existing_groups) self.summary["num_package_categories_exported"] = len(existing_cats) progress_status["packagegroups"]["state"] = "FINISHED" else: progress_status["packagegroups"]["state"] = "SKIPPED" _LOG.info("packagegroup unit type in skip list [%s]; skipping export" % skip_types) if self.canceled: return publish_conduit.build_failure_report(self.summary, self.details) # generate metadata metadata_status, metadata_errors = metadata.generate_metadata( repo_working_dir, publish_conduit, config, progress_callback, groups_xml_path) _LOG.info("metadata generation complete at target location %s" % repo_working_dir) # export errata criteria = UnitAssociationCriteria(type_ids=[TYPE_ID_ERRATA]) errata_units = publish_conduit.get_units(criteria=criteria) errata_summary, errata_errors = repo_exporter.export_errata(errata_units, progress_callback=progress_callback) # distro units criteria = UnitAssociationCriteria(type_ids=[TYPE_ID_DISTRO]) distro_units = publish_conduit.get_units(criteria=criteria) distro_summary, distro_errors = repo_exporter.export_distributions(distro_units, progress_callback=progress_callback) # sum up summary and details self.details["errors"] = rpm_errors + distro_errors + errata_errors + metadata_errors self.summary = dict(self.summary.items() + rpm_summary.items() + errata_summary.items() + distro_summary.items()) # build iso and publish via HTTPS self._publish_isos(repo, config, progress_callback=progress_callback) _LOG.info("Publish complete: summary = <%s>, details = <%s>" % (self.summary, self.details)) # remove exported content from working dirctory iso_util.cleanup_working_dir(self.repo_working_dir) if self.details["errors"]: return publish_conduit.build_failure_report(self.summary, self.details) return publish_conduit.build_success_report(self.summary, self.details)