def symlink_distribution_unit_files(self, units, symlink_dir, progress_callback=None): """ Publishing distriubution unit involves publishing files underneath the unit. Distribution is an aggregate unit with distribution files. This call looksup each distribution unit and symlinks the files from the storage location to working directory. @param units @type AssociatedUnit @param symlink_dir: path of where we want the symlink to reside @type symlink_dir str @param progress_callback: callback to report progress info to publish_conduit @type progress_callback: function @return tuple of status and list of error messages if any occurred @rtype (bool, [str]) """ distro_progress_status = self.init_progress() self.set_progress("distribution", distro_progress_status, progress_callback) _LOG.info("Process symlinking distribution files with %s units to %s dir" % (len(units), symlink_dir)) errors = [] for u in units: source_path_dir = u.storage_path if not u.metadata.has_key('files'): msg = "No distribution files found for unit %s" % u _LOG.error(msg) distro_files = u.metadata['files'] _LOG.info("Found %s distribution files to symlink" % len(distro_files)) distro_progress_status['items_total'] = len(distro_files) distro_progress_status['items_left'] = len(distro_files) for dfile in distro_files: self.set_progress("distribution", distro_progress_status, progress_callback) source_path = os.path.join(source_path_dir, dfile['relativepath']) symlink_path = os.path.join(symlink_dir, dfile['relativepath']) if not os.path.exists(source_path): msg = "Source path: %s is missing" % source_path errors.append((source_path, symlink_path, msg)) distro_progress_status['num_error'] += 1 distro_progress_status["items_left"] -= 1 continue try: if not util.create_symlink(source_path, symlink_path): msg = "Unable to create symlink for: %s pointing to %s" % (symlink_path, source_path) _LOG.error(msg) errors.append((source_path, symlink_path, msg)) distro_progress_status['num_error'] += 1 distro_progress_status["items_left"] -= 1 continue distro_progress_status['num_success'] += 1 except Exception, e: tb_info = traceback.format_exc() _LOG.error("%s" % tb_info) _LOG.critical(e) errors.append((source_path, symlink_path, str(e))) distro_progress_status['num_error'] += 1 distro_progress_status["items_left"] -= 1 continue distro_progress_status["items_left"] -= 1
def handle_symlinks(self, units, symlink_dir, progress_callback=None): """ @param units list of units that belong to the repo and should be published @type units [AssociatedUnit] @param symlink_dir where to create symlinks @type symlink_dir str @param progress_callback: callback to report progress info to publish_conduit @type progress_callback: function @return tuple of status and list of error messages if any occurred @rtype (bool, [str]) """ packages_progress_status = self.init_progress() _LOG.info("handle_symlinks invoked with %s units to %s dir" % (len(units), symlink_dir)) self.set_progress("packages", packages_progress_status, progress_callback) errors = [] packages_progress_status["items_total"] = len(units) packages_progress_status["items_left"] = len(units) for u in units: self.set_progress("packages", packages_progress_status, progress_callback) relpath = util.get_relpath_from_unit(u) source_path = u.storage_path symlink_path = os.path.join(symlink_dir, relpath) if not os.path.exists(source_path): msg = "Source path: %s is missing" % (source_path) errors.append((source_path, symlink_path, msg)) packages_progress_status["num_error"] += 1 packages_progress_status["items_left"] -= 1 continue _LOG.info("Unit exists at: %s we need to symlink to: %s" % (source_path, symlink_path)) try: if not util.create_symlink(source_path, symlink_path): msg = "Unable to create symlink for: %s pointing to %s" % (symlink_path, source_path) _LOG.error(msg) errors.append((source_path, symlink_path, msg)) packages_progress_status["num_error"] += 1 packages_progress_status["items_left"] -= 1 continue packages_progress_status["num_success"] += 1 except Exception, e: tb_info = traceback.format_exc() _LOG.error("%s" % (tb_info)) _LOG.critical(e) errors.append((source_path, symlink_path, str(e))) packages_progress_status["num_error"] += 1 packages_progress_status["items_left"] -= 1 continue packages_progress_status["items_left"] -= 1
def test_create_symlink(self): target_dir = os.path.join(self.temp_dir, "a", "b", "c", "d", "e") distributor = YumDistributor() # Create an empty file to serve as the source_path source_path = os.path.join(self.temp_dir, "some_test_file.txt") open(source_path, "a") symlink_path = os.path.join(self.temp_dir, "symlink_dir", "a", "b", "file_path.lnk") # Confirm subdir of symlink_path doesn't exist self.assertFalse(os.path.isdir(os.path.dirname(symlink_path))) self.assertTrue(util.create_symlink(source_path, symlink_path)) # Confirm we created the subdir self.assertTrue(os.path.isdir(os.path.dirname(symlink_path))) self.assertTrue(os.path.exists(symlink_path)) self.assertTrue(os.path.islink(symlink_path)) # Verify the symlink points to the source_path a = os.readlink(symlink_path) self.assertEqual(a, source_path)
def symlink_distribution_unit_files(self, units, symlink_dir, publish_conduit, progress_callback=None): """ Publishing distriubution unit involves publishing files underneath the unit. Distribution is an aggregate unit with distribution files. This call looksup each distribution unit and symlinks the files from the storage location to working directory. @param units @type AssociatedUnit @param symlink_dir: path of where we want the symlink to reside @type symlink_dir str @param progress_callback: callback to report progress info to publish_conduit @type progress_callback: function @return tuple of status and list of error messages if any occurred @rtype (bool, [str]) """ distro_progress_status = self.init_progress() self.set_progress("distribution", distro_progress_status, progress_callback) _LOG.debug("Process symlinking distribution files with %s units to %s dir" % (len(units), symlink_dir)) # handle orphaned existing_scratchpad = publish_conduit.get_scratchpad() or {} scratchpad = self._handle_orphaned_distributions(units, symlink_dir, existing_scratchpad) errors = [] for u in units: source_path_dir = u.storage_path if not u.metadata.has_key('files'): msg = "No distribution files found for unit %s" % u _LOG.error(msg) distro_files = u.metadata['files'] _LOG.debug("Found %s distribution files to symlink" % len(distro_files)) distro_progress_status['items_total'] = len(distro_files) distro_progress_status['items_left'] = len(distro_files) # Lookup treeinfo file in the source location src_treeinfo_path = None for treeinfo in constants.TREE_INFO_LIST: src_treeinfo_path = os.path.join(source_path_dir, treeinfo) if os.path.exists(src_treeinfo_path): # we found the treeinfo file break if src_treeinfo_path is not None: # create a symlink from content location to repo location. symlink_treeinfo_path = os.path.join(symlink_dir, treeinfo) _LOG.debug("creating treeinfo symlink from %s to %s" % (src_treeinfo_path, symlink_treeinfo_path)) util.create_symlink(src_treeinfo_path, symlink_treeinfo_path) published_distro_files = [] for dfile in distro_files: self.set_progress("distribution", distro_progress_status, progress_callback) source_path = os.path.join(source_path_dir, dfile['relativepath']) symlink_path = os.path.join(symlink_dir, dfile['relativepath']) if os.path.exists(symlink_path): # path already exists, skip symlink distro_progress_status["items_left"] -= 1 published_distro_files.append(symlink_path) continue if not os.path.exists(source_path): msg = "Source path: %s is missing" % source_path errors.append((source_path, symlink_path, msg)) distro_progress_status['num_error'] += 1 distro_progress_status["items_left"] -= 1 continue try: if not util.create_symlink(source_path, symlink_path): msg = "Unable to create symlink for: %s pointing to %s" % (symlink_path, source_path) _LOG.error(msg) errors.append((source_path, symlink_path, msg)) distro_progress_status['num_error'] += 1 distro_progress_status["items_left"] -= 1 continue distro_progress_status['num_success'] += 1 published_distro_files.append(symlink_path) except Exception, e: tb_info = traceback.format_exc() _LOG.error("%s" % tb_info) _LOG.critical(e) errors.append((source_path, symlink_path, str(e))) distro_progress_status['num_error'] += 1 distro_progress_status["items_left"] -= 1 continue distro_progress_status["items_left"] -= 1 scratchpad.update({constants.PUBLISHED_DISTRIBUTION_FILES_KEY : {u.id : published_distro_files}})
def publish_repo(self, repo, publish_conduit, config): summary = {} details = {} progress_status = { "packages": {"state": "NOT_STARTED"}, "distribution": {"state": "NOT_STARTED"}, "metadata": {"state": "NOT_STARTED"}, "packagegroups": {"state": "NOT_STARTED"}, "publish_http": {"state": "NOT_STARTED"}, "publish_https": {"state": "NOT_STARTED"}, } def progress_callback(type_id, status): progress_status[type_id] = status publish_conduit.set_progress(progress_status) self.repo_working_dir = repo.working_dir if self.canceled: return publish_conduit.build_cancel_report(summary, details) skip_list = config.get('skip') or [] # Determine Content in this repo pkg_units = [] pkg_errors = [] if 'rpm' not in skip_list: for type_id in [TYPE_ID_RPM, TYPE_ID_SRPM]: criteria = UnitAssociationCriteria(type_ids=type_id, unit_fields=['id', 'name', 'version', 'release', 'arch', 'epoch', '_storage_path', "checksum", "checksumtype" ]) pkg_units += publish_conduit.get_units(criteria=criteria) drpm_units = [] if 'drpm' not in skip_list: criteria = UnitAssociationCriteria(type_ids=TYPE_ID_DRPM) drpm_units = publish_conduit.get_units(criteria=criteria) pkg_units += drpm_units # Create symlinks under repo.working_dir pkg_status, pkg_errors = self.handle_symlinks(pkg_units, repo.working_dir, progress_callback) if not pkg_status: _LOG.error("Unable to publish %s items" % (len(pkg_errors))) distro_errors = [] distro_units = [] if 'distribution' not in skip_list: criteria = UnitAssociationCriteria(type_ids=TYPE_ID_DISTRO) distro_units = publish_conduit.get_units(criteria=criteria) # symlink distribution files if any under repo.working_dir distro_status, distro_errors = self.symlink_distribution_unit_files(distro_units, repo.working_dir, publish_conduit, progress_callback) if not distro_status: _LOG.error("Unable to publish distribution tree %s items" % (len(distro_errors))) updateinfo_xml_path = None if 'erratum' not in skip_list: criteria = UnitAssociationCriteria(type_ids=TYPE_ID_ERRATA) errata_units = publish_conduit.get_units(criteria=criteria) updateinfo_xml_path = updateinfo.updateinfo(errata_units, repo.working_dir) if self.canceled: return publish_conduit.build_cancel_report(summary, details) groups_xml_path = None existing_cats = [] existing_groups = [] if 'packagegroup' not in skip_list: criteria = UnitAssociationCriteria(type_ids=[TYPE_ID_PKG_GROUP, TYPE_ID_PKG_CATEGORY]) existing_units = publish_conduit.get_units(criteria) existing_groups = filter(lambda u : u.type_id in [TYPE_ID_PKG_GROUP], existing_units) existing_cats = filter(lambda u : u.type_id in [TYPE_ID_PKG_CATEGORY], existing_units) groups_xml_path = comps_util.write_comps_xml(repo.working_dir, existing_groups, existing_cats) metadata_start_time = time.time() # update/generate metadata for the published repo self.use_createrepo = config.get('use_createrepo') if self.use_createrepo: metadata_status, metadata_errors = metadata.generate_metadata( repo.working_dir, publish_conduit, config, progress_callback, groups_xml_path) else: metadata_status, metadata_errors = metadata.generate_yum_metadata(repo.id, repo.working_dir, publish_conduit, config, progress_callback, is_cancelled=self.canceled, group_xml_path=groups_xml_path, updateinfo_xml_path=updateinfo_xml_path, repo_scratchpad=publish_conduit.get_repo_scratchpad()) metadata_end_time = time.time() relpath = self.get_repo_relative_path(repo, config) if relpath.startswith("/"): relpath = relpath[1:] # Build the https and http publishing paths https_publish_dir = self.get_https_publish_dir(config) https_repo_publish_dir = os.path.join(https_publish_dir, relpath).rstrip('/') http_publish_dir = self.get_http_publish_dir(config) http_repo_publish_dir = os.path.join(http_publish_dir, relpath).rstrip('/') # Clean up the old publish directories, if they exist. scratchpad = publish_conduit.get_repo_scratchpad() if OLD_REL_PATH_KEYWORD in scratchpad: old_relative_path = scratchpad[OLD_REL_PATH_KEYWORD] old_https_repo_publish_dir = os.path.join(https_publish_dir, old_relative_path) if os.path.exists(old_https_repo_publish_dir): util.remove_repo_publish_dir(https_publish_dir, old_https_repo_publish_dir) old_http_repo_publish_dir = os.path.join(http_publish_dir, old_relative_path) if os.path.exists(old_http_repo_publish_dir): util.remove_repo_publish_dir(http_publish_dir, old_http_repo_publish_dir) # Now write the current publish relative path to the scratch pad. This way, if the relative path # changes before the next publish, we can clean up the old path. scratchpad[OLD_REL_PATH_KEYWORD] = relpath publish_conduit.set_repo_scratchpad(scratchpad) # Handle publish link for HTTPS if config.get("https"): # Publish for HTTPS self.set_progress("publish_https", {"state" : "IN_PROGRESS"}, progress_callback) try: _LOG.info("HTTPS Publishing repo <%s> to <%s>" % (repo.id, https_repo_publish_dir)) util.create_symlink(repo.working_dir, https_repo_publish_dir) util.generate_listing_files(https_publish_dir, https_repo_publish_dir) summary["https_publish_dir"] = https_repo_publish_dir self.set_progress("publish_https", {"state" : "FINISHED"}, progress_callback) except: self.set_progress("publish_https", {"state" : "FAILED"}, progress_callback) else: self.set_progress("publish_https", {"state" : "SKIPPED"}, progress_callback) if os.path.lexists(https_repo_publish_dir): _LOG.debug("Removing link for %s since https is not set" % https_repo_publish_dir) util.remove_repo_publish_dir(https_publish_dir, https_repo_publish_dir) # Handle publish link for HTTP if config.get("http"): # Publish for HTTP self.set_progress("publish_http", {"state" : "IN_PROGRESS"}, progress_callback) try: _LOG.info("HTTP Publishing repo <%s> to <%s>" % (repo.id, http_repo_publish_dir)) util.create_symlink(repo.working_dir, http_repo_publish_dir) util.generate_listing_files(http_publish_dir, http_repo_publish_dir) summary["http_publish_dir"] = http_repo_publish_dir self.set_progress("publish_http", {"state" : "FINISHED"}, progress_callback) except: self.set_progress("publish_http", {"state" : "FAILED"}, progress_callback) else: self.set_progress("publish_http", {"state" : "SKIPPED"}, progress_callback) if os.path.lexists(http_repo_publish_dir): _LOG.debug("Removing link for %s since http is not set" % http_repo_publish_dir) util.remove_repo_publish_dir(http_publish_dir, http_repo_publish_dir) summary["num_package_units_attempted"] = len(pkg_units) summary["num_package_units_published"] = len(pkg_units) - len(pkg_errors) summary["num_package_units_errors"] = len(pkg_errors) summary["num_distribution_units_attempted"] = len(distro_units) summary["num_distribution_units_published"] = len(distro_units) - len(distro_errors) summary["num_distribution_units_errors"] = len(distro_errors) summary["num_package_groups_published"] = len(existing_groups) summary["num_package_categories_published"] = len(existing_cats) summary["relative_path"] = relpath if metadata_status is False and not len(metadata_errors): summary["skip_metadata_update"] = True else: summary["skip_metadata_update"] = False details["errors"] = pkg_errors + distro_errors # metadata_errors details['time_metadata_sec'] = metadata_end_time - metadata_start_time # metadata generate skipped vs run _LOG.info("Publish complete: summary = <%s>, details = <%s>" % (summary, details)) if details["errors"]: return publish_conduit.build_failure_report(summary, details) return publish_conduit.build_success_report(summary, details)
continue distro_progress_status['num_success'] += 1 published_distro_files.append(symlink_path) except Exception, e: tb_info = traceback.format_exc() _LOG.error("%s" % tb_info) _LOG.critical(e) errors.append((source_path, symlink_path, str(e))) distro_progress_status['num_error'] += 1 distro_progress_status["items_left"] -= 1 continue distro_progress_status["items_left"] -= 1 scratchpad.update({constants.PUBLISHED_DISTRIBUTION_FILES_KEY : {u.id : published_distro_files}}) # create the Packages symlink to the content dir, in the content dir packages_symlink_path = os.path.join(symlink_dir, 'Packages') if not util.create_symlink(symlink_dir, packages_symlink_path): msg = 'Unable to create Packages symlink required for RHEL 5 distributions' _LOG.error(msg) errors.append((symlink_dir, packages_symlink_path, msg)) publish_conduit.set_scratchpad(scratchpad) if errors: distro_progress_status["error_details"] = errors distro_progress_status["state"] = "FAILED" self.set_progress("distribution", distro_progress_status, progress_callback) return False, errors distro_progress_status["state"] = "FINISHED" self.set_progress("distribution", distro_progress_status, progress_callback) return True, [] def _handle_orphaned_distributions(self, units, repo_working_dir, scratchpad): distro_unit_ids = [u.id for u in units]
def publish_repo(self, repo, publish_conduit, config): summary = {} details = {} progress_status = { "packages": {"state": "NOT_STARTED"}, "distribution": {"state": "NOT_STARTED"}, "metadata": {"state": "NOT_STARTED"}, "packagegroups": {"state": "NOT_STARTED"}, "publish_http": {"state": "NOT_STARTED"}, "publish_https": {"state": "NOT_STARTED"}, } def progress_callback(type_id, status): progress_status[type_id] = status publish_conduit.set_progress(progress_status) self.repo_working_dir = repo.working_dir if self.canceled: return publish_conduit.build_failure_report(summary, details) skip_list = config.get('skip') or [] # Determine Content in this repo unfiltered_units = publish_conduit.get_units() # filter compatible units rpm_units = filter(lambda u : u.type_id in [TYPE_ID_RPM, TYPE_ID_SRPM], unfiltered_units) drpm_units = filter(lambda u : u.type_id == TYPE_ID_DRPM, unfiltered_units) rpm_errors = [] if 'rpm' not in skip_list: _LOG.info("Publish on %s invoked. %s existing units, %s of which are supported to be published." \ % (repo.id, len(unfiltered_units), len(rpm_units))) # Create symlinks under repo.working_dir rpm_status, rpm_errors = self.handle_symlinks(rpm_units, repo.working_dir, progress_callback) if not rpm_status: _LOG.error("Unable to publish %s items" % (len(rpm_errors))) drpm_errors = [] if 'drpm' not in skip_list: _LOG.info("Publish on %s invoked. %s existing units, %s of which are supported to be published." \ % (repo.id, len(unfiltered_units), len(drpm_units))) # Create symlinks under repo.working_dir drpm_status, drpm_errors = self.handle_symlinks(drpm_units, repo.working_dir, progress_callback) if not drpm_status: _LOG.error("Unable to publish %s items" % (len(drpm_errors))) pkg_errors = rpm_errors + drpm_errors pkg_units = rpm_units + drpm_units distro_errors = [] distro_units = filter(lambda u: u.type_id == TYPE_ID_DISTRO, unfiltered_units) if 'distribution' not in skip_list: # symlink distribution files if any under repo.working_dir distro_status, distro_errors = self.symlink_distribution_unit_files(distro_units, repo.working_dir, progress_callback) if not distro_status: _LOG.error("Unable to publish distribution tree %s items" % (len(distro_errors))) # update/generate metadata for the published repo repo_scratchpad = publish_conduit.get_repo_scratchpad() src_working_dir = '' if repo_scratchpad.has_key("importer_working_dir"): src_working_dir = repo_scratchpad['importer_working_dir'] if self.canceled: return publish_conduit.build_failure_report(summary, details) groups_xml_path = None existing_cats = [] existing_groups = [] if 'packagegroup' not in skip_list: criteria = UnitAssociationCriteria(type_ids=[TYPE_ID_PKG_GROUP, TYPE_ID_PKG_CATEGORY]) existing_units = publish_conduit.get_units(criteria) existing_groups = filter(lambda u : u.type_id in [TYPE_ID_PKG_GROUP], existing_units) existing_cats = filter(lambda u : u.type_id in [TYPE_ID_PKG_CATEGORY], existing_units) groups_xml_path = comps_util.write_comps_xml(repo, existing_groups, existing_cats) metadata_start_time = time.time() self.copy_importer_repodata(src_working_dir, repo.working_dir) metadata_status, metadata_errors = metadata.generate_metadata( repo, publish_conduit, config, progress_callback, groups_xml_path) metadata_end_time = time.time() relpath = self.get_repo_relative_path(repo, config) if relpath.startswith("/"): relpath = relpath[1:] # # Handle publish link for HTTPS # https_publish_dir = self.get_https_publish_dir(config) https_repo_publish_dir = os.path.join(https_publish_dir, relpath).rstrip('/') if config.get("https"): # Publish for HTTPS self.set_progress("publish_https", {"state" : "IN_PROGRESS"}, progress_callback) try: _LOG.info("HTTPS Publishing repo <%s> to <%s>" % (repo.id, https_repo_publish_dir)) util.create_symlink(repo.working_dir, https_repo_publish_dir) summary["https_publish_dir"] = https_repo_publish_dir self.set_progress("publish_https", {"state" : "FINISHED"}, progress_callback) except: self.set_progress("publish_https", {"state" : "FAILED"}, progress_callback) else: self.set_progress("publish_https", {"state" : "SKIPPED"}, progress_callback) if os.path.lexists(https_repo_publish_dir): _LOG.debug("Removing link for %s since https is not set" % https_repo_publish_dir) util.remove_symlink(https_publish_dir, https_repo_publish_dir) # # Handle publish link for HTTP # http_publish_dir = self.get_http_publish_dir(config) http_repo_publish_dir = os.path.join(http_publish_dir, relpath).rstrip('/') if config.get("http"): # Publish for HTTP self.set_progress("publish_http", {"state" : "IN_PROGRESS"}, progress_callback) try: _LOG.info("HTTP Publishing repo <%s> to <%s>" % (repo.id, http_repo_publish_dir)) util.create_symlink(repo.working_dir, http_repo_publish_dir) summary["http_publish_dir"] = http_repo_publish_dir self.set_progress("publish_http", {"state" : "FINISHED"}, progress_callback) except: self.set_progress("publish_http", {"state" : "FAILED"}, progress_callback) else: self.set_progress("publish_http", {"state" : "SKIPPED"}, progress_callback) if os.path.lexists(http_repo_publish_dir): _LOG.debug("Removing link for %s since http is not set" % http_repo_publish_dir) util.remove_symlink(http_publish_dir, http_repo_publish_dir) summary["num_package_units_attempted"] = len(pkg_units) summary["num_package_units_published"] = len(pkg_units) - len(pkg_errors) summary["num_package_units_errors"] = len(pkg_errors) summary["num_distribution_units_attempted"] = len(distro_units) summary["num_distribution_units_published"] = len(distro_units) - len(distro_errors) summary["num_distribution_units_errors"] = len(distro_errors) summary["num_package_groups_published"] = len(existing_groups) summary["num_package_categories_published"] = len(existing_cats) summary["relative_path"] = relpath if metadata_status is False and not len(metadata_errors): summary["skip_metadata_update"] = True else: summary["skip_metadata_update"] = False details["errors"] = pkg_errors + distro_errors + metadata_errors details['time_metadata_sec'] = metadata_end_time - metadata_start_time # metadata generate skipped vs run _LOG.info("Publish complete: summary = <%s>, details = <%s>" % (summary, details)) if details["errors"]: return publish_conduit.build_failure_report(summary, details) return publish_conduit.build_success_report(summary, details)