def _createrepo(path): mdconf = createrepo.MetaDataConfig() mdconf.directory = path mdgen = createrepo.MetaDataGenerator(mdconf) mdgen.doPkgMetadata() mdgen.doRepoMetadata() mdgen.doFinalMove()
def create_metadata(repo, packages=None, comps=None): """ Generate YUM metadata for a repository. This method accepts a repository object and, based on its configuration, generates YUM metadata for it using the createrepo sister library. """ util.validate_repo(repo) conf = createrepo.MetaDataConfig() conf.directory = os.path.dirname(repo.pkgdir) conf.outputdir = os.path.dirname(repo.pkgdir) if packages: conf.pkglist = packages conf.quiet = True if comps: groupdir = tempfile.mkdtemp() conf.groupfile = os.path.join(groupdir, 'groups.xml') with open(conf.groupfile, 'w') as f: f.write(comps) generator = createrepo.SplitMetaDataGenerator(conf) generator.doPkgMetadata() generator.doRepoMetadata() generator.doFinalMove() if comps and os.path.exists(groupdir): shutil.rmtree(groupdir)
def _createrepo(self): mdconf = createrepo.MetaDataConfig() mdconf.directory = self.tmpdir mdgen = createrepo.MetaDataGenerator(mdconf, MDCallback()) mdgen.doPkgMetadata() mdgen.doRepoMetadata() mdgen.doFinalMove()
def __init__(self, repolist=[], yumbase=None, mdconf=None, mdbase_class=None): self.repolist = repolist self.outputdir = '%s/merged_repo' % os.getcwd() self.exclude_tuples = [] self.sort_func = self._sort_func # callback function to magically sort pkgs if not mdconf: self.mdconf = createrepo.MetaDataConfig() else: self.mdconf = mdconf if not mdbase_class: self.mdbase_class = createrepo.MetaDataGenerator else: self.mdbase_class = mdbase_class if not yumbase: self.yumbase = yum.YumBase() else: self.yumbase = yumbase self.yumbase.conf.cachedir = getCacheDir() self.yumbase.conf.cache = 0 # default to all arches self.archlist = unique(rpmUtils.arch.arches.keys() + rpmUtils.arch.arches.values()) self.groups = True self.updateinfo = True
def __init__(self, directory): self.config = createrepo.MetaDataConfig() if not directory.endswith('/'): self.config.directory = directory + '/' else: self.config.directory = directory self.config.database_only = True self.output_dir = os.path.join(self.config.directory, 'repodata') self.temp_dir = os.path.join(self.config.directory, '.repodata')
def update_repodata(repopath, rpmfiles, options): tmpdir = tempfile.mkdtemp() s3base = urlparse.urlunsplit(('s3', options.bucket, repopath, '', '')) s3grabber = S3Grabber(s3base) # Set up temporary repo that will fetch repodata from s3 yumbase = yum.YumBase() yumbase.preconf.disabled_plugins = '*' yumbase.conf.cachedir = os.path.join(tmpdir, 'cache') yumbase.repos.disableRepo('*') repo = yumbase.add_enable_repo('s3') repo._grab = s3grabber repo._urls = [os.path.join(s3base, '')] # Ensure that missing base path doesn't cause trouble repo._sack = yum.sqlitesack.YumSqlitePackageSack( createrepo.readMetadata.CreaterepoPkgOld) # Create metadata generator mdconf = createrepo.MetaDataConfig() mdconf.directory = tmpdir mdconf.pkglist = yum.packageSack.MetaSack() mdgen = createrepo.MetaDataGenerator(mdconf, LoggerCallback()) mdgen.tempdir = tmpdir mdgen._grabber = s3grabber # Combine existing package sack with new rpm file list new_packages = yum.packageSack.PackageSack() for rpmfile in rpmfiles: newpkg = mdgen.read_in_package(os.path.join(s3base, rpmfile)) newpkg._baseurl = '' # don't leave s3 base urls in primary metadata older_pkgs = yumbase.pkgSack.searchNevra(name=newpkg.name) # Remove older versions of this package (or if it's the same version) for i, older in enumerate(reversed(older_pkgs), 1): if i > options.keep or older.pkgtup == newpkg.pkgtup: yumbase.pkgSack.delPackage(older) if options.delete_old and i > options.keep: s3grabber.urldelete(older.remote_url) else: logging.info('ignoring: %s', older.ui_nevra) new_packages.addPackage(newpkg) mdconf.pkglist.addSack('existing', yumbase.pkgSack) mdconf.pkglist.addSack('new', new_packages) # Write out new metadata to tmpdir mdgen.doPkgMetadata() mdgen.doRepoMetadata() mdgen.doFinalMove() # Replace metadata on s3 s3grabber.syncdir(os.path.join(tmpdir, 'repodata'), 'repodata') shutil.rmtree(tmpdir)
def make_cr_conf(): def_workers = os.nice(0) if def_workers > 0: def_workers = 1 else: def_workers = 0 conf = createrepo.MetaDataConfig() conf.excludes = ['debug/*'] conf.quiet = True conf.checksum = yum.misc._default_checksums[0] conf.database = True conf.update = True conf.retain_old_md = 15 conf.compress_type = 'compat' conf.workers = def_workers conf.split = True return conf
def _create_repo_metadata(self): print 'Creating repodata for %r' % self ensure_dir(self.output_dir) conf = createrepo.MetaDataConfig() conf.database = False conf.outputdir = self.output_dir conf.directory = self.output_dir conf.directories = [self.output_dir] if re.match(r'RedHatEnterpriseLinux[345]$', self.distro): conf.sumtype = 'sha' conf.pkglist = self.rpm_filenames # not sure what this is, but it defaults to True in newer createrepo, # but it makes yum explode conf.collapse_glibc_requires = False mdgen = createrepo.MetaDataGenerator(config_obj=conf) mdgen.doPkgMetadata() mdgen.doRepoMetadata() mdgen.doFinalMove()
def _rebuild_repository(self, conn, repo, packages, groupstree=None): basepath = utils.get_path_from_url(repo.url) self.logger.info("rebuild repository in %s", basepath) md_config = createrepo.MetaDataConfig() mdfile_path = os.path.join(basepath, md_config.finaldir, md_config.repomdfile) update = packages is not None and os.path.exists(mdfile_path) groupsfile = None if groupstree is None and update: # The createrepo lose the groups information on update # to prevent this set group info manually groupstree = self._load_groups(conn, repo) if groupstree is not None: groupsfile = os.path.join(tempfile.gettempdir(), "groups.xml") with open(groupsfile, "w") as fd: groupstree.write(fd) try: md_config.workers = 1 md_config.directory = str(basepath) md_config.groupfile = groupsfile md_config.update = update if not packages: # only generate meta-files, without packages info md_config.excludes = ["*"] mdgen = createrepo.MetaDataGenerator(config_obj=md_config, callback=CreaterepoCallBack( self.logger)) mdgen.doPkgMetadata() mdgen.doRepoMetadata() mdgen.doFinalMove() except createrepo.MDError as e: err_msg = six.text_type(e) self.logger.exception("failed to create yum repository in %s: %s", basepath, err_msg) shutil.rmtree(os.path.join(md_config.outputdir, md_config.tempdir), ignore_errors=True) raise RuntimeError( "Failed to create yum repository in {0}.".format(err_msg)) finally: if groupsfile is not None: os.unlink(groupsfile)
def main(args): """createrepo from cli main flow""" start_st = time.time() conf = createrepo.MetaDataConfig() conf = parse_args(args, conf) if conf.profile: print ('start time: %0.3f' % (time.time() - start_st)) mid_st = time.time() try: if conf.split: mdgen = createrepo.SplitMetaDataGenerator(config_obj=conf, callback=MDCallBack()) else: mdgen = createrepo.MetaDataGenerator(config_obj=conf, callback=MDCallBack()) if mdgen.checkTimeStamps(): if mdgen.conf.verbose: print _('repo is up to date') mdgen._cleanup_tmp_repodata_dir() sys.exit(0) if conf.profile: print ('mid time: %0.3f' % (time.time() - mid_st)) pm_st = time.time() mdgen.doPkgMetadata() if conf.profile: print ('pm time: %0.3f' % (time.time() - pm_st)) rm_st = time.time() mdgen.doRepoMetadata() if conf.profile: print ('rm time: %0.3f' % (time.time() - rm_st)) fm_st = time.time() mdgen.doFinalMove() if conf.profile: print ('fm time: %0.3f' % (time.time() - fm_st)) except MDError, errormsg: errorprint(_('%s') % errormsg) sys.exit(1)
def build_metadata(self): staging = tempfile.mkdtemp(prefix='yumsync-', suffix='-metadata') if self._packages is None: packages = [] else: packages = [ os.path.join(os.path.basename(self.package_dir), pkg) for pkg in self._packages ] if self.checksum == 'sha' or self.checksum == 'sha1': sumtype = 'sha' else: sumtype = 'sha256' conf = createrepo.MetaDataConfig() conf.directory = os.path.dirname(self.package_dir) conf.outputdir = staging conf.sumtype = sumtype conf.pkglist = packages conf.quiet = True if self._comps: groupdir = tempfile.mkdtemp(prefix='yumsync-', suffix='-groupdata') conf.groupfile = os.path.join(groupdir, 'groups.xml') with open(conf.groupfile, 'w') as f: f.write(self._comps) generator = createrepo.SplitMetaDataGenerator(conf) generator.doPkgMetadata() generator.doRepoMetadata() generator.doFinalMove() if self._comps and os.path.exists(groupdir): shutil.rmtree(groupdir) return staging
def mkmetadatadir(dir): """ Generate package metadata for a given directory; if it doesn't exist, then create it. """ log.debug("mkmetadatadir(%s)" % dir) if not isdir(dir): os.makedirs(dir) cache = config.get('createrepo_cache_dir') try: import createrepo conf = createrepo.MetaDataConfig() conf.cachedir = cache conf.outputdir = dir conf.directory = dir conf.quiet = True mdgen = createrepo.MetaDataGenerator(conf) mdgen.doPkgMetadata() mdgen.doRepoMetadata() mdgen.doFinalMove() except ImportError: sys.path.append('/usr/share/createrepo') import genpkgmetadata genpkgmetadata.main(['--cachedir', str(cache), '-q', str(dir)])
def update_repodata(bucketName, key, operation): logger.debug("key={0}".format(key)) if key.rfind("/") > -1: fileName = key[key.rfind("/") + 1:] (repoPath, relativeFileName) = extract_repo_file(key) packagePath = relativeFileName[:relativeFileName.rfind("/")] else: fileName = key relativeFileName = fileName repoPath = "" packagePath = '' (name, version, release, epoch, arch) = splitFilename(fileName) logger.debug("fileName={0}".format(fileName)) logger.debug("relativeFileName={0}".format(relativeFileName)) logger.debug("packagePath={0}".format(packagePath)) logger.debug("repoPath={0}".format(repoPath)) tmpdir = tempfile.mkdtemp() os.makedirs(os.path.join(tmpdir, packagePath)) s3base = urlparse.urlunsplit(("s3", bucketName, repoPath, "", "")) overridekey = os.path.join(tmpdir, fileName) overrideval = os.path.join(tmpdir, relativeFileName) s3grabber = S3Grabber(s3base, {overridekey: overrideval}) # Set up temporary repo that will fetch repodata from s3 yumbase = yum.YumBase() yumbase.preconf.disabled_plugins = '*' yumbase.conf.cachedir = os.path.join(tmpdir, 'cache') yumbase.repos.disableRepo('*') repo = yumbase.add_enable_repo('s3') repo._grab = s3grabber repo._urls = [os.path.join(s3base, '')] # Ensure that missing base path doesn't cause trouble repo._sack = yum.sqlitesack.YumSqlitePackageSack( createrepo.readMetadata.CreaterepoPkgOld) # Create metadata generator mdconf = createrepo.MetaDataConfig() mdconf.directory = tmpdir mdconf.pkglist = yum.packageSack.MetaSack() mdgen = createrepo.MetaDataGenerator(mdconf, LoggerCallback()) mdgen.tempdir = tmpdir mdgen._grabber = s3grabber new_packages = yum.packageSack.PackageSack() if operation == "add": # Combine existing package sack with new rpm file list newpkg = mdgen.read_in_package(os.path.join(s3base, relativeFileName)) newpkg._baseurl = '' # don't leave s3 base urls in primary metadata new_packages.addPackage(newpkg) else: # Remove deleted package logger.debug("Delete package {0}".format(key)) older_pkgs = yumbase.pkgSack.searchNevra(name=name) for i, older in enumerate(older_pkgs, 1): if older.version == version and older.release == release: yumbase.pkgSack.delPackage(older) mdconf.pkglist.addSack('existing', yumbase.pkgSack) mdconf.pkglist.addSack('new', new_packages) # Write out new metadata to tmpdir mdgen.doPkgMetadata() mdgen.doRepoMetadata() mdgen.doFinalMove() # Replace metadata on s3 s3grabber.syncdir(os.path.join(tmpdir, 'repodata'), 'repodata') shutil.rmtree(tmpdir)
beg = "%*d/%d - " % (len(str(total)), current, total) left = 80 - len(beg) sys.stdout.write("\r%s%-*.*s" % (beg, left, left, item)) sys.stdout.flush() def main(args): """createrepo from cli main flow""" try: os.getcwd() except OSError, e: if e.errno != errno.ENOENT: raise print('No getcwd() access in current directory.') sys.exit(1) start_st = time.time() conf = createrepo.MetaDataConfig() conf = parse_args(args, conf) if conf.profile: print('start time: %0.3f' % (time.time() - start_st)) mid_st = time.time() try: if conf.split: mdgen = createrepo.SplitMetaDataGenerator(config_obj=conf, callback=MDCallBack()) else: mdgen = createrepo.MetaDataGenerator(config_obj=conf, callback=MDCallBack()) if mdgen.checkTimeStamps(): if mdgen.conf.verbose: print _('repo is up to date')