def main():
""" Main entrypoint for sinkhole
"""
# top-level parser
parser = argparse.ArgumentParser(description="sinkhole")
parser.add_argument("--revision", action="store")
parser.add_argument("--config", action="store", required=True)
subparsers = parser.add_subparsers(dest="subcommand")
subparsers.required = True
# reposync
parser_reposync = subparsers.add_parser("reposync",
help="Sync repositories")
parser_reposync.add_argument("--destdir", action="store", required=True)
parser_reposync.add_argument("--repofile",
action="append",
dest="repofns",
required=True)
parser_reposync.add_argument("--include", action="append", default=None)
parser_reposync.add_argument("--exclude", action="append", default=None)
# kojidownload
parser_koji = subparsers.add_parser("kojidownload",
help="Download builds from Koji")
parser_koji.add_argument("--profile",
action="store",
default="koji",
help="Koji instance to retrieve builds from")
parser_koji.add_argument("--arch", action="store", nargs="+", default=None)
parser_koji.add_argument("--builds",
action="store",
nargs="+",
required=True)
parser_koji.add_argument("--destdir", action="store", required=True)
# config_file
parser_config = subparsers.add_parser("config_file",
help="Use config file")
# Run appropriate command
args = parser.parse_args()
config = Config.read_config(args.config) if args.config else Config()
if args.subcommand == "reposync":
config.output_dir = args.destdir
reposync = Reposync(args.repofns, args.include, args.exclude)
reposync.run()
elif args.subcommand == "kojidownload":
config.output_dir = args.destdir
kojid = KojiDownloader(args.profile, args.builds, args.arch)
kojid.run()
else:
sources = SourceBuilder.build()
for source in sources:
source.run()
rm = RepoMasher(config.output_dir, revision=args.revision)
rm.run()
def test_build_non_empty(self):
"""Check that RepoSync provider is correctly built
"""
config = Config()
config.sources = [{"name": "sources.fedora",
"type": "repofile",
"repofile": "something.repo"}]
sources = SourceBuilder.build()
assert len(sources) == 1
def test_shared_state(self):
"""Check that state is shared across instances of Config
"""
config1 = Config()
config1.workers = 4
config1.output_dir = "/tmp/some_dir"
config2 = Config()
assert config1.workers == config2.workers == 4
assert config1.output_dir == config2.output_dir == "/tmp/some_dir"
def test_build_empty(self):
"""Check that unrecognized source providers are
correctly processed
"""
config = Config()
config.sources = [{"name": "sources.something",
"type": "foobar"}]
sources = SourceBuilder.build()
assert len(sources) == 0
def __init__(self, repofns=None, include_pkgs=None, exclude_pkgs=None):
self.include_pkgs = include_pkgs
self.exclude_pkgs = exclude_pkgs
self.repofns = repofns if repofns is not None else []
self.base = dnf.Base()
self.conf = self.base.conf
config = Config()
self.conf.substitutions['releasever'] = \
config.default_substitutions["releasever"]
self.conf.substitutions['basearch'] = \
config.default_substitutions["basearch"]
self.conf.cachedir = os.path.join(Config().output_dir, "cache")
self.repos = self.base.repos
def run(self):
""" Do the repo syncing
"""
self._setup_repos()
self.base.fill_sack()
available_pkgs = self.base.sack.query().available().run()
download_pkgs = self._filter_download_pkgs(available_pkgs)
#
# sinkhole --repofile fedora.repo --destdir tmp2
# 23.27s user 2.14s system 1% cpu 28:19.24 total
download_pkgs = [pkg.remote_location() for pkg in download_pkgs]
config = Config()
packages_dir = os.path.join(config.output_dir, "Packages")
if not os.path.exists(packages_dir):
os.makedirs(packages_dir)
pool = Pool(config.workers)
pool.map(partial(download_packages, destdir=packages_dir),
download_pkgs)
pool.close()
pool.join()
# 3161
# sinkhole --repofile fedora.repo --destdir tmp2
# 699.75s user 195.08s system 14% cpu 1:41:17.52 total
# self.base.download_packages(download_pkgs,
# MultiFileProgressMeter(fo=sys.stdout))
self._cleanup_dnf_artefacts()
def __init__(self, profile, builds=None, exclude=None):
self.profile = profile
self.koji = setup_kojiclient(self.profile)
self._builds = self.builds(builds)
self._config = Config()
basearch = self._config.default_substitutions['basearch']
self._arches = ARCHES_DICT[basearch]
self._exclude = exclude
def __init__(self, destdir, revision=None):
self.destdir = destdir
config = Config()
opts = config.masher_opts
workers = config.masher_workers
if revision:
opts = "%s --revision %s" % (opts, revision)
self.createrepo = createrepo.bake(*opts.split(), workers=workers)
def build():
""" Build all sources
"""
config = Config()
sources = []
for info in config.sources:
typ = info["type"]
source = SOURCES[typ].build(info)
if source:
sources.append(source)
return sources
def test_read_config(self):
"""Check that configuration file are properly parsed
"""
data = StringIO("""[main]
workers = 8
output_dir = output/
releasever = 28
basearch = x86_64
[sources.fedora]
type=repofile
repofile=sources/fedora.repo
constraints=sources/fedora_constraints
""")
config = Config.read_config(data)
assert config.workers == 8
assert config.output_dir == "output/"
assert config.default_substitutions["releasever"] == "28"
assert config.default_substitutions["basearch"] == "x86_64"
def run(self):
""" Execute downloads
"""
urls = []
pathinfo = koji.PathInfo(topdir=self.koji.opts['topurl'])
config = Config()
for build in self._builds:
try:
info = self.koji.getBuild(build)
rpms = self.koji.listRPMs(buildID=info['id'],
arches=self._arches)
fnames = [pathinfo.rpm(rpm) for rpm in rpms]
urls += [
pathinfo.build(info) + '/' + fname for fname in fnames
]
except Exception:
print('SKIPPED: build {} does not exists'.format(build))
for url in urls:
download_packages(url, config.output_dir)
def __init__(self, destdir):
self.destdir = destdir
config = Config()
opts = config.masher_opts
workers = config.masher_workers
self.createrepo = createrepo.bake(*opts.split(), workers=workers)