def queryPackages(suiteStr, requestPackages, regexStr, archStr, componentStr, fieldStr, noUpdate=False, querySources=False):
'''
queries Packages by the args provided on the command line and returns a
tuple of (queryResults, requestFields)
'''
suites = apt_repos.getSuites(suiteStr.split(','))
requestArchs = { a for a in archStr.split(',') } if archStr else {}
requestComponents = { c for c in componentStr.split(',') } if componentStr else {}
requestFields = PackageField.getByFieldsString(fieldStr)
result = set()
showProgress = True
pp(showProgress, "{}querying packages lists for {} suites".format(
"updating (use --no-update to skip) and " if not noUpdate else "", len(suites)))
for x, suite in enumerate(suites):
pp(showProgress, '.')
try:
suite.scan(not noUpdate)
pp(showProgress, x+1)
if not querySources:
result = result.union(suite.queryPackages(requestPackages, regexStr, requestArchs, requestComponents, requestFields))
else:
result = result.union(suite.querySources(requestPackages, regexStr, requestArchs, requestComponents, requestFields))
except SystemError as e:
logger.warn("Could not retrieve {} for suite {}:\n{}".format("sources" if querySources else "packages", suite.getSuiteName(), e))
pp(showProgress, '\n')
return (result, requestFields)
def dsc(args):
'''
subcommand dsc: list urls of dsc-files available for source-packages.
'''
# parse --suite and determine the specific suite scan-order
suites = list()
for selector in args.suite.split(','):
suites.extend(sorted(apt_repos.getSuites([selector]), reverse=True))
requestPackages = { p for p in args.source }
requestComponents = { c for c in args.component.split(',') } if args.component else {}
showProgress = True
pp(showProgress, "{}querying sources lists for {} suites".format(
"updating (use --no-update to skip) and " if not args.no_update else "", len(suites)))
results = {}
for package in requestPackages: # pre-seed results
results[package] = list()
for x, suite in enumerate(suites):
pp(showProgress, ".{}".format(x+1))
queryDscFiles(results, suite, requestComponents, logger, not args.no_update, args.first)
if args.first and gotAllFirsts(results):
break
pp(showProgress, '\n')
for package, urls in sorted(results.items()):
for url in urls[: 1 if (args.first and len(urls) > 0) else len(urls)]:
print(url)
def testQuerySources():
apt_repos.setAptReposBaseDir(".")
fields = PackageField.getByFieldsString('CvsaSFB')
repoSuite = list(apt_repos.getSuites(["ubuntu:trusty"]))[0]
repoSuite.scan(True)
res = repoSuite.querySources(['git'], False, None, None, fields)
for qr in sorted(res):
print(qr)
def __init__(self, suiteName, packages):
suites = sorted(apt_repos.getSuites([suiteName]))
if len(suites) != 1:
raise BundleError(
"Can't create UpdateRule for suite selector '{}' since it doesn't select exactly one suite."
.format(suiteName))
self.suite = suites[0]
self.packages = packages
def suites(args):
'''
subcommand suites: print a list of registered suites
'''
suites = apt_repos.getSuites(args.suite.split(','))
for s in sorted(suites):
print("# {}{}".format(s.getSuiteName(), (" [" + (":, ".join(sorted(s.getTags())) + ":]")) if len(s.getTags()) > 0 else ""))
if args.verbose:
print(s.getSourcesList() + "\n")
def testSuiteSelectors():
apt_repos.setAptReposBaseDir(".")
selectors = [
None, [":"], ["default:"], ["ubuntu:xenial"], ["xenial"],
["ub:trusty"], ["ubuntu:"], ["u:"], ["u:trusty"],
["ubuntu:trusty-security", "ubuntu:xenial-security"], ["debian:"],
["ubuntu:"], ["ubuntu:de-bionic"], ["ubuntu:nonexistent"], ["stable:"],
["test"], ["stable:stretch"], ["stable:nonexistent"], ["testscan:"]
]
for selector in selectors:
dumpSelectedSuites(apt_repos.getSuites(selector), selector)
def testSuiteProperties():
apt_repos.setAptReposBaseDir(".")
for s in sorted(apt_repos.getSuites([":"])):
print(s.getSuiteName())
print(s.getAptSuite())
print(s.getRepoUrl())
print(s.getDistsUrl())
print(s.getComponents())
print(s.hasSources())
print(s.getArchitectures())
print(s.getSourcesList())
print()
def testGetSourcesFiles():
apt_repos.setAptReposBaseDir(".")
repoSuite = list(apt_repos.getSuites(["ubuntu:trusty"]))[0]
repoSuite.scan(True)
for file in repoSuite.getSourcesFiles():
# we can't just print the absolute filename which is not diffable, so
# we print the trailing 4 parts of the path.
keep = ""
for unused_i in range(0, 7):
keep = os.path.basename(file) + ("/"
if len(keep) > 0 else "") + keep
file = os.path.dirname(file)
print(("<testfolder>/" if len(file) > 0 else "") + keep)
def getBundleRepoSuites(ids=["bundle:"], cwd=PROJECT_DIR):
'''
This method uses apt-repos to get a list of all currently available (rolled out)
bundles as a dict of ID to apt_repos.RepoSuite Objects
WARNING: This method modifies the global apt-repos base directory and therefore must
not be used in a multithreaded environment!
'''
res = dict()
apt_repos.setAptReposBaseDir(os.path.join(cwd, ".apt-repos"))
for suite in sorted(apt_repos.getSuites(ids)):
res[suite.getSuiteName()] = suite
return res
def parseSuitesStr(self, suitesStr):
'''
Parses and replaces contained placeholders in suiteStr and returns a tuple
(list, selector) where `list` is a the list of apt_repos.RepoSuite objects
matched by the resulting suite selector and `selector` is the replaced suitesStr.
'''
if not suitesStr:
return ([], suitesStr)
subst = {
"distribution": self.distribution,
"user": getpass.getuser(),
"bundle": self.bundleName
}
suitesStr = suitesStr.format(**subst)
return (sorted(apt_repos.getSuites(suitesStr.split(','))), suitesStr)
def getTargetRepoSuites(stage=None, cwd=PROJECT_DIR):
'''
This method uses apt-repos to get a list of all currently available target
repositories/suites. If `stage` is specified, only target suites for this
stage are returned. The result is a dict of ID to apt_repos.RepoSuite Objects.
WARNING: This method modifies the global apt-repos base directory and therefore must
not be used in a multithreaded environment!
'''
res = dict()
apt_repos.setAptReposBaseDir(os.path.join(cwd, ".apt-repos"))
for suite in sorted(apt_repos.getSuites(["bundle-compose-target:"])):
if not stage or "bundle-stage.{}".format(stage) in suite.getTags():
res[suite.getSuiteName()] = suite
return res
def apt_repos_query_packages(suiteSelectors, searchStrings, cwd=PROJECT_DIR):
apt_repos.setAptReposBaseDir(os.path.join(cwd, ".apt-repos"))
requestFields = PackageField.getByFieldsString("pvsaSC")
packages = []
for suite in sorted(apt_repos.getSuites(suiteSelectors)):
suite.scan(True)
packages.extend(
suite.queryPackages(searchStrings, True, None, None,
requestFields))
res = []
for package in sorted(packages):
(packageName, version, packageSuite, architecture, section,
source) = package.getData()
res.append(
common_interfaces.Package(packageName, version, packageSuite,
architecture, section, source))
return res
def apt_repos_get_suites(suiteSelectors, cwd=PROJECT_DIR):
res = []
apt_repos.setAptReposBaseDir(os.path.join(cwd, ".apt-repos"))
for suite in sorted(apt_repos.getSuites(suiteSelectors)):
res.append(common_interfaces.Suite(suite))
return res
def createTargetRepreproConfigForRepository(bundles, repoTargets, repoConfDir,
bundle_update_template,
bundle_base_update_template,
dist_template):
'''
creates the complete configuration for all suites (targets) belonging to the same repository
'''
logger.info("Creating reprepro-config at '{}' with {} suites".format(
repoConfDir, len(repoTargets)))
autogenerated = "# This file is auto-generated by '{}'. Don't edit it manually!\n".format(
progname)
update_line = {}
update_rules = dict()
# create update rules for bundles
for target in repoTargets:
for unused_bid, bundle in sorted(bundles.items()):
if not bundle.isSupposedForTarget(target):
continue
ruleName = 'update-' + bundle.getID()
keyIds = sorted(getPublicKeyIDs(target.getTrustedGPGFile()))
chunk = bundle_update_template.render(
ruleName=ruleName,
repoUrl=bundle.getRepoUrl(),
suite=bundle.getAptSuite(),
components=" ".join(bundle.getComponents()),
architectures=" ".join(bundle.getArchitectures()),
publicKeys=("!|".join(keyIds) +
"!" if len(keyIds) > 0 else ""))
update_rules[ruleName] = chunk
update_line[target] = update_line.get(target,
"") + '\n ' + ruleName
with open(
os.path.join(repoConfDir, 'distributions',
'bundle-compose_dynamic.conf'), "w") as upconf:
upconf.write(autogenerated)
for target in repoTargets:
# create update rule for bundle-base suites
updates = update_line.get(target, "")
baseDist = getBaseDist(target)
if not baseDist:
logger.warning(
"Skipping target {} as it has no 'base-dist.*' tag and no 'bundle-dist.*' tag!"
.format(target))
continue
for suite in sorted(
apt_repos.getSuites(["bundle-base.{}:".format(baseDist)])):
ruleName = "update-" + suite.getSuiteName()
keyIds = sorted(getPublicKeyIDs(suite.getTrustedGPGFile()))
updates += '\n ' + ruleName
chunk = bundle_base_update_template.render(
ruleName=ruleName,
repoUrl=suite.getRepoUrl(),
suite=suite.getAptSuite(),
components=" ".join(suite.getComponents()),
architectures=" ".join(suite.getArchitectures()),
targetDistribution=baseDist,
publicKeys=("!|".join(keyIds) +
"!" if len(keyIds) > 0 else ""))
update_rules[ruleName] = chunk
# create distribution file
logger.debug("Updating target {}".format(target))
upconf.write(
dist_template.render(
updates=updates,
suite=target.getAptSuite(),
components=" ".join(target.getComponents()),
architectures=" ".join(target.getArchitectures())))
# create updates file
with open(
os.path.join(repoConfDir, 'updates',
'bundle-compose_dynamic.conf'), "w") as upconf:
upconf.write(autogenerated)
upconf.write("".join([v for _, v in sorted(update_rules.items())]))
for root, dirs, files in os.walk(TEMPLATES_DIR):
path = os.path.relpath(root, TEMPLATES_DIR)
for f in dirs:
d = os.path.join(repoConfDir, path, f)
if not os.path.exists(d):
logger.debug("Creating dir {}".format(d))
os.mkdir(d)
assert (os.path.isdir(d))
for f in files:
srcPath = os.path.join(TEMPLATES_DIR, path, f)
if f.endswith(".skel"):
continue
elif f.endswith(".symlink"):
relPath = os.path.relpath(srcPath, repoConfDir)
name = f[:-len(".symlink")]
targetPath = os.path.join(repoConfDir, path, name)
logger.debug("Creating symlink {} --> {}".format(
relPath, targetPath))
if os.path.exists(targetPath):
os.remove(targetPath)
os.symlink(relPath, targetPath)
elif f.endswith(".once"):
targetPath = os.path.join(repoConfDir, path, f[:-5])
if not os.path.exists(targetPath):
logger.debug(
"Once creating file from template {} --> {}".format(
srcPath, targetPath))
copyfile(srcPath, targetPath)
else:
targetPath = os.path.join(repoConfDir, path, f)
logger.debug("Copying file {} --> {}".format(
srcPath, targetPath))
copyfile(srcPath, targetPath)
