def queryPackages(suiteStr, requestPackages, regexStr, archStr, componentStr, fieldStr, noUpdate=False, querySources=False): ''' queries Packages by the args provided on the command line and returns a tuple of (queryResults, requestFields) ''' suites = apt_repos.getSuites(suiteStr.split(',')) requestArchs = { a for a in archStr.split(',') } if archStr else {} requestComponents = { c for c in componentStr.split(',') } if componentStr else {} requestFields = PackageField.getByFieldsString(fieldStr) result = set() showProgress = True pp(showProgress, "{}querying packages lists for {} suites".format( "updating (use --no-update to skip) and " if not noUpdate else "", len(suites))) for x, suite in enumerate(suites): pp(showProgress, '.') try: suite.scan(not noUpdate) pp(showProgress, x+1) if not querySources: result = result.union(suite.queryPackages(requestPackages, regexStr, requestArchs, requestComponents, requestFields)) else: result = result.union(suite.querySources(requestPackages, regexStr, requestArchs, requestComponents, requestFields)) except SystemError as e: logger.warn("Could not retrieve {} for suite {}:\n{}".format("sources" if querySources else "packages", suite.getSuiteName(), e)) pp(showProgress, '\n') return (result, requestFields)
def dsc(args): ''' subcommand dsc: list urls of dsc-files available for source-packages. ''' # parse --suite and determine the specific suite scan-order suites = list() for selector in args.suite.split(','): suites.extend(sorted(apt_repos.getSuites([selector]), reverse=True)) requestPackages = { p for p in args.source } requestComponents = { c for c in args.component.split(',') } if args.component else {} showProgress = True pp(showProgress, "{}querying sources lists for {} suites".format( "updating (use --no-update to skip) and " if not args.no_update else "", len(suites))) results = {} for package in requestPackages: # pre-seed results results[package] = list() for x, suite in enumerate(suites): pp(showProgress, ".{}".format(x+1)) queryDscFiles(results, suite, requestComponents, logger, not args.no_update, args.first) if args.first and gotAllFirsts(results): break pp(showProgress, '\n') for package, urls in sorted(results.items()): for url in urls[: 1 if (args.first and len(urls) > 0) else len(urls)]: print(url)
def testQuerySources(): apt_repos.setAptReposBaseDir(".") fields = PackageField.getByFieldsString('CvsaSFB') repoSuite = list(apt_repos.getSuites(["ubuntu:trusty"]))[0] repoSuite.scan(True) res = repoSuite.querySources(['git'], False, None, None, fields) for qr in sorted(res): print(qr)
def __init__(self, suiteName, packages): suites = sorted(apt_repos.getSuites([suiteName])) if len(suites) != 1: raise BundleError( "Can't create UpdateRule for suite selector '{}' since it doesn't select exactly one suite." .format(suiteName)) self.suite = suites[0] self.packages = packages
def suites(args): ''' subcommand suites: print a list of registered suites ''' suites = apt_repos.getSuites(args.suite.split(',')) for s in sorted(suites): print("# {}{}".format(s.getSuiteName(), (" [" + (":, ".join(sorted(s.getTags())) + ":]")) if len(s.getTags()) > 0 else "")) if args.verbose: print(s.getSourcesList() + "\n")
def testSuiteSelectors(): apt_repos.setAptReposBaseDir(".") selectors = [ None, [":"], ["default:"], ["ubuntu:xenial"], ["xenial"], ["ub:trusty"], ["ubuntu:"], ["u:"], ["u:trusty"], ["ubuntu:trusty-security", "ubuntu:xenial-security"], ["debian:"], ["ubuntu:"], ["ubuntu:de-bionic"], ["ubuntu:nonexistent"], ["stable:"], ["test"], ["stable:stretch"], ["stable:nonexistent"], ["testscan:"] ] for selector in selectors: dumpSelectedSuites(apt_repos.getSuites(selector), selector)
def testSuiteProperties(): apt_repos.setAptReposBaseDir(".") for s in sorted(apt_repos.getSuites([":"])): print(s.getSuiteName()) print(s.getAptSuite()) print(s.getRepoUrl()) print(s.getDistsUrl()) print(s.getComponents()) print(s.hasSources()) print(s.getArchitectures()) print(s.getSourcesList()) print()
def testGetSourcesFiles(): apt_repos.setAptReposBaseDir(".") repoSuite = list(apt_repos.getSuites(["ubuntu:trusty"]))[0] repoSuite.scan(True) for file in repoSuite.getSourcesFiles(): # we can't just print the absolute filename which is not diffable, so # we print the trailing 4 parts of the path. keep = "" for unused_i in range(0, 7): keep = os.path.basename(file) + ("/" if len(keep) > 0 else "") + keep file = os.path.dirname(file) print(("<testfolder>/" if len(file) > 0 else "") + keep)
def getBundleRepoSuites(ids=["bundle:"], cwd=PROJECT_DIR): ''' This method uses apt-repos to get a list of all currently available (rolled out) bundles as a dict of ID to apt_repos.RepoSuite Objects WARNING: This method modifies the global apt-repos base directory and therefore must not be used in a multithreaded environment! ''' res = dict() apt_repos.setAptReposBaseDir(os.path.join(cwd, ".apt-repos")) for suite in sorted(apt_repos.getSuites(ids)): res[suite.getSuiteName()] = suite return res
def parseSuitesStr(self, suitesStr): ''' Parses and replaces contained placeholders in suiteStr and returns a tuple (list, selector) where `list` is a the list of apt_repos.RepoSuite objects matched by the resulting suite selector and `selector` is the replaced suitesStr. ''' if not suitesStr: return ([], suitesStr) subst = { "distribution": self.distribution, "user": getpass.getuser(), "bundle": self.bundleName } suitesStr = suitesStr.format(**subst) return (sorted(apt_repos.getSuites(suitesStr.split(','))), suitesStr)
def getTargetRepoSuites(stage=None, cwd=PROJECT_DIR): ''' This method uses apt-repos to get a list of all currently available target repositories/suites. If `stage` is specified, only target suites for this stage are returned. The result is a dict of ID to apt_repos.RepoSuite Objects. WARNING: This method modifies the global apt-repos base directory and therefore must not be used in a multithreaded environment! ''' res = dict() apt_repos.setAptReposBaseDir(os.path.join(cwd, ".apt-repos")) for suite in sorted(apt_repos.getSuites(["bundle-compose-target:"])): if not stage or "bundle-stage.{}".format(stage) in suite.getTags(): res[suite.getSuiteName()] = suite return res
def apt_repos_query_packages(suiteSelectors, searchStrings, cwd=PROJECT_DIR): apt_repos.setAptReposBaseDir(os.path.join(cwd, ".apt-repos")) requestFields = PackageField.getByFieldsString("pvsaSC") packages = [] for suite in sorted(apt_repos.getSuites(suiteSelectors)): suite.scan(True) packages.extend( suite.queryPackages(searchStrings, True, None, None, requestFields)) res = [] for package in sorted(packages): (packageName, version, packageSuite, architecture, section, source) = package.getData() res.append( common_interfaces.Package(packageName, version, packageSuite, architecture, section, source)) return res
def apt_repos_get_suites(suiteSelectors, cwd=PROJECT_DIR): res = [] apt_repos.setAptReposBaseDir(os.path.join(cwd, ".apt-repos")) for suite in sorted(apt_repos.getSuites(suiteSelectors)): res.append(common_interfaces.Suite(suite)) return res
def createTargetRepreproConfigForRepository(bundles, repoTargets, repoConfDir, bundle_update_template, bundle_base_update_template, dist_template): ''' creates the complete configuration for all suites (targets) belonging to the same repository ''' logger.info("Creating reprepro-config at '{}' with {} suites".format( repoConfDir, len(repoTargets))) autogenerated = "# This file is auto-generated by '{}'. Don't edit it manually!\n".format( progname) update_line = {} update_rules = dict() # create update rules for bundles for target in repoTargets: for unused_bid, bundle in sorted(bundles.items()): if not bundle.isSupposedForTarget(target): continue ruleName = 'update-' + bundle.getID() keyIds = sorted(getPublicKeyIDs(target.getTrustedGPGFile())) chunk = bundle_update_template.render( ruleName=ruleName, repoUrl=bundle.getRepoUrl(), suite=bundle.getAptSuite(), components=" ".join(bundle.getComponents()), architectures=" ".join(bundle.getArchitectures()), publicKeys=("!|".join(keyIds) + "!" if len(keyIds) > 0 else "")) update_rules[ruleName] = chunk update_line[target] = update_line.get(target, "") + '\n ' + ruleName with open( os.path.join(repoConfDir, 'distributions', 'bundle-compose_dynamic.conf'), "w") as upconf: upconf.write(autogenerated) for target in repoTargets: # create update rule for bundle-base suites updates = update_line.get(target, "") baseDist = getBaseDist(target) if not baseDist: logger.warning( "Skipping target {} as it has no 'base-dist.*' tag and no 'bundle-dist.*' tag!" .format(target)) continue for suite in sorted( apt_repos.getSuites(["bundle-base.{}:".format(baseDist)])): ruleName = "update-" + suite.getSuiteName() keyIds = sorted(getPublicKeyIDs(suite.getTrustedGPGFile())) updates += '\n ' + ruleName chunk = bundle_base_update_template.render( ruleName=ruleName, repoUrl=suite.getRepoUrl(), suite=suite.getAptSuite(), components=" ".join(suite.getComponents()), architectures=" ".join(suite.getArchitectures()), targetDistribution=baseDist, publicKeys=("!|".join(keyIds) + "!" if len(keyIds) > 0 else "")) update_rules[ruleName] = chunk # create distribution file logger.debug("Updating target {}".format(target)) upconf.write( dist_template.render( updates=updates, suite=target.getAptSuite(), components=" ".join(target.getComponents()), architectures=" ".join(target.getArchitectures()))) # create updates file with open( os.path.join(repoConfDir, 'updates', 'bundle-compose_dynamic.conf'), "w") as upconf: upconf.write(autogenerated) upconf.write("".join([v for _, v in sorted(update_rules.items())])) for root, dirs, files in os.walk(TEMPLATES_DIR): path = os.path.relpath(root, TEMPLATES_DIR) for f in dirs: d = os.path.join(repoConfDir, path, f) if not os.path.exists(d): logger.debug("Creating dir {}".format(d)) os.mkdir(d) assert (os.path.isdir(d)) for f in files: srcPath = os.path.join(TEMPLATES_DIR, path, f) if f.endswith(".skel"): continue elif f.endswith(".symlink"): relPath = os.path.relpath(srcPath, repoConfDir) name = f[:-len(".symlink")] targetPath = os.path.join(repoConfDir, path, name) logger.debug("Creating symlink {} --> {}".format( relPath, targetPath)) if os.path.exists(targetPath): os.remove(targetPath) os.symlink(relPath, targetPath) elif f.endswith(".once"): targetPath = os.path.join(repoConfDir, path, f[:-5]) if not os.path.exists(targetPath): logger.debug( "Once creating file from template {} --> {}".format( srcPath, targetPath)) copyfile(srcPath, targetPath) else: targetPath = os.path.join(repoConfDir, path, f) logger.debug("Copying file {} --> {}".format( srcPath, targetPath)) copyfile(srcPath, targetPath)