def __init__(self, logger=None, debug=False): if logger: self.logger = logger else: self.logger = logging self.debug = debug self.config = configuration.GetConfig() self.rest_client = rest.RestClient( pkgdb_url=self.config.get('rest', 'pkgdb'), releases_url=self.config.get('rest', 'releases'))
def __init__(self, debug=False): self.debug = debug config = configuration.GetConfig() username, password = rest.GetUsernameAndPassword() self.rest_client = rest.RestClient( pkgdb_url=config.get('rest', 'pkgdb'), releases_url=config.get('rest', 'releases'), username=username, password=password, debug=debug)
def __init__(self, srv4_pkg, stats_basedir=None, md5sum=None, debug=False): super(PackageStats, self).__init__() self.srv4_pkg = srv4_pkg self.md5sum = md5sum self.dir_format_pkg = None self.all_stats = {} self.db_pkg_stats = None config = configuration.GetConfig() self.rest_client = rest.RestClient( pkgdb_url=config.get('rest', 'pkgdb'), releases_url=config.get('rest', 'releases'))
def main(): oldcatrel, newcatrel, arch, osrel = GetCLIOptions() config = configuration.GetConfig() username, password = rest.GetUsernameAndPassword() rest_client = rest.RestClient(pkgdb_url=config.get('rest', 'pkgdb'), releases_url=config.get('rest', 'releases'), username=username, password=password) removelst, rebuildlst, obsolete = (ComputeRemoveAndRebuild( oldcatrel, newcatrel, arch, osrel, rest_client)) WriteToTextFiles(removelst, rebuildlst, newcatrel, arch, osrel, obsolete)
def __init__(self, binary_path, debug=False): self.debug = debug self._binary_path = binary_path self.config = configuration.GetConfig() username, password = rest.GetUsernameAndPassword() self.rest_client = rest.RestClient( pkgdb_url=self.config.get('rest', 'pkgdb'), releases_url=self.config.get('rest', 'releases'), username=username, password=password) fd = open(self._binary_path, 'rb') self._mmap = mmap.mmap(fd.fileno(), 0, access=mmap.PROT_READ) self._elffile = ELFFile(self._mmap)
def __init__(self, osrel, arch, debug=False): super(InstallContentsImporter, self).__init__() self.osrel = osrel self.arch = arch self.pkginst_cache = {} config = configuration.GetConfig() username, password = rest.GetUsernameAndPassword() self.rest_client = rest.RestClient( pkgdb_url=config.get('rest', 'pkgdb'), releases_url=config.get('rest', 'releases'), username=username, password=password, debug=debug)
def __init__(self, catrel, arch, osrel, fn_ts, gen_catalog_bin, chkcat="/opt/csw/bin/chkcat", cattiming_class=CatalogTiming, tsrecord_class=TimestampRecord): """Constructor. "fn_ts" is the path name to the time stamp file. "gen_catalog_bin" is the path to the gen-catalog-index binary "chkcat" is the path to `chkcat'. By default `/opt/csw/bin/chkcat'. "cattiming_class" and "tsrecord_class" are mainly used for unit tests in order to provide classes with overridden methods. """ config = configuration.GetConfig() username, password = GetUsernameAndPassword() self.rest_client = RestClient(pkgdb_url=config.get('rest', 'pkgdb'), releases_url=config.get( 'rest', 'releases'), username=username, password=password) self.__catalogfgen = GenerateCatalog(catrel, arch, osrel, gen_catalog_bin) self.__chkcat = chkcat # store for later use self._catrel = catrel self._arch = arch self._osrel = osrel # will be set by __enter__ and unset by __exit__ self.tmpdir = None self.__tsrecord_class = tsrecord_class self.__cattiming_class = cattiming_class self.__timestamp_record = tsrecord_class(fn_ts)
def __init__(self, name, sqo_pkgs_list, osrel, arch, catrel, debug=False, show_progress=False): super(CheckpkgManagerBase, self).__init__() self.debug = debug self.name = name self.sqo_pkgs_list = sqo_pkgs_list self.osrel = osrel self.arch = arch self.catrel = catrel self.show_progress = show_progress self._ResetState() self.individual_checks = [] self.set_checks = [] config = configuration.GetConfig() username, password = rest.GetUsernameAndPassword() self.rest_client = rest.RestClient( pkgdb_url=config.get('rest', 'pkgdb'), releases_url=config.get('rest', 'releases'), username=username, password=password)
def main(): parser = optparse.OptionParser() parser.add_option("-c", "--catalogname", dest="catalogname") parser.add_option("--catalog-release", dest="catrel", default="kiel") parser.add_option("--arch", dest="arch", default="sparc") parser.add_option("--os-release", dest="osrel", default="SunOS5.9") parser.add_option("--md5", dest="md5") parser.add_option("--debug", dest="debug", default=False, action="store_true") options, args = parser.parse_args() if options.debug: logging.basicConfig(level=logging.DEBUG) else: logging.basicConfig(level=logging.INFO) config = configuration.GetConfig() rest_client = rest.RestClient(pkgdb_url=config.get('rest', 'pkgdb'), releases_url=config.get('rest', 'releases')) if options.md5: md5 = options.md5 else: pkg_basic = rest_client.Srv4ByCatalogAndCatalogname( options.catrel, options.arch, options.osrel, options.catalogname) md5 = pkg_basic["md5_sum"] logging.info("Fetching %s", md5) pkgstats = rest_client.GetBlob('pkgstats', md5) # getting elfdump info elfdump_data = {} for _, binary_md5 in pkgstats["binary_md5_sums"]: elfdump_data[binary_md5] = rest_client.GetBlob('elfdump', binary_md5) catalogname = pkgstats["basic_stats"]["catalogname"] target_file = "lib/python/testdata/%s_stats.py" % catalogname pkgstats['elfdump_info'] = elfdump_data with open(target_file, "wb") as fd: fd.write("# Generated by running:\n") fd.write("# %s\n\n" % ' '.join(pipes.quote(x) for x in sys.argv)) fd.write("pkgstats = [") fd.write(pprint.pformat(pkgstats)) fd.write("]\n\n") print target_file, "done"
def main(): parser = optparse.OptionParser(USAGE) parser.add_option("-c", "--catalogname", dest="catalogname", help='the name of the package in catalog') parser.add_option("--os-releases", dest="os_releases", help=("Comma separated OS releases, e.g. " "SunOS5.9,SunOS5.10")) parser.add_option("--debug", dest="debug", action="store_true") parser.add_option("--dry-run", dest="dry_run", default=False, action="store_true", help=("Don't apply changes (no REST calls).")) options, args = parser.parse_args() logging_level = logging.INFO if options.debug: logging_level = logging.DEBUG fmt = '%(levelname)s %(asctime)s %(filename)s:%(lineno)d %(message)s' logging.basicConfig(format=fmt, level=logging_level) if not options.catalogname: logging.error('option catalogname required \n%s', USAGE) sys.exit(1) os_releases = common_constants.OS_RELS if options.os_releases: os_releases = options.os_releases.split(",") config = configuration.GetConfig() username, password = rest.GetUsernameAndPassword() rest_client = rest.RestClient(pkgdb_url=config.get('rest', 'pkgdb'), releases_url=config.get('rest', 'releases'), username=username, password=password) pr = PackageRemover(rest_client) pr.RemovePackage(options.catalogname, not options.dry_run, os_releases)
def __init__(self, pkg_path, debug): self.debug = debug self.pkg_path = pkg_path self._work_dir = None self._admin_file = None self._gunzipped_path = None self._md5_sum = None self._stat = None self._mtime = None self._transformed = False self._pkgname = None self._pkginfo_dict = None self._dir_format_base_dir = None self._files_metadata = None self._binaries = None self._file_paths = None self.config = configuration.GetConfig() username, password = rest.GetUsernameAndPassword() self.rest_client = rest.RestClient( pkgdb_url=self.config.get('rest', 'pkgdb'), releases_url=self.config.get('rest', 'releases'), username=username, password=password)
def main(): parser = optparse.OptionParser() parser.add_option("--debug", dest="debug", default=False, action="store_true") parser.add_option("--catalog-release", dest="catrel", default="unstable") parser.add_option("--arch", dest="arch", default="i386") parser.add_option("--os-release", dest="osrel", default="SunOS5.10") parser.add_option("--md5", dest="md5") parser.add_option("--check-name", dest="check_name") options, args = parser.parse_args() if options.debug: logging.basicConfig(level=logging.DEBUG) else: logging.basicConfig(level=logging.INFO) configuration.SetUpSqlobjectConnection() osrel = options.osrel catrel = options.catrel arch = options.arch md5 = options.md5 config = configuration.GetConfig() rest_client = rest.RestClient(pkgdb_url=config.get('rest', 'pkgdb'), releases_url=config.get('rest', 'releases')) logging.info("Fetching %s", md5) pkgstats = rest_client.GetBlob('pkgstats', md5) catalog = checkpkg_lib.Catalog() check_function = getattr(package_checks, options.check_name) examined_files_by_pkg = {} check_interface = LoggingCheckInterface(osrel, arch, catrel, catalog, examined_files_by_pkg) messenger = checkpkg_lib.CheckpkgMessenger() pkgstats['elfdump_info'] = checkpkg_lib.LazyElfinfo(rest_client) check_function([pkgstats], check_interface, logger=logging, messenger=messenger)
import re import web from sqlobject import sqlbuilder # sys.stderr.write("Running Python binary from {}.\n".format(sys.executable)) # sys.stderr.write("Python path is {}.\n".format(sys.path)) from lib.python import checkpkg_lib from lib.python import configuration from lib.python import models from lib.python import representations from lib.web import web_lib config = configuration.GetConfig() LOG_FILE_TMPL = config.get("pkgdb_app", "log_file") urls_html = ( r'/', 'index', r'/favicon\.ico', 'Favicon', r'/srv4/', 'Srv4List', r'/srv4/([0-9a-f]{32})/', 'Srv4Detail', r'/srv4/([0-9a-f]{32})/files/', 'Srv4DetailFiles', r'/srv4/([0-9a-f]{32})/struct-dump/', 'Srv4StructDump', r'/elfdump/([0-9a-f]{32})/', 'Elfdump', r'/catalogs/', 'CatalogList', r'/catalogs/([\w-]+)-(sparc|i386)-(SunOS[^-]+)/', 'CatalogDetail', r'/maintainers/', 'MaintainerList', r'/maintainers/(\d+)/', 'MaintainerDetail', r'/maintainers/(\d+)/checkpkg/', 'MaintainerCheckpkgReport',
def main(): parser = optparse.OptionParser(USAGE) parser.add_option("-d", "--debug", dest="debug", action="store_true", default=False, help="Switch on debugging messages") parser.add_option("-q", "--quiet", dest="quiet", action="store_true", default=False, help="Display less messages") parser.add_option( "--catalog-release", dest="catrel", default="current", help="A catalog release: current, unstable, testing, stable.") parser.add_option( "-r", "--os-releases", dest="osrel_commas", help=("Comma separated list of ['SunOS5.9', 'SunOS5.10'], " "e.g. 'SunOS5.9,SunOS5.10'.")) parser.add_option("-a", "--catalog-architecture", dest="arch", help="Architecture: i386, sparc.") parser.add_option("--profile", dest="profile", default=False, action="store_true", help="Enable profiling (a developer option).") options, args = parser.parse_args() assert len(args), "The list of files or md5 sums must be not empty." logging_level = logging.INFO if options.quiet: logging_level = logging.WARNING elif options.debug: # If both flags are set, debug wins. logging_level = logging.DEBUG fmt = '%(levelname)s %(asctime)s %(filename)s:%(lineno)d %(message)s' logging.basicConfig(format=fmt, level=logging_level) logging.debug("Starting.") configuration.SetUpSqlobjectConnection() err_msg_list = [] if not options.osrel_commas: err_msg_list.append("Please specify --os-releases.") if not options.arch: err_msg_list.append("Please specify --catalog-architecture.") if options.arch not in cc.PHYSICAL_ARCHITECTURES: err_msg_list.append( "Valid --catalog-architecture values are: %s, you passed: %r" % (cc.PHYSICAL_ARCHITECTURES, options.arch)) if err_msg_list: raise UsageError(" ".join(err_msg_list)) md5_sums_from_files = [] collector = package_stats.StatsCollector(logger=logging, debug=options.debug) # We need to separate files and md5 sums. md5_sums, file_list = [], [] for arg in args: if struct_util.IsMd5(arg): md5_sums.append(arg) else: file_list.append(arg) config = configuration.GetConfig() username, password = rest.GetUsernameAndPassword() rest_client = rest.RestClient(pkgdb_url=config.get('rest', 'pkgdb'), releases_url=config.get('rest', 'releases'), username=username, password=password) if file_list: def MakeEntry(file_name): file_hash = hashlib.md5() with open(file_name, "r") as fd: chunk_size = 2 * 1024 * 1024 data = fd.read(chunk_size) while data: file_hash.update(data) data = fd.read(chunk_size) md5_sum = file_hash.hexdigest() del file_hash _, file_basename = os.path.split(file_name) return { 'pkg_path': file_name, 'md5sum': md5_sum, 'file_basename': file_basename, } entries = [MakeEntry(x) for x in file_list] md5_sums_from_files = collector.CollectStatsFromCatalogEntries( entries, False) for md5_sum in md5_sums_from_files: if not rest_client.IsRegisteredLevelOne(md5_sum): rest_client.RegisterLevelOne(md5_sum) # We need the md5 sums of these files md5_sums.extend(md5_sums_from_files) assert md5_sums, "The list of md5 sums must not be empty." logging.debug("md5_sums: %s", md5_sums) osrel_list = options.osrel_commas.split(",") logging.debug("Reading packages data from the database.") # This part might need improvements in order to handle a whole # catalog. On the other hand, if we already have the whole catalog in # the database, we can do it altogether differently. # Transforming the result to a list in order to force object # retrieval. sqo_pkgs = list( models.Srv4FileStats.select( sqlobject.IN(models.Srv4FileStats.q.md5_sum, md5_sums))) tags_for_all_osrels = [] try: sqo_catrel = models.CatalogRelease.selectBy( name=options.catrel).getOne() except sqlobject.main.SQLObjectNotFound as e: logging.fatal("Fetching from the db has failed: catrel=%s", repr(str(options.catrel))) logging.fatal("Available catalog releases:") sqo_catrels = models.CatalogRelease.select() for sqo_catrel in sqo_catrels: logging.fatal(" - %s", sqo_catrel.name) raise sqo_arch = models.Architecture.selectBy(name=options.arch).getOne() for osrel in osrel_list: sqo_osrel = models.OsRelease.selectBy(short_name=osrel).getOne() VerifyContents(sqo_osrel, sqo_arch) check_manager = checkpkg_lib.CheckpkgManager2( CHECKPKG_MODULE_NAME, sqo_pkgs, osrel, options.arch, options.catrel, debug=options.debug, show_progress=(os.isatty(1) and not options.quiet)) # Running the checks, reporting and exiting. exit_code, screen_report, tags_report = check_manager.Run() screen_report = unicode(screen_report) if not options.quiet and screen_report: # TODO: Write this to screen only after overrides are applied. sys.stdout.write(screen_report) else: logging.debug("No screen report.") overrides_list = [list(pkg.GetOverridesResult()) for pkg in sqo_pkgs] override_list = reduce(operator.add, overrides_list) args = (sqo_osrel, sqo_arch, sqo_catrel) tag_lists = [list(pkg.GetErrorTagsResult(*args)) for pkg in sqo_pkgs] error_tags = reduce(operator.add, tag_lists) (tags_after_overrides, unapplied_overrides) = overrides.ApplyOverrides( error_tags, override_list) tags_for_all_osrels.extend(tags_after_overrides) if not options.quiet: if tags_after_overrides: print(textwrap.fill(BEFORE_OVERRIDES, 80)) for checkpkg_tag in tags_after_overrides: print checkpkg_tag.ToGarSyntax() print for paragraph in AFTER_OVERRIDES: print(textwrap.fill(paragraph, 80)) print elif error_tags: msg = ( 'Fair enough, there were %d error tags, ' 'but they were all overridden. ' "Just make sure you didn't override anything silly, like " 'sparc binaries in a i386 package.' % len(error_tags)) print else: print('Jolly good! All checks passed, no error tags reported.') if unapplied_overrides: print textwrap.fill(UNAPPLIED_OVERRIDES, 80) for override in unapplied_overrides: print u"* Unused %s" % override exit_code = bool(tags_for_all_osrels) sys.exit(exit_code)
def main(): parser = optparse.OptionParser(USAGE) parser.add_option("-d", "--debug", dest="debug", default=False, action="store_true", help="Turn on debugging messages") parser.add_option( "-t", "--pkg-review-template", dest="pkg_review_template", help="A Cheetah template used for package review reports.") parser.add_option("-r", "--os-release", dest="osrel", default="SunOS5.10", help="E.g. SunOS5.10") parser.add_option("-a", "--arch", dest="arch", default="sparc", help="'i386' or 'sparc'") parser.add_option("-c", "--catalog-release", dest="catrel", default="unstable", help="E.g. unstable, dublin") parser.add_option("--replace", dest="replace", default=False, action="store_true", help="Replace packages when importing (importpkg)") parser.add_option("--profile", dest="profile", default=False, action="store_true", help="Turn on profiling") parser.add_option("--force-unpack", dest="force_unpack", default=False, action="store_true", help="Force unpacking of packages") options, args = parser.parse_args() logging_level = logging.INFO if options.debug: logging_level = logging.DEBUG fmt = '%(levelname)s %(asctime)s %(filename)s:%(lineno)d %(message)s' logging.basicConfig(format=fmt, level=logging_level) if not args: raise UsageError("Please specify a command. See --help.") # SetUpSqlobjectConnection needs to be called after # logging.basicConfig configuration.SetUpSqlobjectConnection() command = args[0] args = args[1:] if command == 'show': subcommand = args[0] args = args[1:] elif command == 'pkg': subcommand = args[0] args = args[1:] else: subcommand = None md5_sums = args if (command, subcommand) == ('show', 'errors'): for md5_sum in md5_sums: srv4 = GetPkg(md5_sum) res = m.CheckpkgErrorTag.select( m.CheckpkgErrorTag.q.srv4_file == srv4) for row in res: print 'overridden' if row.overridden else 'active', print row.pkgname, row.tag_name, row.tag_info, row.catrel.name, row.arch.name, print row.os_rel.short_name elif (command, subcommand) == ('show', 'overrides'): for md5_sum in md5_sums: srv4 = GetPkg(md5_sum) res = m.CheckpkgOverride.select( m.CheckpkgOverride.q.srv4_file == srv4) for row in res: print row.pkgname, row.tag_name, row.tag_info elif (command, subcommand) == ('show', 'pkg'): for md5_sum in md5_sums: srv4 = GetPkg(md5_sum) t = Template(SHOW_PKG_TMPL, searchList=[srv4]) sys.stdout.write(unicode(t)) elif command == 'gen-html': config = configuration.GetConfig() username, password = rest.GetUsernameAndPassword() rest_client = rest.RestClient(pkgdb_url=config.get('rest', 'pkgdb'), releases_url=config.get( 'rest', 'releases'), username=username, password=password, debug=options.debug) g = HtmlGenerator(md5_sums, options.pkg_review_template, rest_client, options.debug) sys.stdout.write(g.GenerateHtml()) elif command == 'initdb': config = configuration.GetConfig() database.InitDB(config) elif command == 'importpkg': collector = package_stats.StatsCollector(logger=logging, debug=options.debug) file_list = args catalog_entries = [] for file_name in file_list: file_hash = hashlib.md5() chunk_size = 2 * 1024 * 1024 with open(file_name, 'rb') as fd: data = fd.read(chunk_size) while data: file_hash.update(data) data = fd.read(chunk_size) data_md5_sum = file_hash.hexdigest() catalog_entry = { 'md5sum': data_md5_sum, 'file_basename': os.path.basename(file_name), 'pkg_path': file_name, } catalog_entries.append(catalog_entry) md5_list = collector.CollectStatsFromCatalogEntries( catalog_entries, force_unpack=options.force_unpack) config = configuration.GetConfig() rest_client = rest.RestClient(pkgdb_url=config.get('rest', 'pkgdb'), releases_url=config.get( 'rest', 'releases'), debug=options.debug) for md5_sum in md5_list: logging.debug("Importing %s", md5_sum) rest_client.RegisterLevelTwo(md5_sum) elif command == 'removepkg': for md5_sum in md5_sums: srv4 = GetPkg(md5_sum) in_catalogs = list(srv4.in_catalogs) if in_catalogs: for in_catalog in in_catalogs: logging.warning("%s", in_catalog) logging.warning( "Not removing from the database, because the package " "in question is part of at least one catalog.") else: logging.info("Removing %s", srv4) srv4.DeleteAllDependentObjects() srv4.destroySelf() elif command == 'add-to-cat': if len(args) < 4: raise UsageError("Not enough arguments, see usage.") user = getpass.getuser() osrel, arch, catrel = args[:3] username, password = rest.GetUsernameAndPassword() rest_client = rest.RestClient(username=username, password=password) md5_sums = args[3:] for md5_sum in md5_sums: rest_client.AddSvr4ToCatalog(catrel, arch, osrel, md5_sum) elif command == 'del-from-cat': if len(args) < 4: raise UsageError("Not enough arguments, see usage.") osrel, arch, catrel = args[:3] md5_sums = args[3:] username, password = rest.GetUsernameAndPassword() rest_client = rest.RestClient(username=username, password=password) for md5_sum in md5_sums: rest_client.RemoveSvr4FromCatalog(catrel, arch, osrel, md5_sum) elif command == 'system-metadata-to-disk': logging.debug("Args: %s", args) outfile = None infile_contents = common_constants.DEFAULT_INSTALL_CONTENTS_FILE infile_pkginfo = None osrel, arch = (None, None) if len(args) >= 2: infile_contents = args[0] infile_pkginfo = args[1] if len(args) >= 3: outfile = args[2] if len(args) >= 4: if len(args) == 5: osrel, arch = args[3:5] else: raise UsageError("Wrong number of arguments (%s), see usage." % len(args)) spi = system_pkgmap.Indexer(outfile, infile_contents, infile_pkginfo, osrel, arch) spi.IndexAndSave() elif command == 'import-system-metadata': if len(args) < 2: raise UsageError( "Usage: ... import-system-metadata <osrel> <arch>") osrel = args[0] arch = args[1] importer = system_pkgmap.InstallContentsImporter(osrel, arch, debug=options.debug) importer.Import(show_progress=(not options.debug)) elif (command, subcommand) == ('pkg', 'search'): logging.debug("Searching for %s", args) sqo_osrel = m.OsRelease.selectBy(short_name=options.osrel).getOne() sqo_arch = m.Architecture.selectBy(name=options.arch).getOne() sqo_catrel = m.CatalogRelease.selectBy(name=options.catrel).getOne() if len(args) < 1: logging.fatal("Wrong number of arguments: %s", len(args)) raise SystemExit for catalogname in args: join = [ sqlbuilder.INNERJOINOn( None, m.Srv4FileInCatalog, m.Srv4FileInCatalog.q.srv4file == m.Srv4FileStats.q.id), ] res = m.Srv4FileStats.select( sqlobject.AND( m.Srv4FileInCatalog.q.osrel == sqo_osrel, m.Srv4FileInCatalog.q.arch == sqo_arch, m.Srv4FileInCatalog.q.catrel == sqo_catrel, m.Srv4FileStats.q.catalogname.contains(catalogname), m.Srv4FileStats.q.use_to_generate_catalogs == True), join=join, ).orderBy("catalogname") for sqo_srv4 in res: print "%s %s" % (sqo_srv4.basename, sqo_srv4.md5_sum) elif command == 'sync-cat-from-file': if len(args) != 4: raise UsageError("Wrong number of arguments, see usage.") osrel, arch, catrel, catalog_file = args ci = CatalogImporter(debug=options.debug) ci.SyncFromCatalogFile(osrel, arch, catrel, catalog_file) elif command == 'sync-catalogs-from-tree': if len(args) != 2: raise UsageError("Wrong number of arguments, see usage.") ci = CatalogImporter(debug=options.debug) catrel, base_dir = args ci.SyncFromCatalogTree(catrel, base_dir, options.force_unpack) elif (command, subcommand) == ('show', 'cat'): sqo_osrel, sqo_arch, sqo_catrel = m.GetSqoTriad( options.osrel, options.arch, options.catrel) res = m.GetCatPackagesResult(sqo_osrel, sqo_arch, sqo_catrel) for obj in res: print obj.catalogname, obj.basename, obj.md5_sum elif (command, subcommand) == ('show', 'files'): md5_sum = args[0] join = [ sqlbuilder.INNERJOINOn( None, m.Srv4FileStats, m.CswFile.q.srv4_file == m.Srv4FileStats.q.id), ] res = m.CswFile.select( m.Srv4FileStats.q.md5_sum == md5_sum, join=join, ) for obj in res: print os.path.join(obj.path, obj.basename) elif (command, subcommand) == ('show', 'basename'): db_catalog = checkpkg_lib.Catalog() for arg in args: pkgs_by_path = db_catalog.GetPathsAndPkgnamesByBasename( arg, options.osrel, options.arch, options.catrel) for file_path in pkgs_by_path: print os.path.join(file_path, arg), ", ".join(pkgs_by_path[file_path]) elif (command, subcommand) == ('show', 'filename'): db_catalog = checkpkg_lib.Catalog() for arg in args: pkgs = db_catalog.GetPkgByPath(arg, options.osrel, options.arch, options.catrel) print " ".join(pkgs) else: raise UsageError("Command unrecognized: %s" % command)
def main(): parser = optparse.OptionParser() parser.add_option( "--from-catalog", dest="catrel_from", default="unstable", help="Catalog release to integrate from, e.g. 'unstable'.") parser.add_option("--to-catalog", dest="catrel_to", default="testing", help="Catalog release to integrate to, e.g. 'testing'.") parser.add_option( "--from-json", dest="from_json", help=("If specified, loads data from a JSON file instead of polling " "the database.")) parser.add_option("--save-json", dest="save_json", help="If specified, saves JSON data to a file.") parser.add_option("-o", "--output-file", dest="output_file", help="Filename to save output to.") parser.add_option("--no-include-downgrades", dest="include_downgrades", default=True, action="store_false", help="Skip package downgrades.") parser.add_option( "--no-include-version-changes", dest="include_version_changes", default=True, action="store_false", help="Skip version upgrades (only accept revision upgrades).") parser.add_option("--debug", dest="debug", default=False, action="store_true") options, args = parser.parse_args() logging_level = logging.INFO if options.debug: logging_level = logging.DEBUG fmt = '%(levelname)s %(asctime)s %(filename)s:%(lineno)d %(message)s' logging.basicConfig(format=fmt, level=logging_level) config = configuration.GetConfig() username, password = rest.GetUsernameAndPassword() rest_client = rest.RestClient(pkgdb_url=config.get('rest', 'pkgdb'), releases_url=config.get('rest', 'releases'), username=username, password=password) if not options.output_file: raise UsageError("Please specify the output file. See --help.") catrel_from = options.catrel_from catrel_to = options.catrel_to if options.from_json: with open(options.from_json, "rb") as fd: logging.info("Loading %s", options.from_json) (bundles_by_md5, jsonable_catalogs, diffs_by_catalogname) = cjson.decode(fd.read()) catalogs = dict((tuple(cjson.decode(x)), jsonable_catalogs[x]) for x in jsonable_catalogs) else: catalogs = GetCatalogs(catrel_from, catrel_to, options.include_version_changes, options.include_downgrades, rest_client) diffs_by_catalogname = ComposeDiffsByCatalogname( catalogs, catrel_from, catrel_to, options.include_version_changes, options.include_downgrades) bundles_by_md5 = {} bundles_missing = set() cp = rest.CachedPkgstats("pkgstats", rest_client) for key in catalogs: if catalogs[key]: # could be None for pkg in catalogs[key]: md5 = pkg["md5_sum"] if md5 not in bundles_by_md5 and md5 not in bundles_missing: stats = cp.GetPkgstats(md5) bundle_key = "OPENCSW_BUNDLE" if stats: if bundle_key in stats["pkginfo"]: bundles_by_md5[md5] = stats["pkginfo"][ bundle_key] else: logging.debug( "%r (%r) does not have the bundle set", stats["basic_stats"]["pkg_basename"], md5) bundles_missing.add(md5) # Here's a good place to calculate the mapping between catalognames and # bundle names. change_types = "new_pkgs", "removed_pkgs", "updated_pkgs" bundles_by_catalogname = {} for catalogname in diffs_by_catalogname: l = bundles_by_catalogname.setdefault(catalogname, set()) for change_type in change_types: if change_type in diffs_by_catalogname[catalogname]: for change_info in diffs_by_catalogname[catalogname][ change_type]: pkg = change_info[2] if "to" in pkg: md5s = [x["md5_sum"] for x in (pkg["from"], pkg["to"])] else: md5s = [pkg["md5_sum"]] for md5 in md5s: if md5 in bundles_by_md5: l.add(bundles_by_md5[md5]) namespace = { "bundles_by_catalogname": bundles_by_catalogname, "bundles_by_md5": bundles_by_md5, "diffs_by_catalogname": diffs_by_catalogname, "catrel_to": catrel_to, "catrel_from": catrel_from, "prog": sys.argv[0], } if options.save_json: with open(options.save_json, "wb") as fd: jsonable_catalogs = dict( (cjson.encode(x), catalogs[x]) for x in catalogs) fd.write( cjson.encode( (bundles_by_md5, jsonable_catalogs, diffs_by_catalogname))) t = Template.Template(CATALOG_MOD_TMPL, searchList=[namespace]) if options.output_file: logging.info("Saving output to %s", options.output_file) with open(options.output_file, "wb") as fd: fd.write(unicode(t)) else: sys.stdout.write(unicode(t))
def __init__(self, filename, rest_client): self.filename = filename self.d = anydbm.open("%s.db" % self.filename, "c") config = configuration.GetConfig() self.rest_client = rest_client self.deps = anydbm.open("%s-deps.db" % self.filename, "c")