if md5_sums_to_add: logging.info("To add to from %s %s %s:", osrel, arch, catrel) for md5 in md5_sums_to_add: logging.info(" + %s", cat_entry_by_md5[md5]["file_basename"]) # Remove # We could use checkpkg_lib.Catalog.RemoveSrv4(), but it would redo # many of the database queries and would be much slower. if md5_sums_to_remove: logging.info("Removing assignments from the catalog.") for md5 in md5_sums_to_remove: db_srv4s_in_cat_by_md5[md5].destroySelf() # Add if md5_sums_to_add: logging.info("Adding srv4 files to the %s %s %s catalog.", osrel, arch, catrel) db_catalog = checkpkg_lib.Catalog() for md5 in md5_sums_to_add: logging.debug("Adding %s", cat_entry_by_md5[md5]["file_basename"]) sqo_srv4 = m.Srv4FileStats.selectBy(md5_sum=md5).getOne() # If a package was previously examined, but not registered, we need to # register it now, to allow inclusion in a catalog. if not sqo_srv4.registered: logging.debug( "Package %s was not registered for releases. Registering it.", sqo_srv4.basename) stats = sqo_srv4.GetStatsStruct() package_stats.PackageStats.ImportPkg(stats, True) try: db_catalog.AddSrv4ToCatalog(sqo_srv4, osrel, arch, catrel)
def _ImportFiles(self, data, include_prefixes=None, show_progress=False): logging.debug("_ImportFiles()") osrel = data["osrel"] arch = data["arch"] contents = data["contents"] catalog = checkpkg_lib.Catalog() srv4_files_to_catalog = set() # The progressbar library doesn't like handling larger numbers # It displays up to 99% if we feed it a maxval in the range of hundreds of # thousands. progressbar_divisor = int(len(contents) / 1000) if progressbar_divisor < 1: progressbar_divisor = 1 update_period = 1L count = itertools.count() if show_progress: pbar = progressbar.ProgressBar() else: pbar = mute_progressbar.MuteProgressBar() pbar.maxval = len(contents) / progressbar_divisor pbar.start() cleaned_pkgs = set() for d in contents: i = count.next() if not i % update_period and (i / progressbar_divisor) <= pbar.maxval: pbar.update(i / progressbar_divisor) for pkgname in d["pkgnames"]: pkgname = self.SanitizeInstallContentsPkgname(pkgname) # If a package is a packge of our own, # it should not be imported that way; own packages should be # only managed by adding them to specific catalogs. skip_pkgname = False for prefix in common_constants.OWN_PKGNAME_PREFIXES: if pkgname.startswith(prefix): skip_pkgname = True break # Prefix whilelist - whitelisted prefixes win. if include_prefixes: for prefix_to_include in include_prefixes: if pkgname.startswith(prefix_to_include): skip_pkgname = False break if skip_pkgname: continue # We'll create one file instance for each package try: sqo_srv4 = self._GetFakeSrv4(pkgname, osrel, arch) except sqlobject.main.SQLObjectNotFound, e: print d raise if sqo_srv4 not in cleaned_pkgs: sqo_srv4.RemoveAllCswFiles() cleaned_pkgs.add(sqo_srv4) sqo_pkginst = self._GetPkginst(pkgname) f_path, f_basename = os.path.split(d["path"]) # This is really slow (one run ~1h), but works. # To speed it up, raw SQL + cursor.executemany() could be used, but # there's a incompatibility between MySQL and sqlite drivers: # MySQL: INSERT ... VALUES (%s, %s, %s); # sqlite: INSERT ... VALUES (?, ?, ?); # For now, using the sqlobject ORM which is slow, but at least # handles compatibility issues. csw_file = m.CswFile(pkginst=sqo_pkginst, line=d["line"], path=f_path, basename=f_basename, srv4_file=sqo_srv4) srv4_files_to_catalog.add(sqo_srv4)
def main(): parser = optparse.OptionParser(USAGE) parser.add_option("-d", "--debug", dest="debug", default=False, action="store_true", help="Turn on debugging messages") parser.add_option( "-t", "--pkg-review-template", dest="pkg_review_template", help="A Cheetah template used for package review reports.") parser.add_option("-r", "--os-release", dest="osrel", default="SunOS5.9", help="E.g. SunOS5.9") parser.add_option("-a", "--arch", dest="arch", default="sparc", help="'i386' or 'sparc'") parser.add_option("-c", "--catalog-release", dest="catrel", default="current", help="E.g. current, unstable, testing, stable") parser.add_option("--replace", dest="replace", default=False, action="store_true", help="Replace packages when importing (importpkg)") parser.add_option("--profile", dest="profile", default=False, action="store_true", help="Turn on profiling") parser.add_option("--force-unpack", dest="force_unpack", default=False, action="store_true", help="Force unpacking of packages") options, args = parser.parse_args() if options.debug: logging.basicConfig(level=logging.DEBUG) logging.debug("Debugging on") else: logging.basicConfig(level=logging.INFO) if not args: raise UsageError("Please specify a command. Se --help.") # SetUpSqlobjectConnection needs to be called after # logging.basicConfig configuration.SetUpSqlobjectConnection() command = args[0] args = args[1:] if command == 'show': subcommand = args[0] args = args[1:] elif command == 'pkg': subcommand = args[0] args = args[1:] else: subcommand = None md5_sums = args dm = database.DatabaseManager() # Automanage is not what we want to do, if the intention is to initialize # the database. if command != 'initdb': dm.AutoManage() if (command, subcommand) == ('show', 'errors'): for md5_sum in md5_sums: srv4 = GetPkg(md5_sum) res = m.CheckpkgErrorTag.select( m.CheckpkgErrorTag.q.srv4_file == srv4) for row in res: print row.pkgname, row.tag_name, row.tag_info, row.catrel.name, row.arch.name, print row.os_rel.short_name elif (command, subcommand) == ('show', 'overrides'): for md5_sum in md5_sums: srv4 = GetPkg(md5_sum) res = m.CheckpkgOverride.select( m.CheckpkgOverride.q.srv4_file == srv4) for row in res: print row.pkgname, row.tag_name, row.tag_info elif (command, subcommand) == ('show', 'pkg'): for md5_sum in md5_sums: srv4 = GetPkg(md5_sum) t = Template(SHOW_PKG_TMPL, searchList=[srv4]) sys.stdout.write(unicode(t)) elif command == 'gen-html': g = HtmlGenerator(md5_sums, options.pkg_review_template) sys.stdout.write(g.GenerateHtml()) elif command == 'initdb': config = configuration.GetConfig() db_uri = configuration.ComposeDatabaseUri(config) dbc = database.CatalogDatabase(uri=db_uri) dbc.CreateTables() dbc.InitialDataImport() elif command == 'importpkg': collector = package_stats.StatsCollector(logger=logging, debug=options.debug) file_list = args stats_list = collector.CollectStatsFromFiles( file_list, None, force_unpack=options.force_unpack) for stats in stats_list: logging.debug("Importing %s, %s", stats["basic_stats"]["md5_sum"], stats["basic_stats"]["pkg_basename"]) package_stats.PackageStats.ImportPkg(stats, options.replace) elif command == 'removepkg': for md5_sum in md5_sums: srv4 = GetPkg(md5_sum) in_catalogs = list(srv4.in_catalogs) if in_catalogs: for in_catalog in in_catalogs: logging.warning("%s", in_catalog) logging.warning( "Not removing from the database, because the package " "in question is part of at least one catalog.") else: logging.info("Removing %s", srv4) srv4.DeleteAllDependentObjects() srv4.destroySelf() elif command == 'add-to-cat': if len(args) <= 3: raise UsageError("Not enough arguments, see usage.") osrel, arch, catrel = args[:3] c = checkpkg_lib.Catalog() md5_sums = args[3:] for md5_sum in md5_sums: logging.debug("Adding %s to the catalog", md5_sum) try: sqo_srv4 = m.Srv4FileStats.select( m.Srv4FileStats.q.md5_sum == md5_sum).getOne() c.AddSrv4ToCatalog(sqo_srv4, osrel, arch, catrel) except sqlobject.main.SQLObjectNotFound, e: logging.warning("Srv4 file %s was not found in the database.", md5_sum)