if md5_sums_to_add:
logging.info("To add to from %s %s %s:", osrel, arch, catrel)
for md5 in md5_sums_to_add:
logging.info(" + %s", cat_entry_by_md5[md5]["file_basename"])
# Remove
# We could use checkpkg_lib.Catalog.RemoveSrv4(), but it would redo
# many of the database queries and would be much slower.
if md5_sums_to_remove:
logging.info("Removing assignments from the catalog.")
for md5 in md5_sums_to_remove:
db_srv4s_in_cat_by_md5[md5].destroySelf()
# Add
if md5_sums_to_add:
logging.info("Adding srv4 files to the %s %s %s catalog.", osrel,
arch, catrel)
db_catalog = checkpkg_lib.Catalog()
for md5 in md5_sums_to_add:
logging.debug("Adding %s",
cat_entry_by_md5[md5]["file_basename"])
sqo_srv4 = m.Srv4FileStats.selectBy(md5_sum=md5).getOne()
# If a package was previously examined, but not registered, we need to
# register it now, to allow inclusion in a catalog.
if not sqo_srv4.registered:
logging.debug(
"Package %s was not registered for releases. Registering it.",
sqo_srv4.basename)
stats = sqo_srv4.GetStatsStruct()
package_stats.PackageStats.ImportPkg(stats, True)
try:
db_catalog.AddSrv4ToCatalog(sqo_srv4, osrel, arch, catrel)
def _ImportFiles(self, data, include_prefixes=None, show_progress=False):
logging.debug("_ImportFiles()")
osrel = data["osrel"]
arch = data["arch"]
contents = data["contents"]
catalog = checkpkg_lib.Catalog()
srv4_files_to_catalog = set()
# The progressbar library doesn't like handling larger numbers
# It displays up to 99% if we feed it a maxval in the range of hundreds of
# thousands.
progressbar_divisor = int(len(contents) / 1000)
if progressbar_divisor < 1:
progressbar_divisor = 1
update_period = 1L
count = itertools.count()
if show_progress:
pbar = progressbar.ProgressBar()
else:
pbar = mute_progressbar.MuteProgressBar()
pbar.maxval = len(contents) / progressbar_divisor
pbar.start()
cleaned_pkgs = set()
for d in contents:
i = count.next()
if not i % update_period and (i / progressbar_divisor) <= pbar.maxval:
pbar.update(i / progressbar_divisor)
for pkgname in d["pkgnames"]:
pkgname = self.SanitizeInstallContentsPkgname(pkgname)
# If a package is a packge of our own,
# it should not be imported that way; own packages should be
# only managed by adding them to specific catalogs.
skip_pkgname = False
for prefix in common_constants.OWN_PKGNAME_PREFIXES:
if pkgname.startswith(prefix):
skip_pkgname = True
break
# Prefix whilelist - whitelisted prefixes win.
if include_prefixes:
for prefix_to_include in include_prefixes:
if pkgname.startswith(prefix_to_include):
skip_pkgname = False
break
if skip_pkgname:
continue
# We'll create one file instance for each package
try:
sqo_srv4 = self._GetFakeSrv4(pkgname, osrel, arch)
except sqlobject.main.SQLObjectNotFound, e:
print d
raise
if sqo_srv4 not in cleaned_pkgs:
sqo_srv4.RemoveAllCswFiles()
cleaned_pkgs.add(sqo_srv4)
sqo_pkginst = self._GetPkginst(pkgname)
f_path, f_basename = os.path.split(d["path"])
# This is really slow (one run ~1h), but works.
# To speed it up, raw SQL + cursor.executemany() could be used, but
# there's a incompatibility between MySQL and sqlite drivers:
# MySQL: INSERT ... VALUES (%s, %s, %s);
# sqlite: INSERT ... VALUES (?, ?, ?);
# For now, using the sqlobject ORM which is slow, but at least
# handles compatibility issues.
csw_file = m.CswFile(pkginst=sqo_pkginst,
line=d["line"], path=f_path, basename=f_basename,
srv4_file=sqo_srv4)
srv4_files_to_catalog.add(sqo_srv4)
def main():
parser = optparse.OptionParser(USAGE)
parser.add_option("-d",
"--debug",
dest="debug",
default=False,
action="store_true",
help="Turn on debugging messages")
parser.add_option(
"-t",
"--pkg-review-template",
dest="pkg_review_template",
help="A Cheetah template used for package review reports.")
parser.add_option("-r",
"--os-release",
dest="osrel",
default="SunOS5.9",
help="E.g. SunOS5.9")
parser.add_option("-a",
"--arch",
dest="arch",
default="sparc",
help="'i386' or 'sparc'")
parser.add_option("-c",
"--catalog-release",
dest="catrel",
default="current",
help="E.g. current, unstable, testing, stable")
parser.add_option("--replace",
dest="replace",
default=False,
action="store_true",
help="Replace packages when importing (importpkg)")
parser.add_option("--profile",
dest="profile",
default=False,
action="store_true",
help="Turn on profiling")
parser.add_option("--force-unpack",
dest="force_unpack",
default=False,
action="store_true",
help="Force unpacking of packages")
options, args = parser.parse_args()
if options.debug:
logging.basicConfig(level=logging.DEBUG)
logging.debug("Debugging on")
else:
logging.basicConfig(level=logging.INFO)
if not args:
raise UsageError("Please specify a command. Se --help.")
# SetUpSqlobjectConnection needs to be called after
# logging.basicConfig
configuration.SetUpSqlobjectConnection()
command = args[0]
args = args[1:]
if command == 'show':
subcommand = args[0]
args = args[1:]
elif command == 'pkg':
subcommand = args[0]
args = args[1:]
else:
subcommand = None
md5_sums = args
dm = database.DatabaseManager()
# Automanage is not what we want to do, if the intention is to initialize
# the database.
if command != 'initdb':
dm.AutoManage()
if (command, subcommand) == ('show', 'errors'):
for md5_sum in md5_sums:
srv4 = GetPkg(md5_sum)
res = m.CheckpkgErrorTag.select(
m.CheckpkgErrorTag.q.srv4_file == srv4)
for row in res:
print row.pkgname, row.tag_name, row.tag_info, row.catrel.name, row.arch.name,
print row.os_rel.short_name
elif (command, subcommand) == ('show', 'overrides'):
for md5_sum in md5_sums:
srv4 = GetPkg(md5_sum)
res = m.CheckpkgOverride.select(
m.CheckpkgOverride.q.srv4_file == srv4)
for row in res:
print row.pkgname, row.tag_name, row.tag_info
elif (command, subcommand) == ('show', 'pkg'):
for md5_sum in md5_sums:
srv4 = GetPkg(md5_sum)
t = Template(SHOW_PKG_TMPL, searchList=[srv4])
sys.stdout.write(unicode(t))
elif command == 'gen-html':
g = HtmlGenerator(md5_sums, options.pkg_review_template)
sys.stdout.write(g.GenerateHtml())
elif command == 'initdb':
config = configuration.GetConfig()
db_uri = configuration.ComposeDatabaseUri(config)
dbc = database.CatalogDatabase(uri=db_uri)
dbc.CreateTables()
dbc.InitialDataImport()
elif command == 'importpkg':
collector = package_stats.StatsCollector(logger=logging,
debug=options.debug)
file_list = args
stats_list = collector.CollectStatsFromFiles(
file_list, None, force_unpack=options.force_unpack)
for stats in stats_list:
logging.debug("Importing %s, %s", stats["basic_stats"]["md5_sum"],
stats["basic_stats"]["pkg_basename"])
package_stats.PackageStats.ImportPkg(stats, options.replace)
elif command == 'removepkg':
for md5_sum in md5_sums:
srv4 = GetPkg(md5_sum)
in_catalogs = list(srv4.in_catalogs)
if in_catalogs:
for in_catalog in in_catalogs:
logging.warning("%s", in_catalog)
logging.warning(
"Not removing from the database, because the package "
"in question is part of at least one catalog.")
else:
logging.info("Removing %s", srv4)
srv4.DeleteAllDependentObjects()
srv4.destroySelf()
elif command == 'add-to-cat':
if len(args) <= 3:
raise UsageError("Not enough arguments, see usage.")
osrel, arch, catrel = args[:3]
c = checkpkg_lib.Catalog()
md5_sums = args[3:]
for md5_sum in md5_sums:
logging.debug("Adding %s to the catalog", md5_sum)
try:
sqo_srv4 = m.Srv4FileStats.select(
m.Srv4FileStats.q.md5_sum == md5_sum).getOne()
c.AddSrv4ToCatalog(sqo_srv4, osrel, arch, catrel)
except sqlobject.main.SQLObjectNotFound, e:
logging.warning("Srv4 file %s was not found in the database.",
md5_sum)