def main():
parser = optparse.OptionParser(USAGE)
parser.add_option("-d",
"--debug",
dest="debug",
default=False,
action="store_true",
help="Turn on debugging messages")
parser.add_option(
"-t",
"--pkg-review-template",
dest="pkg_review_template",
help="A Cheetah template used for package review reports.")
parser.add_option("-r",
"--os-release",
dest="osrel",
default="SunOS5.9",
help="E.g. SunOS5.9")
parser.add_option("-a",
"--arch",
dest="arch",
default="sparc",
help="'i386' or 'sparc'")
parser.add_option("-c",
"--catalog-release",
dest="catrel",
default="unstable",
help="E.g. unstable, dublin")
parser.add_option("--replace",
dest="replace",
default=False,
action="store_true",
help="Replace packages when importing (importpkg)")
parser.add_option("--profile",
dest="profile",
default=False,
action="store_true",
help="Turn on profiling")
parser.add_option("--force-unpack",
dest="force_unpack",
default=False,
action="store_true",
help="Force unpacking of packages")
options, args = parser.parse_args()
if options.debug:
logging.basicConfig(level=logging.DEBUG)
logging.debug("Debugging on")
else:
logging.basicConfig(level=logging.INFO)
if not args:
raise UsageError("Please specify a command. Se --help.")
# SetUpSqlobjectConnection needs to be called after
# logging.basicConfig
configuration.SetUpSqlobjectConnection()
command = args[0]
args = args[1:]
if command == 'show':
subcommand = args[0]
args = args[1:]
elif command == 'pkg':
subcommand = args[0]
args = args[1:]
else:
subcommand = None
md5_sums = args
dm = database.DatabaseManager()
# Automanage is not what we want to do, if the intention is to initialize
# the database.
if command != 'initdb':
dm.AutoManage()
if (command, subcommand) == ('show', 'errors'):
for md5_sum in md5_sums:
srv4 = GetPkg(md5_sum)
res = m.CheckpkgErrorTag.select(
m.CheckpkgErrorTag.q.srv4_file == srv4)
for row in res:
print row.pkgname, row.tag_name, row.tag_info, row.catrel.name, row.arch.name,
print row.os_rel.short_name
elif (command, subcommand) == ('show', 'overrides'):
for md5_sum in md5_sums:
srv4 = GetPkg(md5_sum)
res = m.CheckpkgOverride.select(
m.CheckpkgOverride.q.srv4_file == srv4)
for row in res:
print row.pkgname, row.tag_name, row.tag_info
elif (command, subcommand) == ('show', 'pkg'):
for md5_sum in md5_sums:
srv4 = GetPkg(md5_sum)
t = Template(SHOW_PKG_TMPL, searchList=[srv4])
sys.stdout.write(unicode(t))
elif command == 'gen-html':
g = HtmlGenerator(md5_sums, options.pkg_review_template)
sys.stdout.write(g.GenerateHtml())
elif command == 'initdb':
config = configuration.GetConfig()
db_uri = configuration.ComposeDatabaseUri(config)
dbc = database.CatalogDatabase(uri=db_uri)
dbc.CreateTables()
dbc.InitialDataImport()
elif command == 'importpkg':
collector = package_stats.StatsCollector(logger=logging,
debug=options.debug)
file_list = args
try:
stats_list = collector.CollectStatsFromFiles(
file_list, None, force_unpack=options.force_unpack)
except sqlobject.dberrors.OperationalError, e:
exception_msg = (
"DELETE command denied to user "
"'pkg_maintainer'@'192.168.1.2' for table 'csw_file'")
if exception_msg in str(e):
logging.fatal(
"You don't have sufficient privileges to overwrite previously "
"imported package. Did you run checkpkg before running "
"csw-upload-pkg?")
sys.exit(1)
else:
raise e
for stats in stats_list:
logging.debug("Importing %s, %s", stats["basic_stats"]["md5_sum"],
stats["basic_stats"]["pkg_basename"])
try:
package_stats.PackageStats.ImportPkg(stats, options.replace)
except sqlobject.dberrors.OperationalError, e:
logging.fatal(
"A problem when importing package data has occurred: %s",
e)
sys.exit(1)
class CatalogImporter(object):
def __init__(self, debug=False):
self.debug = debug
def SyncFromCatalogFile(self,
osrel,
arch,
catrel,
catalog_file,
force_unpack=False):
"""Syncs a given catalog from a catalog file.
Imports srv4 files if necessary.
"""
if catrel not in CATALOGS_ALLOWED_TO_BE_IMPORTED:
raise UsageError("Catalogs that can be imported: %s" %
CATALOGS_ALLOWED_TO_BE_IMPORTED)
catalog_dir = os.path.dirname(catalog_file)
# The plan:
# - read in the catalog file, and build a md5-filename correspondence
# data structure.
logging.debug("Reading the catalog file from disk.")
src_catalog = catalog.OpencswCatalog(open(catalog_file, "rb"))
catalog_data = src_catalog.GetCatalogData()
cat_entry_by_md5 = {}
cat_entry_by_basename = {}
for catalog_entry in catalog_data:
cat_entry_by_md5[catalog_entry["md5sum"]] = catalog_entry
cat_entry_by_basename[
catalog_entry["file_basename"]] = catalog_entry
# - import all srv4 files that were not in the database so far
sqo_objects = set()
entries_to_import = []
logging.debug("Checking which srv4 files are already in the db.")
for md5 in cat_entry_by_md5:
try:
sqo_list = m.Srv4FileStats.selectBy(md5_sum=md5).getOne()
except sqlobject.main.SQLObjectNotFound, e:
entries_to_import.append(cat_entry_by_md5[md5])
basenames = [x["file_basename"] for x in entries_to_import]
file_list = []
if entries_to_import:
logging.info("Srv4 files to import:")
for basename in sorted(basenames):
logging.info(" + %s", basename)
file_list.append(os.path.join(catalog_dir, basename))
new_statdicts = []
if file_list:
collector = package_stats.StatsCollector(logger=logging,
debug=self.debug)
new_statdicts = collector.CollectStatsFromFiles(
file_list, None, force_unpack=force_unpack)
new_statdicts_by_md5 = {}
if new_statdicts:
logging.info("Marking imported packages as registered.")
for statdict in new_statdicts:
new_statdicts_by_md5[statdict["basic_stats"]
["md5_sum"]] = statdict
package_stats.PackageStats.ImportPkg(statdict)
# - sync the specific catalog
# - find the md5 sum list of the current catalog
logging.debug("Retrieving current catalog assigments from the db.")
sqo_osrel = m.OsRelease.selectBy(short_name=osrel).getOne()
sqo_arch = m.Architecture.selectBy(name=arch).getOne()
sqo_catrel = m.CatalogRelease.selectBy(name=catrel).getOne()
res = m.Srv4FileInCatalog.select(
sqlobject.AND(m.Srv4FileInCatalog.q.osrel == sqo_osrel,
m.Srv4FileInCatalog.q.arch == sqo_arch,
m.Srv4FileInCatalog.q.catrel == sqo_catrel))
db_srv4s_in_cat_by_md5 = {}
for srv4_in_cat in res:
try:
srv4 = srv4_in_cat.srv4file
if srv4.use_to_generate_catalogs:
db_srv4s_in_cat_by_md5[srv4.md5_sum] = srv4_in_cat
except sqlobject.main.SQLObjectNotFound, e:
logging.warning(
"Could not retrieve a srv4 file from the db: %s", e)
# Since the srv4_in_cat object has lost its reference, there's no use
# keeping it around.
srv4_in_cat.destroySelf()
def main():
parser = optparse.OptionParser(USAGE)
parser.add_option("-d",
"--debug",
dest="debug",
default=False,
action="store_true",
help="Turn on debugging messages")
parser.add_option(
"-t",
"--pkg-review-template",
dest="pkg_review_template",
help="A Cheetah template used for package review reports.")
parser.add_option("-r",
"--os-release",
dest="osrel",
default="SunOS5.9",
help="E.g. SunOS5.9")
parser.add_option("-a",
"--arch",
dest="arch",
default="sparc",
help="'i386' or 'sparc'")
parser.add_option("-c",
"--catalog-release",
dest="catrel",
default="current",
help="E.g. current, unstable, testing, stable")
parser.add_option("--replace",
dest="replace",
default=False,
action="store_true",
help="Replace packages when importing (importpkg)")
parser.add_option("--profile",
dest="profile",
default=False,
action="store_true",
help="Turn on profiling")
parser.add_option("--force-unpack",
dest="force_unpack",
default=False,
action="store_true",
help="Force unpacking of packages")
options, args = parser.parse_args()
if options.debug:
logging.basicConfig(level=logging.DEBUG)
logging.debug("Debugging on")
else:
logging.basicConfig(level=logging.INFO)
if not args:
raise UsageError("Please specify a command. Se --help.")
# SetUpSqlobjectConnection needs to be called after
# logging.basicConfig
configuration.SetUpSqlobjectConnection()
command = args[0]
args = args[1:]
if command == 'show':
subcommand = args[0]
args = args[1:]
elif command == 'pkg':
subcommand = args[0]
args = args[1:]
else:
subcommand = None
md5_sums = args
dm = database.DatabaseManager()
# Automanage is not what we want to do, if the intention is to initialize
# the database.
if command != 'initdb':
dm.AutoManage()
if (command, subcommand) == ('show', 'errors'):
for md5_sum in md5_sums:
srv4 = GetPkg(md5_sum)
res = m.CheckpkgErrorTag.select(
m.CheckpkgErrorTag.q.srv4_file == srv4)
for row in res:
print row.pkgname, row.tag_name, row.tag_info, row.catrel.name, row.arch.name,
print row.os_rel.short_name
elif (command, subcommand) == ('show', 'overrides'):
for md5_sum in md5_sums:
srv4 = GetPkg(md5_sum)
res = m.CheckpkgOverride.select(
m.CheckpkgOverride.q.srv4_file == srv4)
for row in res:
print row.pkgname, row.tag_name, row.tag_info
elif (command, subcommand) == ('show', 'pkg'):
for md5_sum in md5_sums:
srv4 = GetPkg(md5_sum)
t = Template(SHOW_PKG_TMPL, searchList=[srv4])
sys.stdout.write(unicode(t))
elif command == 'gen-html':
g = HtmlGenerator(md5_sums, options.pkg_review_template)
sys.stdout.write(g.GenerateHtml())
elif command == 'initdb':
config = configuration.GetConfig()
db_uri = configuration.ComposeDatabaseUri(config)
dbc = database.CatalogDatabase(uri=db_uri)
dbc.CreateTables()
dbc.InitialDataImport()
elif command == 'importpkg':
collector = package_stats.StatsCollector(logger=logging,
debug=options.debug)
file_list = args
stats_list = collector.CollectStatsFromFiles(
file_list, None, force_unpack=options.force_unpack)
for stats in stats_list:
logging.debug("Importing %s, %s", stats["basic_stats"]["md5_sum"],
stats["basic_stats"]["pkg_basename"])
package_stats.PackageStats.ImportPkg(stats, options.replace)
elif command == 'removepkg':
for md5_sum in md5_sums:
srv4 = GetPkg(md5_sum)
in_catalogs = list(srv4.in_catalogs)
if in_catalogs:
for in_catalog in in_catalogs:
logging.warning("%s", in_catalog)
logging.warning(
"Not removing from the database, because the package "
"in question is part of at least one catalog.")
else:
logging.info("Removing %s", srv4)
srv4.DeleteAllDependentObjects()
srv4.destroySelf()
elif command == 'add-to-cat':
if len(args) <= 3:
raise UsageError("Not enough arguments, see usage.")
osrel, arch, catrel = args[:3]
c = checkpkg_lib.Catalog()
md5_sums = args[3:]
for md5_sum in md5_sums:
logging.debug("Adding %s to the catalog", md5_sum)
try:
sqo_srv4 = m.Srv4FileStats.select(
m.Srv4FileStats.q.md5_sum == md5_sum).getOne()
c.AddSrv4ToCatalog(sqo_srv4, osrel, arch, catrel)
except sqlobject.main.SQLObjectNotFound, e:
logging.warning("Srv4 file %s was not found in the database.",
md5_sum)
def main():
parser = optparse.OptionParser(USAGE)
parser.add_option("-d", "--debug",
dest="debug",
action="store_true",
default=False,
help="Switch on debugging messages")
parser.add_option("-q", "--quiet",
dest="quiet",
action="store_true",
default=False,
help="Display less messages")
parser.add_option("--catalog-release",
dest="catrel",
default="current",
help="A catalog release: current, unstable, testing, stable.")
parser.add_option("-r", "--os-releases",
dest="osrel_commas",
help=("Comma separated list of ['SunOS5.9', 'SunOS5.10'], "
"e.g. 'SunOS5.9,SunOS5.10'."))
parser.add_option("-a", "--architecture",
dest="arch",
help="Architecture: i386, sparc.")
parser.add_option("--profile", dest="profile",
default=False, action="store_true",
help="Enable profiling (a developer option).")
options, args = parser.parse_args()
assert len(args), "The list of files or md5 sums must be not empty."
logging_level = logging.INFO
if options.quiet:
logging_level = logging.WARNING
elif options.debug:
# If both flags are set, debug wins.
logging_level = logging.DEBUG
logging.basicConfig(level=logging_level)
logging.debug("Starting.")
configuration.SetUpSqlobjectConnection()
dm = database.DatabaseManager()
dm.AutoManage()
err_msg_list = []
if not options.osrel_commas:
err_msg_list.append("Please specify --os-releases.")
if not options.arch:
err_msg_list.append("Please specify --architecture.")
if options.arch not in cc.PHYSICAL_ARCHITECTURES:
err_msg_list.append(
"Valid --architecture values are: %s, you passed: %r"
% (cc.PHYSICAL_ARCHITECTURES, options.arch))
if err_msg_list:
raise UsageError(" ".join(err_msg_list))
stats_list = []
collector = package_stats.StatsCollector(
logger=logging,
debug=options.debug)
# We need to separate files and md5 sums.
md5_sums, file_list = [], []
for arg in args:
if struct_util.IsMd5(arg):
md5_sums.append(arg)
else:
file_list.append(arg)
if file_list:
stats_list = collector.CollectStatsFromFiles(file_list, None)
# We need the md5 sums of these files
md5_sums.extend([x["basic_stats"]["md5_sum"] for x in stats_list])
assert md5_sums, "The list of md5 sums must not be empty."
logging.debug("md5_sums: %s", md5_sums)
osrel_list = options.osrel_commas.split(",")
logging.debug("Reading packages data from the database.")
# This part might need improvements in order to handle a whole
# catalog. On the other hand, if we already have the whole catalog in
# the database, we can do it altogether differently.
# Transforming the result to a list in order to force object
# retrieval.
sqo_pkgs = list(models.Srv4FileStats.select(
sqlobject.IN(models.Srv4FileStats.q.md5_sum, md5_sums)))
tags_for_all_osrels = []
try:
sqo_catrel = models.CatalogRelease.selectBy(name=options.catrel).getOne()
except sqlobject.main.SQLObjectNotFound as e:
logging.fatal("Fetching from the db has failed: catrel=%s",
repr(str(options.catrel)))
logging.fatal("Available catalog releases:")
sqo_catrels = models.CatalogRelease.select()
for sqo_catrel in sqo_catrels:
logging.fatal(" - %s", sqo_catrel.name)
raise
sqo_arch = models.Architecture.selectBy(name=options.arch).getOne()
for osrel in osrel_list:
sqo_osrel = models.OsRelease.selectBy(short_name=osrel).getOne()
dm.VerifyContents(sqo_osrel, sqo_arch)
check_manager = checkpkg_lib.CheckpkgManager2(
CHECKPKG_MODULE_NAME,
sqo_pkgs,
osrel,
options.arch,
options.catrel,
debug=options.debug,
show_progress=(os.isatty(1) and not options.quiet))
# Running the checks, reporting and exiting.
exit_code, screen_report, tags_report = check_manager.Run()
screen_report = unicode(screen_report)
if not options.quiet and screen_report:
# TODO: Write this to screen only after overrides are applied.
sys.stdout.write(screen_report)
else:
logging.debug("No screen report.")
overrides_list = [list(pkg.GetOverridesResult()) for pkg in sqo_pkgs]
override_list = reduce(operator.add, overrides_list)
args = (sqo_osrel, sqo_arch, sqo_catrel)
tag_lists = [list(pkg.GetErrorTagsResult(*args)) for pkg in sqo_pkgs]
error_tags = reduce(operator.add, tag_lists)
(tags_after_overrides,
unapplied_overrides) = overrides.ApplyOverrides(error_tags, override_list)
tags_for_all_osrels.extend(tags_after_overrides)
if not options.quiet:
if tags_after_overrides:
print(textwrap.fill(BEFORE_OVERRIDES, 80))
for checkpkg_tag in tags_after_overrides:
print checkpkg_tag.ToGarSyntax()
print textwrap.fill(AFTER_OVERRIDES, 80)
if unapplied_overrides:
print textwrap.fill(UNAPPLIED_OVERRIDES, 80)
for override in unapplied_overrides:
print u"* Unused %s" % override
exit_code = bool(tags_for_all_osrels)
sys.exit(exit_code)
def main():
parser = optparse.OptionParser(USAGE)
parser.add_option("-d", "--debug",
dest="debug",
action="store_true",
default=False,
help="Switch on debugging messages")
parser.add_option("-q", "--quiet",
dest="quiet",
action="store_true",
default=False,
help="Display less messages")
parser.add_option("--catalog-release",
dest="catrel",
default="current",
help="A catalog release: current, unstable, testing, stable.")
parser.add_option("-r", "--os-releases",
dest="osrel_commas",
help=("Comma separated list of ['SunOS5.9', 'SunOS5.10'], "
"e.g. 'SunOS5.9,SunOS5.10'."))
parser.add_option("-a", "--architecture",
dest="arch",
help="Architecture: i386, sparc.")
parser.add_option("--profile", dest="profile",
default=False, action="store_true",
help="Enable profiling (a developer option).")
options, args = parser.parse_args()
assert len(args), "The list of files or md5 sums must be not empty."
logging_level = logging.INFO
if options.quiet:
logging_level = logging.WARNING
elif options.debug:
# If both flags are set, debug wins.
logging_level = logging.DEBUG
logging.basicConfig(level=logging_level)
logging.debug("Starting.")
configuration.SetUpSqlobjectConnection()
dm = database.DatabaseManager()
dm.AutoManage()
err_msg_list = []
if not options.osrel_commas:
err_msg_list.append("Please specify --os-releases.")
if not options.arch:
err_msg_list.append("Please specify --architecture.")
if err_msg_list:
raise UsageError(" ".join(err_msg_list))
stats_list = []
collector = package_stats.StatsCollector(
logger=logging,
debug=options.debug)
# We need to separate files and md5 sums.
md5_sums, file_list = [], []
for arg in args:
if struct_util.IsMd5(arg):
md5_sums.append(arg)
else:
file_list.append(arg)
if file_list:
stats_list = collector.CollectStatsFromFiles(file_list, None)
# We need the md5 sums of these files
md5_sums.extend([x["basic_stats"]["md5_sum"] for x in stats_list])
assert md5_sums, "The list of md5 sums must not be empty."
logging.debug("md5_sums: %s", md5_sums)
osrel_list = options.osrel_commas.split(",")
logging.debug("Reading packages data from the database.")
# This part might need improvements in order to handle a whole
# catalog. On the other hand, if we already have the whole catalog in
# the database, we can do it altogether differently.
# Transforming the result to a list in order to force object
# retrieval.
sqo_pkgs = list(models.Srv4FileStats.select(
sqlobject.IN(models.Srv4FileStats.q.md5_sum, md5_sums)))
tags_for_all_osrels = []
try:
sqo_catrel = models.CatalogRelease.selectBy(name=options.catrel).getOne()
except sqlobject.main.SQLObjectNotFound, e:
logging.fatal("Fetching from the db has failed: catrel=%s",
repr(str(options.catrel)))
logging.fatal("Available catalog releases:")
sqo_catrels = models.CatalogRelease.select()
for sqo_catrel in sqo_catrels:
logging.fatal(" - %s", sqo_catrel.name)
raise