def __init__(self, logger=None, debug=False):
if logger:
self.logger = logger
else:
self.logger = logging
self.debug = debug
self.config = configuration.GetConfig()
self.rest_client = rest.RestClient(
pkgdb_url=self.config.get('rest', 'pkgdb'),
releases_url=self.config.get('rest', 'releases'))
def __init__(self, debug=False):
self.debug = debug
config = configuration.GetConfig()
username, password = rest.GetUsernameAndPassword()
self.rest_client = rest.RestClient(
pkgdb_url=config.get('rest', 'pkgdb'),
releases_url=config.get('rest', 'releases'),
username=username,
password=password,
debug=debug)
def __init__(self, srv4_pkg, stats_basedir=None, md5sum=None, debug=False):
super(PackageStats, self).__init__()
self.srv4_pkg = srv4_pkg
self.md5sum = md5sum
self.dir_format_pkg = None
self.all_stats = {}
self.db_pkg_stats = None
config = configuration.GetConfig()
self.rest_client = rest.RestClient(
pkgdb_url=config.get('rest', 'pkgdb'),
releases_url=config.get('rest', 'releases'))
def main():
oldcatrel, newcatrel, arch, osrel = GetCLIOptions()
config = configuration.GetConfig()
username, password = rest.GetUsernameAndPassword()
rest_client = rest.RestClient(pkgdb_url=config.get('rest', 'pkgdb'),
releases_url=config.get('rest', 'releases'),
username=username,
password=password)
removelst, rebuildlst, obsolete = (ComputeRemoveAndRebuild(
oldcatrel, newcatrel, arch, osrel, rest_client))
WriteToTextFiles(removelst, rebuildlst, newcatrel, arch, osrel, obsolete)
def __init__(self, binary_path, debug=False):
self.debug = debug
self._binary_path = binary_path
self.config = configuration.GetConfig()
username, password = rest.GetUsernameAndPassword()
self.rest_client = rest.RestClient(
pkgdb_url=self.config.get('rest', 'pkgdb'),
releases_url=self.config.get('rest', 'releases'),
username=username,
password=password)
fd = open(self._binary_path, 'rb')
self._mmap = mmap.mmap(fd.fileno(), 0, access=mmap.PROT_READ)
self._elffile = ELFFile(self._mmap)
def __init__(self, osrel, arch, debug=False):
super(InstallContentsImporter, self).__init__()
self.osrel = osrel
self.arch = arch
self.pkginst_cache = {}
config = configuration.GetConfig()
username, password = rest.GetUsernameAndPassword()
self.rest_client = rest.RestClient(
pkgdb_url=config.get('rest', 'pkgdb'),
releases_url=config.get('rest', 'releases'),
username=username,
password=password,
debug=debug)
def __init__(self,
catrel,
arch,
osrel,
fn_ts,
gen_catalog_bin,
chkcat="/opt/csw/bin/chkcat",
cattiming_class=CatalogTiming,
tsrecord_class=TimestampRecord):
"""Constructor.
"fn_ts" is the path name to the time stamp file.
"gen_catalog_bin" is the path to the gen-catalog-index binary
"chkcat" is the path to `chkcat'. By default
`/opt/csw/bin/chkcat'.
"cattiming_class" and "tsrecord_class" are mainly used for
unit tests in order to provide classes with overridden
methods.
"""
config = configuration.GetConfig()
username, password = GetUsernameAndPassword()
self.rest_client = RestClient(pkgdb_url=config.get('rest', 'pkgdb'),
releases_url=config.get(
'rest', 'releases'),
username=username,
password=password)
self.__catalogfgen = GenerateCatalog(catrel, arch, osrel,
gen_catalog_bin)
self.__chkcat = chkcat
# store for later use
self._catrel = catrel
self._arch = arch
self._osrel = osrel
# will be set by __enter__ and unset by __exit__
self.tmpdir = None
self.__tsrecord_class = tsrecord_class
self.__cattiming_class = cattiming_class
self.__timestamp_record = tsrecord_class(fn_ts)
def __init__(self, name, sqo_pkgs_list, osrel, arch, catrel, debug=False,
show_progress=False):
super(CheckpkgManagerBase, self).__init__()
self.debug = debug
self.name = name
self.sqo_pkgs_list = sqo_pkgs_list
self.osrel = osrel
self.arch = arch
self.catrel = catrel
self.show_progress = show_progress
self._ResetState()
self.individual_checks = []
self.set_checks = []
config = configuration.GetConfig()
username, password = rest.GetUsernameAndPassword()
self.rest_client = rest.RestClient(
pkgdb_url=config.get('rest', 'pkgdb'),
releases_url=config.get('rest', 'releases'),
username=username,
password=password)
def main():
parser = optparse.OptionParser()
parser.add_option("-c", "--catalogname", dest="catalogname")
parser.add_option("--catalog-release", dest="catrel", default="kiel")
parser.add_option("--arch", dest="arch", default="sparc")
parser.add_option("--os-release", dest="osrel", default="SunOS5.9")
parser.add_option("--md5", dest="md5")
parser.add_option("--debug",
dest="debug",
default=False,
action="store_true")
options, args = parser.parse_args()
if options.debug:
logging.basicConfig(level=logging.DEBUG)
else:
logging.basicConfig(level=logging.INFO)
config = configuration.GetConfig()
rest_client = rest.RestClient(pkgdb_url=config.get('rest', 'pkgdb'),
releases_url=config.get('rest', 'releases'))
if options.md5:
md5 = options.md5
else:
pkg_basic = rest_client.Srv4ByCatalogAndCatalogname(
options.catrel, options.arch, options.osrel, options.catalogname)
md5 = pkg_basic["md5_sum"]
logging.info("Fetching %s", md5)
pkgstats = rest_client.GetBlob('pkgstats', md5)
# getting elfdump info
elfdump_data = {}
for _, binary_md5 in pkgstats["binary_md5_sums"]:
elfdump_data[binary_md5] = rest_client.GetBlob('elfdump', binary_md5)
catalogname = pkgstats["basic_stats"]["catalogname"]
target_file = "lib/python/testdata/%s_stats.py" % catalogname
pkgstats['elfdump_info'] = elfdump_data
with open(target_file, "wb") as fd:
fd.write("# Generated by running:\n")
fd.write("# %s\n\n" % ' '.join(pipes.quote(x) for x in sys.argv))
fd.write("pkgstats = [")
fd.write(pprint.pformat(pkgstats))
fd.write("]\n\n")
print target_file, "done"
def main():
parser = optparse.OptionParser(USAGE)
parser.add_option("-c",
"--catalogname",
dest="catalogname",
help='the name of the package in catalog')
parser.add_option("--os-releases",
dest="os_releases",
help=("Comma separated OS releases, e.g. "
"SunOS5.9,SunOS5.10"))
parser.add_option("--debug", dest="debug", action="store_true")
parser.add_option("--dry-run",
dest="dry_run",
default=False,
action="store_true",
help=("Don't apply changes (no REST calls)."))
options, args = parser.parse_args()
logging_level = logging.INFO
if options.debug:
logging_level = logging.DEBUG
fmt = '%(levelname)s %(asctime)s %(filename)s:%(lineno)d %(message)s'
logging.basicConfig(format=fmt, level=logging_level)
if not options.catalogname:
logging.error('option catalogname required \n%s', USAGE)
sys.exit(1)
os_releases = common_constants.OS_RELS
if options.os_releases:
os_releases = options.os_releases.split(",")
config = configuration.GetConfig()
username, password = rest.GetUsernameAndPassword()
rest_client = rest.RestClient(pkgdb_url=config.get('rest', 'pkgdb'),
releases_url=config.get('rest', 'releases'),
username=username,
password=password)
pr = PackageRemover(rest_client)
pr.RemovePackage(options.catalogname, not options.dry_run, os_releases)
def __init__(self, pkg_path, debug):
self.debug = debug
self.pkg_path = pkg_path
self._work_dir = None
self._admin_file = None
self._gunzipped_path = None
self._md5_sum = None
self._stat = None
self._mtime = None
self._transformed = False
self._pkgname = None
self._pkginfo_dict = None
self._dir_format_base_dir = None
self._files_metadata = None
self._binaries = None
self._file_paths = None
self.config = configuration.GetConfig()
username, password = rest.GetUsernameAndPassword()
self.rest_client = rest.RestClient(
pkgdb_url=self.config.get('rest', 'pkgdb'),
releases_url=self.config.get('rest', 'releases'),
username=username,
password=password)
def main():
parser = optparse.OptionParser()
parser.add_option("--debug",
dest="debug",
default=False,
action="store_true")
parser.add_option("--catalog-release", dest="catrel", default="unstable")
parser.add_option("--arch", dest="arch", default="i386")
parser.add_option("--os-release", dest="osrel", default="SunOS5.10")
parser.add_option("--md5", dest="md5")
parser.add_option("--check-name", dest="check_name")
options, args = parser.parse_args()
if options.debug:
logging.basicConfig(level=logging.DEBUG)
else:
logging.basicConfig(level=logging.INFO)
configuration.SetUpSqlobjectConnection()
osrel = options.osrel
catrel = options.catrel
arch = options.arch
md5 = options.md5
config = configuration.GetConfig()
rest_client = rest.RestClient(pkgdb_url=config.get('rest', 'pkgdb'),
releases_url=config.get('rest', 'releases'))
logging.info("Fetching %s", md5)
pkgstats = rest_client.GetBlob('pkgstats', md5)
catalog = checkpkg_lib.Catalog()
check_function = getattr(package_checks, options.check_name)
examined_files_by_pkg = {}
check_interface = LoggingCheckInterface(osrel, arch, catrel, catalog,
examined_files_by_pkg)
messenger = checkpkg_lib.CheckpkgMessenger()
pkgstats['elfdump_info'] = checkpkg_lib.LazyElfinfo(rest_client)
check_function([pkgstats],
check_interface,
logger=logging,
messenger=messenger)
import re
import web
from sqlobject import sqlbuilder
# sys.stderr.write("Running Python binary from {}.\n".format(sys.executable))
# sys.stderr.write("Python path is {}.\n".format(sys.path))
from lib.python import checkpkg_lib
from lib.python import configuration
from lib.python import models
from lib.python import representations
from lib.web import web_lib
config = configuration.GetConfig()
LOG_FILE_TMPL = config.get("pkgdb_app", "log_file")
urls_html = (
r'/', 'index',
r'/favicon\.ico', 'Favicon',
r'/srv4/', 'Srv4List',
r'/srv4/([0-9a-f]{32})/', 'Srv4Detail',
r'/srv4/([0-9a-f]{32})/files/', 'Srv4DetailFiles',
r'/srv4/([0-9a-f]{32})/struct-dump/', 'Srv4StructDump',
r'/elfdump/([0-9a-f]{32})/', 'Elfdump',
r'/catalogs/', 'CatalogList',
r'/catalogs/([\w-]+)-(sparc|i386)-(SunOS[^-]+)/', 'CatalogDetail',
r'/maintainers/', 'MaintainerList',
r'/maintainers/(\d+)/', 'MaintainerDetail',
r'/maintainers/(\d+)/checkpkg/', 'MaintainerCheckpkgReport',
def main():
parser = optparse.OptionParser(USAGE)
parser.add_option("-d",
"--debug",
dest="debug",
action="store_true",
default=False,
help="Switch on debugging messages")
parser.add_option("-q",
"--quiet",
dest="quiet",
action="store_true",
default=False,
help="Display less messages")
parser.add_option(
"--catalog-release",
dest="catrel",
default="current",
help="A catalog release: current, unstable, testing, stable.")
parser.add_option(
"-r",
"--os-releases",
dest="osrel_commas",
help=("Comma separated list of ['SunOS5.9', 'SunOS5.10'], "
"e.g. 'SunOS5.9,SunOS5.10'."))
parser.add_option("-a",
"--catalog-architecture",
dest="arch",
help="Architecture: i386, sparc.")
parser.add_option("--profile",
dest="profile",
default=False,
action="store_true",
help="Enable profiling (a developer option).")
options, args = parser.parse_args()
assert len(args), "The list of files or md5 sums must be not empty."
logging_level = logging.INFO
if options.quiet:
logging_level = logging.WARNING
elif options.debug:
# If both flags are set, debug wins.
logging_level = logging.DEBUG
fmt = '%(levelname)s %(asctime)s %(filename)s:%(lineno)d %(message)s'
logging.basicConfig(format=fmt, level=logging_level)
logging.debug("Starting.")
configuration.SetUpSqlobjectConnection()
err_msg_list = []
if not options.osrel_commas:
err_msg_list.append("Please specify --os-releases.")
if not options.arch:
err_msg_list.append("Please specify --catalog-architecture.")
if options.arch not in cc.PHYSICAL_ARCHITECTURES:
err_msg_list.append(
"Valid --catalog-architecture values are: %s, you passed: %r" %
(cc.PHYSICAL_ARCHITECTURES, options.arch))
if err_msg_list:
raise UsageError(" ".join(err_msg_list))
md5_sums_from_files = []
collector = package_stats.StatsCollector(logger=logging,
debug=options.debug)
# We need to separate files and md5 sums.
md5_sums, file_list = [], []
for arg in args:
if struct_util.IsMd5(arg):
md5_sums.append(arg)
else:
file_list.append(arg)
config = configuration.GetConfig()
username, password = rest.GetUsernameAndPassword()
rest_client = rest.RestClient(pkgdb_url=config.get('rest', 'pkgdb'),
releases_url=config.get('rest', 'releases'),
username=username,
password=password)
if file_list:
def MakeEntry(file_name):
file_hash = hashlib.md5()
with open(file_name, "r") as fd:
chunk_size = 2 * 1024 * 1024
data = fd.read(chunk_size)
while data:
file_hash.update(data)
data = fd.read(chunk_size)
md5_sum = file_hash.hexdigest()
del file_hash
_, file_basename = os.path.split(file_name)
return {
'pkg_path': file_name,
'md5sum': md5_sum,
'file_basename': file_basename,
}
entries = [MakeEntry(x) for x in file_list]
md5_sums_from_files = collector.CollectStatsFromCatalogEntries(
entries, False)
for md5_sum in md5_sums_from_files:
if not rest_client.IsRegisteredLevelOne(md5_sum):
rest_client.RegisterLevelOne(md5_sum)
# We need the md5 sums of these files
md5_sums.extend(md5_sums_from_files)
assert md5_sums, "The list of md5 sums must not be empty."
logging.debug("md5_sums: %s", md5_sums)
osrel_list = options.osrel_commas.split(",")
logging.debug("Reading packages data from the database.")
# This part might need improvements in order to handle a whole
# catalog. On the other hand, if we already have the whole catalog in
# the database, we can do it altogether differently.
# Transforming the result to a list in order to force object
# retrieval.
sqo_pkgs = list(
models.Srv4FileStats.select(
sqlobject.IN(models.Srv4FileStats.q.md5_sum, md5_sums)))
tags_for_all_osrels = []
try:
sqo_catrel = models.CatalogRelease.selectBy(
name=options.catrel).getOne()
except sqlobject.main.SQLObjectNotFound as e:
logging.fatal("Fetching from the db has failed: catrel=%s",
repr(str(options.catrel)))
logging.fatal("Available catalog releases:")
sqo_catrels = models.CatalogRelease.select()
for sqo_catrel in sqo_catrels:
logging.fatal(" - %s", sqo_catrel.name)
raise
sqo_arch = models.Architecture.selectBy(name=options.arch).getOne()
for osrel in osrel_list:
sqo_osrel = models.OsRelease.selectBy(short_name=osrel).getOne()
VerifyContents(sqo_osrel, sqo_arch)
check_manager = checkpkg_lib.CheckpkgManager2(
CHECKPKG_MODULE_NAME,
sqo_pkgs,
osrel,
options.arch,
options.catrel,
debug=options.debug,
show_progress=(os.isatty(1) and not options.quiet))
# Running the checks, reporting and exiting.
exit_code, screen_report, tags_report = check_manager.Run()
screen_report = unicode(screen_report)
if not options.quiet and screen_report:
# TODO: Write this to screen only after overrides are applied.
sys.stdout.write(screen_report)
else:
logging.debug("No screen report.")
overrides_list = [list(pkg.GetOverridesResult()) for pkg in sqo_pkgs]
override_list = reduce(operator.add, overrides_list)
args = (sqo_osrel, sqo_arch, sqo_catrel)
tag_lists = [list(pkg.GetErrorTagsResult(*args)) for pkg in sqo_pkgs]
error_tags = reduce(operator.add, tag_lists)
(tags_after_overrides, unapplied_overrides) = overrides.ApplyOverrides(
error_tags, override_list)
tags_for_all_osrels.extend(tags_after_overrides)
if not options.quiet:
if tags_after_overrides:
print(textwrap.fill(BEFORE_OVERRIDES, 80))
for checkpkg_tag in tags_after_overrides:
print checkpkg_tag.ToGarSyntax()
print
for paragraph in AFTER_OVERRIDES:
print(textwrap.fill(paragraph, 80))
print
elif error_tags:
msg = (
'Fair enough, there were %d error tags, '
'but they were all overridden. '
"Just make sure you didn't override anything silly, like "
'sparc binaries in a i386 package.' % len(error_tags))
print
else:
print('Jolly good! All checks passed, no error tags reported.')
if unapplied_overrides:
print textwrap.fill(UNAPPLIED_OVERRIDES, 80)
for override in unapplied_overrides:
print u"* Unused %s" % override
exit_code = bool(tags_for_all_osrels)
sys.exit(exit_code)
def main():
parser = optparse.OptionParser(USAGE)
parser.add_option("-d",
"--debug",
dest="debug",
default=False,
action="store_true",
help="Turn on debugging messages")
parser.add_option(
"-t",
"--pkg-review-template",
dest="pkg_review_template",
help="A Cheetah template used for package review reports.")
parser.add_option("-r",
"--os-release",
dest="osrel",
default="SunOS5.10",
help="E.g. SunOS5.10")
parser.add_option("-a",
"--arch",
dest="arch",
default="sparc",
help="'i386' or 'sparc'")
parser.add_option("-c",
"--catalog-release",
dest="catrel",
default="unstable",
help="E.g. unstable, dublin")
parser.add_option("--replace",
dest="replace",
default=False,
action="store_true",
help="Replace packages when importing (importpkg)")
parser.add_option("--profile",
dest="profile",
default=False,
action="store_true",
help="Turn on profiling")
parser.add_option("--force-unpack",
dest="force_unpack",
default=False,
action="store_true",
help="Force unpacking of packages")
options, args = parser.parse_args()
logging_level = logging.INFO
if options.debug:
logging_level = logging.DEBUG
fmt = '%(levelname)s %(asctime)s %(filename)s:%(lineno)d %(message)s'
logging.basicConfig(format=fmt, level=logging_level)
if not args:
raise UsageError("Please specify a command. See --help.")
# SetUpSqlobjectConnection needs to be called after
# logging.basicConfig
configuration.SetUpSqlobjectConnection()
command = args[0]
args = args[1:]
if command == 'show':
subcommand = args[0]
args = args[1:]
elif command == 'pkg':
subcommand = args[0]
args = args[1:]
else:
subcommand = None
md5_sums = args
if (command, subcommand) == ('show', 'errors'):
for md5_sum in md5_sums:
srv4 = GetPkg(md5_sum)
res = m.CheckpkgErrorTag.select(
m.CheckpkgErrorTag.q.srv4_file == srv4)
for row in res:
print 'overridden' if row.overridden else 'active',
print row.pkgname, row.tag_name, row.tag_info, row.catrel.name, row.arch.name,
print row.os_rel.short_name
elif (command, subcommand) == ('show', 'overrides'):
for md5_sum in md5_sums:
srv4 = GetPkg(md5_sum)
res = m.CheckpkgOverride.select(
m.CheckpkgOverride.q.srv4_file == srv4)
for row in res:
print row.pkgname, row.tag_name, row.tag_info
elif (command, subcommand) == ('show', 'pkg'):
for md5_sum in md5_sums:
srv4 = GetPkg(md5_sum)
t = Template(SHOW_PKG_TMPL, searchList=[srv4])
sys.stdout.write(unicode(t))
elif command == 'gen-html':
config = configuration.GetConfig()
username, password = rest.GetUsernameAndPassword()
rest_client = rest.RestClient(pkgdb_url=config.get('rest', 'pkgdb'),
releases_url=config.get(
'rest', 'releases'),
username=username,
password=password,
debug=options.debug)
g = HtmlGenerator(md5_sums, options.pkg_review_template, rest_client,
options.debug)
sys.stdout.write(g.GenerateHtml())
elif command == 'initdb':
config = configuration.GetConfig()
database.InitDB(config)
elif command == 'importpkg':
collector = package_stats.StatsCollector(logger=logging,
debug=options.debug)
file_list = args
catalog_entries = []
for file_name in file_list:
file_hash = hashlib.md5()
chunk_size = 2 * 1024 * 1024
with open(file_name, 'rb') as fd:
data = fd.read(chunk_size)
while data:
file_hash.update(data)
data = fd.read(chunk_size)
data_md5_sum = file_hash.hexdigest()
catalog_entry = {
'md5sum': data_md5_sum,
'file_basename': os.path.basename(file_name),
'pkg_path': file_name,
}
catalog_entries.append(catalog_entry)
md5_list = collector.CollectStatsFromCatalogEntries(
catalog_entries, force_unpack=options.force_unpack)
config = configuration.GetConfig()
rest_client = rest.RestClient(pkgdb_url=config.get('rest', 'pkgdb'),
releases_url=config.get(
'rest', 'releases'),
debug=options.debug)
for md5_sum in md5_list:
logging.debug("Importing %s", md5_sum)
rest_client.RegisterLevelTwo(md5_sum)
elif command == 'removepkg':
for md5_sum in md5_sums:
srv4 = GetPkg(md5_sum)
in_catalogs = list(srv4.in_catalogs)
if in_catalogs:
for in_catalog in in_catalogs:
logging.warning("%s", in_catalog)
logging.warning(
"Not removing from the database, because the package "
"in question is part of at least one catalog.")
else:
logging.info("Removing %s", srv4)
srv4.DeleteAllDependentObjects()
srv4.destroySelf()
elif command == 'add-to-cat':
if len(args) < 4:
raise UsageError("Not enough arguments, see usage.")
user = getpass.getuser()
osrel, arch, catrel = args[:3]
username, password = rest.GetUsernameAndPassword()
rest_client = rest.RestClient(username=username, password=password)
md5_sums = args[3:]
for md5_sum in md5_sums:
rest_client.AddSvr4ToCatalog(catrel, arch, osrel, md5_sum)
elif command == 'del-from-cat':
if len(args) < 4:
raise UsageError("Not enough arguments, see usage.")
osrel, arch, catrel = args[:3]
md5_sums = args[3:]
username, password = rest.GetUsernameAndPassword()
rest_client = rest.RestClient(username=username, password=password)
for md5_sum in md5_sums:
rest_client.RemoveSvr4FromCatalog(catrel, arch, osrel, md5_sum)
elif command == 'system-metadata-to-disk':
logging.debug("Args: %s", args)
outfile = None
infile_contents = common_constants.DEFAULT_INSTALL_CONTENTS_FILE
infile_pkginfo = None
osrel, arch = (None, None)
if len(args) >= 2:
infile_contents = args[0]
infile_pkginfo = args[1]
if len(args) >= 3:
outfile = args[2]
if len(args) >= 4:
if len(args) == 5:
osrel, arch = args[3:5]
else:
raise UsageError("Wrong number of arguments (%s), see usage." %
len(args))
spi = system_pkgmap.Indexer(outfile, infile_contents, infile_pkginfo,
osrel, arch)
spi.IndexAndSave()
elif command == 'import-system-metadata':
if len(args) < 2:
raise UsageError(
"Usage: ... import-system-metadata <osrel> <arch>")
osrel = args[0]
arch = args[1]
importer = system_pkgmap.InstallContentsImporter(osrel,
arch,
debug=options.debug)
importer.Import(show_progress=(not options.debug))
elif (command, subcommand) == ('pkg', 'search'):
logging.debug("Searching for %s", args)
sqo_osrel = m.OsRelease.selectBy(short_name=options.osrel).getOne()
sqo_arch = m.Architecture.selectBy(name=options.arch).getOne()
sqo_catrel = m.CatalogRelease.selectBy(name=options.catrel).getOne()
if len(args) < 1:
logging.fatal("Wrong number of arguments: %s", len(args))
raise SystemExit
for catalogname in args:
join = [
sqlbuilder.INNERJOINOn(
None, m.Srv4FileInCatalog,
m.Srv4FileInCatalog.q.srv4file == m.Srv4FileStats.q.id),
]
res = m.Srv4FileStats.select(
sqlobject.AND(
m.Srv4FileInCatalog.q.osrel == sqo_osrel,
m.Srv4FileInCatalog.q.arch == sqo_arch,
m.Srv4FileInCatalog.q.catrel == sqo_catrel,
m.Srv4FileStats.q.catalogname.contains(catalogname),
m.Srv4FileStats.q.use_to_generate_catalogs == True),
join=join,
).orderBy("catalogname")
for sqo_srv4 in res:
print "%s %s" % (sqo_srv4.basename, sqo_srv4.md5_sum)
elif command == 'sync-cat-from-file':
if len(args) != 4:
raise UsageError("Wrong number of arguments, see usage.")
osrel, arch, catrel, catalog_file = args
ci = CatalogImporter(debug=options.debug)
ci.SyncFromCatalogFile(osrel, arch, catrel, catalog_file)
elif command == 'sync-catalogs-from-tree':
if len(args) != 2:
raise UsageError("Wrong number of arguments, see usage.")
ci = CatalogImporter(debug=options.debug)
catrel, base_dir = args
ci.SyncFromCatalogTree(catrel, base_dir, options.force_unpack)
elif (command, subcommand) == ('show', 'cat'):
sqo_osrel, sqo_arch, sqo_catrel = m.GetSqoTriad(
options.osrel, options.arch, options.catrel)
res = m.GetCatPackagesResult(sqo_osrel, sqo_arch, sqo_catrel)
for obj in res:
print obj.catalogname, obj.basename, obj.md5_sum
elif (command, subcommand) == ('show', 'files'):
md5_sum = args[0]
join = [
sqlbuilder.INNERJOINOn(
None, m.Srv4FileStats,
m.CswFile.q.srv4_file == m.Srv4FileStats.q.id),
]
res = m.CswFile.select(
m.Srv4FileStats.q.md5_sum == md5_sum,
join=join,
)
for obj in res:
print os.path.join(obj.path, obj.basename)
elif (command, subcommand) == ('show', 'basename'):
db_catalog = checkpkg_lib.Catalog()
for arg in args:
pkgs_by_path = db_catalog.GetPathsAndPkgnamesByBasename(
arg, options.osrel, options.arch, options.catrel)
for file_path in pkgs_by_path:
print os.path.join(file_path,
arg), ", ".join(pkgs_by_path[file_path])
elif (command, subcommand) == ('show', 'filename'):
db_catalog = checkpkg_lib.Catalog()
for arg in args:
pkgs = db_catalog.GetPkgByPath(arg, options.osrel, options.arch,
options.catrel)
print " ".join(pkgs)
else:
raise UsageError("Command unrecognized: %s" % command)
def main():
parser = optparse.OptionParser()
parser.add_option(
"--from-catalog",
dest="catrel_from",
default="unstable",
help="Catalog release to integrate from, e.g. 'unstable'.")
parser.add_option("--to-catalog",
dest="catrel_to",
default="testing",
help="Catalog release to integrate to, e.g. 'testing'.")
parser.add_option(
"--from-json",
dest="from_json",
help=("If specified, loads data from a JSON file instead of polling "
"the database."))
parser.add_option("--save-json",
dest="save_json",
help="If specified, saves JSON data to a file.")
parser.add_option("-o",
"--output-file",
dest="output_file",
help="Filename to save output to.")
parser.add_option("--no-include-downgrades",
dest="include_downgrades",
default=True,
action="store_false",
help="Skip package downgrades.")
parser.add_option(
"--no-include-version-changes",
dest="include_version_changes",
default=True,
action="store_false",
help="Skip version upgrades (only accept revision upgrades).")
parser.add_option("--debug",
dest="debug",
default=False,
action="store_true")
options, args = parser.parse_args()
logging_level = logging.INFO
if options.debug:
logging_level = logging.DEBUG
fmt = '%(levelname)s %(asctime)s %(filename)s:%(lineno)d %(message)s'
logging.basicConfig(format=fmt, level=logging_level)
config = configuration.GetConfig()
username, password = rest.GetUsernameAndPassword()
rest_client = rest.RestClient(pkgdb_url=config.get('rest', 'pkgdb'),
releases_url=config.get('rest', 'releases'),
username=username,
password=password)
if not options.output_file:
raise UsageError("Please specify the output file. See --help.")
catrel_from = options.catrel_from
catrel_to = options.catrel_to
if options.from_json:
with open(options.from_json, "rb") as fd:
logging.info("Loading %s", options.from_json)
(bundles_by_md5, jsonable_catalogs,
diffs_by_catalogname) = cjson.decode(fd.read())
catalogs = dict((tuple(cjson.decode(x)), jsonable_catalogs[x])
for x in jsonable_catalogs)
else:
catalogs = GetCatalogs(catrel_from, catrel_to,
options.include_version_changes,
options.include_downgrades, rest_client)
diffs_by_catalogname = ComposeDiffsByCatalogname(
catalogs, catrel_from, catrel_to, options.include_version_changes,
options.include_downgrades)
bundles_by_md5 = {}
bundles_missing = set()
cp = rest.CachedPkgstats("pkgstats", rest_client)
for key in catalogs:
if catalogs[key]: # could be None
for pkg in catalogs[key]:
md5 = pkg["md5_sum"]
if md5 not in bundles_by_md5 and md5 not in bundles_missing:
stats = cp.GetPkgstats(md5)
bundle_key = "OPENCSW_BUNDLE"
if stats:
if bundle_key in stats["pkginfo"]:
bundles_by_md5[md5] = stats["pkginfo"][
bundle_key]
else:
logging.debug(
"%r (%r) does not have the bundle set",
stats["basic_stats"]["pkg_basename"], md5)
bundles_missing.add(md5)
# Here's a good place to calculate the mapping between catalognames and
# bundle names.
change_types = "new_pkgs", "removed_pkgs", "updated_pkgs"
bundles_by_catalogname = {}
for catalogname in diffs_by_catalogname:
l = bundles_by_catalogname.setdefault(catalogname, set())
for change_type in change_types:
if change_type in diffs_by_catalogname[catalogname]:
for change_info in diffs_by_catalogname[catalogname][
change_type]:
pkg = change_info[2]
if "to" in pkg:
md5s = [x["md5_sum"] for x in (pkg["from"], pkg["to"])]
else:
md5s = [pkg["md5_sum"]]
for md5 in md5s:
if md5 in bundles_by_md5:
l.add(bundles_by_md5[md5])
namespace = {
"bundles_by_catalogname": bundles_by_catalogname,
"bundles_by_md5": bundles_by_md5,
"diffs_by_catalogname": diffs_by_catalogname,
"catrel_to": catrel_to,
"catrel_from": catrel_from,
"prog": sys.argv[0],
}
if options.save_json:
with open(options.save_json, "wb") as fd:
jsonable_catalogs = dict(
(cjson.encode(x), catalogs[x]) for x in catalogs)
fd.write(
cjson.encode(
(bundles_by_md5, jsonable_catalogs, diffs_by_catalogname)))
t = Template.Template(CATALOG_MOD_TMPL, searchList=[namespace])
if options.output_file:
logging.info("Saving output to %s", options.output_file)
with open(options.output_file, "wb") as fd:
fd.write(unicode(t))
else:
sys.stdout.write(unicode(t))
def __init__(self, filename, rest_client):
self.filename = filename
self.d = anydbm.open("%s.db" % self.filename, "c")
config = configuration.GetConfig()
self.rest_client = rest_client
self.deps = anydbm.open("%s-deps.db" % self.filename, "c")