class PkgnamesAndPathsByBasename(object):
def GET(self, catrel, arch, osrel):
user_data = web.input()
try:
basename = user_data.basename
except AttributeError, e:
raise web.badrequest()
send_filename = (
'%s-%s-%s-%s-packages.txt'
% (catrel, arch, osrel, basename.replace('/', '-')))
db_catalog = checkpkg_lib.Catalog()
try:
data = db_catalog.GetPathsAndPkgnamesByBasename(
basename, osrel, arch, catrel)
except sqlobject.main.SQLObjectNotFound:
raise web.notfound()
web.header(
'Content-type',
'application/x-vnd.opencsw.pkg;type=pkgname-list')
web.header('Content-Disposition',
'attachment; filename=%s' % send_filename)
response = cjson.encode(data)
web.header('Content-Length', str(len(response)))
return response
def PUT(self, catrel_name, arch_name, osrel_name, md5_sum):
"""Adds package to a catalog.
When pycurl calls this function, it often hangs, waiting. A fix for that
is to add the 'Content-Length' header. However, it sometimes still gets
stuck and I don't know why.
"""
configuration.SetUpSqlobjectConnection()
if catrel_name != 'unstable':
# Updates via web are allowed only for the unstable catalog.
# We should return an error message instead.
raise web.notfound()
try:
if arch_name == 'all':
raise checkpkg_lib.CatalogDatabaseError(
"Cannot add to 'all' catalog.")
srv4 = models.Srv4FileStats.selectBy(md5_sum=md5_sum).getOne()
parsed_basename = opencsw.ParsePackageFileName(srv4.basename)
if parsed_basename["vendortag"] != "CSW":
raise checkpkg_lib.CatalogDatabaseError(
"Package vendor tag is %s instead of CSW." %
parsed_basename["vendortag"])
if not srv4.registered:
# Package needs to be registered for releases
stats = srv4.GetStatsStruct()
# This can throw CatalogDatabaseError if the db user doesn't have
# enough permissions.
package_stats.PackageStats.ImportPkg(stats, True)
srv4 = models.Srv4FileStats.selectBy(md5_sum=md5_sum).getOne()
c = checkpkg_lib.Catalog()
# See if there already is a package with that catalogname.
sqo_osrel, sqo_arch, sqo_catrel = pkgdb.GetSqoTriad(
osrel_name, arch_name, catrel_name)
res = c.GetConflictingSrv4ByCatalognameResult(
srv4, srv4.catalogname, sqo_osrel, sqo_arch, sqo_catrel)
if res.count() == 1:
# Removing old version of the package from the catalog
for pkg_in_catalog in res:
srv4_to_remove = pkg_in_catalog.srv4file
c.RemoveSrv4(srv4_to_remove, osrel_name, arch_name,
catrel_name)
c.AddSrv4ToCatalog(srv4, osrel_name, arch_name, catrel_name)
web.header('Content-type',
'application/x-vnd.opencsw.pkg;type=catalog-update')
response = json.dumps([
u"Added to catalog %s %s %s" %
(catrel_name, arch_name, osrel_name),
u"%s" % srv4.basename,
])
web.header('Content-Length', len(response))
return response
except (checkpkg_lib.CatalogDatabaseError,
sqlobject.dberrors.OperationalError), e:
web.header('Content-type',
'application/x-vnd.opencsw.pkg;type=error-message')
response = json.dumps({
"error_message": unicode(e),
})
web.header('Content-Length', len(response))
return response
def GET(self, catrel, arch, osrel):
user_data = web.input()
filename = user_data.filename
send_filename = ('%s-%s-%s-%s-packages.txt' %
(catrel, arch, osrel, filename.replace('/', '-')))
db_catalog = checkpkg_lib.Catalog()
try:
pkgs = db_catalog.GetPkgByPath(filename, osrel, arch, catrel)
except sqlobject.main.SQLObjectNotFound, e:
raise web.notfound()
def DELETE(self, catrel_name, arch_name, osrel_name, md5_sum):
configuration.SetUpSqlobjectConnection()
try:
srv4_to_remove = models.Srv4FileStats.selectBy(
md5_sum=md5_sum).getOne()
c = checkpkg_lib.Catalog()
c.RemoveSrv4(srv4_to_remove, osrel_name, arch_name, catrel_name)
except (sqlobject.main.SQLObjectNotFound,
sqlobject.dberrors.OperationalError), e:
# Some better error reporting would be good here.
raise web.internalerror()
class PkgnamesAndPathsByBasename(object):
def GET(self, catrel, arch, osrel):
user_data = web.input()
try:
basename = user_data.basename
except AttributeError, e:
raise web.badrequest()
send_filename = ('%s-%s-%s-%s-packages.txt' %
(catrel, arch, osrel, basename.replace('/', '-')))
db_catalog = checkpkg_lib.Catalog()
try:
data = db_catalog.GetPathsAndPkgnamesByBasename(
basename, osrel, arch, catrel)
except sqlobject.main.SQLObjectNotFound, e:
raise web.notfound()
def GET(self, catrel, arch, osrel):
user_data = web.input()
filename = user_data.filename
send_filename = (
'%s-%s-%s-%s-packages.txt'
% (catrel, arch, osrel, filename.replace('/', '-')))
db_catalog = checkpkg_lib.Catalog()
try:
pkgs = db_catalog.GetPkgByPath(filename, osrel, arch, catrel)
except sqlobject.main.SQLObjectNotFound:
raise web.notfound()
web.header('Content-type', 'application/x-vnd.opencsw.pkg;type=pkgname-list')
web.header('X-Rest-Info', 'I could tell you about the format, but I won\'t')
web.header('Content-Disposition',
'attachment; filename=%s' % send_filename)
return cjson.encode(sorted(pkgs))
def DELETE(self, catrel_name, arch_name, osrel_name, md5_sum):
configuration.SetUpSqlobjectConnection()
try:
if osrel_name not in common_constants.OS_RELS:
self.ReturnError("%s is not one of %s (OS releases)" %
(osrel_name, common_constants.OS_RELS))
if osrel_name in common_constants.OBSOLETE_OS_RELS:
self.ReturnError(
"package deletions from an obsolete OS release such as %s "
"are not allowed" % osrel_name)
srv4_to_remove = models.Srv4FileStats.selectBy(
md5_sum=md5_sum).getOne()
c = checkpkg_lib.Catalog()
c.RemoveSrv4(srv4_to_remove, osrel_name, arch_name, catrel_name)
except (sqlobject.main.SQLObjectNotFound,
sqlobject.dberrors.OperationalError), e:
self.ReturnError("An error occurred: %s" % e)
def _ComposePkgstats(self, data, include_prefixes=None, show_progress=True):
logging.debug("_ComposePkgstats()")
osrel = data["osrel"]
arch = data["arch"]
contents = data["contents"]
files_metadata = [representations.FileMetadata._make(x)
for x in data["files_metadata"]]
logging.debug("Creating an in-memory index of files metadata (mime types)")
metadata_by_file_name = dict(
(x.path, x) for x in files_metadata)
catalog = checkpkg_lib.Catalog()
srv4_files_to_catalog = set()
cleaned_pkgs = set()
pkgs_by_pkgname = {}
pbar = self._GetPbar(show_progress)
progressbar_divisor = int(len(contents) / 1000)
if progressbar_divisor < 1:
progressbar_divisor = 1
update_period = 1L
pbar.maxval = len(contents)
pbar.start()
count = itertools.count()
plc = fake_pkgstats_composer.PkgstatsListComposer(osrel, arch)
# Create a new class responsible for creating pkgstats lists.
# We might potentially run out of memory here.
for pkgmap_tuple in contents:
pkgmap_entry = representations.PkgmapEntry._make(pkgmap_tuple)
logging.debug("Pkgmap entry: %r", pkgmap_entry)
for pkgname in pkgmap_entry.pkgnames:
pkgname = self.SanitizeInstallContentsPkgname(pkgname)
if not self._SkipPrefix(pkgname, include_prefixes):
# This is a little messy. Some pkgmap_entry objects are the
# /opt/csw ones which don't have the metadata collected.
# We have to skip them.
if pkgmap_entry.path not in metadata_by_file_name:
continue
plc.AddPkgname(pkgname)
plc.AddFile(pkgname,
pkgmap_entry,
metadata_by_file_name[pkgmap_entry.path],
None, None)
i = count.next()
if not i % update_period and (i / progressbar_divisor) <= pbar.maxval:
pbar.update(i / progressbar_divisor)
pbar.finish()
return plc.GetPkgstats()
def main():
parser = optparse.OptionParser()
parser.add_option("--debug",
dest="debug",
default=False,
action="store_true")
parser.add_option("--catalog-release", dest="catrel", default="unstable")
parser.add_option("--arch", dest="arch", default="i386")
parser.add_option("--os-release", dest="osrel", default="SunOS5.10")
parser.add_option("--md5", dest="md5")
parser.add_option("--check-name", dest="check_name")
options, args = parser.parse_args()
if options.debug:
logging.basicConfig(level=logging.DEBUG)
else:
logging.basicConfig(level=logging.INFO)
configuration.SetUpSqlobjectConnection()
osrel = options.osrel
catrel = options.catrel
arch = options.arch
md5 = options.md5
config = configuration.GetConfig()
rest_client = rest.RestClient(pkgdb_url=config.get('rest', 'pkgdb'),
releases_url=config.get('rest', 'releases'))
logging.info("Fetching %s", md5)
pkgstats = rest_client.GetBlob('pkgstats', md5)
catalog = checkpkg_lib.Catalog()
check_function = getattr(package_checks, options.check_name)
examined_files_by_pkg = {}
check_interface = LoggingCheckInterface(osrel, arch, catrel, catalog,
examined_files_by_pkg)
messenger = checkpkg_lib.CheckpkgMessenger()
pkgstats['elfdump_info'] = checkpkg_lib.LazyElfinfo(rest_client)
check_function([pkgstats],
check_interface,
logger=logging,
messenger=messenger)
def main():
parser = optparse.OptionParser(USAGE)
parser.add_option("-d",
"--debug",
dest="debug",
default=False,
action="store_true",
help="Turn on debugging messages")
parser.add_option(
"-t",
"--pkg-review-template",
dest="pkg_review_template",
help="A Cheetah template used for package review reports.")
parser.add_option("-r",
"--os-release",
dest="osrel",
default="SunOS5.10",
help="E.g. SunOS5.10")
parser.add_option("-a",
"--arch",
dest="arch",
default="sparc",
help="'i386' or 'sparc'")
parser.add_option("-c",
"--catalog-release",
dest="catrel",
default="unstable",
help="E.g. unstable, dublin")
parser.add_option("--replace",
dest="replace",
default=False,
action="store_true",
help="Replace packages when importing (importpkg)")
parser.add_option("--profile",
dest="profile",
default=False,
action="store_true",
help="Turn on profiling")
parser.add_option("--force-unpack",
dest="force_unpack",
default=False,
action="store_true",
help="Force unpacking of packages")
options, args = parser.parse_args()
logging_level = logging.INFO
if options.debug:
logging_level = logging.DEBUG
fmt = '%(levelname)s %(asctime)s %(filename)s:%(lineno)d %(message)s'
logging.basicConfig(format=fmt, level=logging_level)
if not args:
raise UsageError("Please specify a command. See --help.")
# SetUpSqlobjectConnection needs to be called after
# logging.basicConfig
configuration.SetUpSqlobjectConnection()
command = args[0]
args = args[1:]
if command == 'show':
subcommand = args[0]
args = args[1:]
elif command == 'pkg':
subcommand = args[0]
args = args[1:]
else:
subcommand = None
md5_sums = args
if (command, subcommand) == ('show', 'errors'):
for md5_sum in md5_sums:
srv4 = GetPkg(md5_sum)
res = m.CheckpkgErrorTag.select(
m.CheckpkgErrorTag.q.srv4_file == srv4)
for row in res:
print 'overridden' if row.overridden else 'active',
print row.pkgname, row.tag_name, row.tag_info, row.catrel.name, row.arch.name,
print row.os_rel.short_name
elif (command, subcommand) == ('show', 'overrides'):
for md5_sum in md5_sums:
srv4 = GetPkg(md5_sum)
res = m.CheckpkgOverride.select(
m.CheckpkgOverride.q.srv4_file == srv4)
for row in res:
print row.pkgname, row.tag_name, row.tag_info
elif (command, subcommand) == ('show', 'pkg'):
for md5_sum in md5_sums:
srv4 = GetPkg(md5_sum)
t = Template(SHOW_PKG_TMPL, searchList=[srv4])
sys.stdout.write(unicode(t))
elif command == 'gen-html':
config = configuration.GetConfig()
username, password = rest.GetUsernameAndPassword()
rest_client = rest.RestClient(pkgdb_url=config.get('rest', 'pkgdb'),
releases_url=config.get(
'rest', 'releases'),
username=username,
password=password,
debug=options.debug)
g = HtmlGenerator(md5_sums, options.pkg_review_template, rest_client,
options.debug)
sys.stdout.write(g.GenerateHtml())
elif command == 'initdb':
config = configuration.GetConfig()
database.InitDB(config)
elif command == 'importpkg':
collector = package_stats.StatsCollector(logger=logging,
debug=options.debug)
file_list = args
catalog_entries = []
for file_name in file_list:
file_hash = hashlib.md5()
chunk_size = 2 * 1024 * 1024
with open(file_name, 'rb') as fd:
data = fd.read(chunk_size)
while data:
file_hash.update(data)
data = fd.read(chunk_size)
data_md5_sum = file_hash.hexdigest()
catalog_entry = {
'md5sum': data_md5_sum,
'file_basename': os.path.basename(file_name),
'pkg_path': file_name,
}
catalog_entries.append(catalog_entry)
md5_list = collector.CollectStatsFromCatalogEntries(
catalog_entries, force_unpack=options.force_unpack)
config = configuration.GetConfig()
rest_client = rest.RestClient(pkgdb_url=config.get('rest', 'pkgdb'),
releases_url=config.get(
'rest', 'releases'),
debug=options.debug)
for md5_sum in md5_list:
logging.debug("Importing %s", md5_sum)
rest_client.RegisterLevelTwo(md5_sum)
elif command == 'removepkg':
for md5_sum in md5_sums:
srv4 = GetPkg(md5_sum)
in_catalogs = list(srv4.in_catalogs)
if in_catalogs:
for in_catalog in in_catalogs:
logging.warning("%s", in_catalog)
logging.warning(
"Not removing from the database, because the package "
"in question is part of at least one catalog.")
else:
logging.info("Removing %s", srv4)
srv4.DeleteAllDependentObjects()
srv4.destroySelf()
elif command == 'add-to-cat':
if len(args) < 4:
raise UsageError("Not enough arguments, see usage.")
user = getpass.getuser()
osrel, arch, catrel = args[:3]
username, password = rest.GetUsernameAndPassword()
rest_client = rest.RestClient(username=username, password=password)
md5_sums = args[3:]
for md5_sum in md5_sums:
rest_client.AddSvr4ToCatalog(catrel, arch, osrel, md5_sum)
elif command == 'del-from-cat':
if len(args) < 4:
raise UsageError("Not enough arguments, see usage.")
osrel, arch, catrel = args[:3]
md5_sums = args[3:]
username, password = rest.GetUsernameAndPassword()
rest_client = rest.RestClient(username=username, password=password)
for md5_sum in md5_sums:
rest_client.RemoveSvr4FromCatalog(catrel, arch, osrel, md5_sum)
elif command == 'system-metadata-to-disk':
logging.debug("Args: %s", args)
outfile = None
infile_contents = common_constants.DEFAULT_INSTALL_CONTENTS_FILE
infile_pkginfo = None
osrel, arch = (None, None)
if len(args) >= 2:
infile_contents = args[0]
infile_pkginfo = args[1]
if len(args) >= 3:
outfile = args[2]
if len(args) >= 4:
if len(args) == 5:
osrel, arch = args[3:5]
else:
raise UsageError("Wrong number of arguments (%s), see usage." %
len(args))
spi = system_pkgmap.Indexer(outfile, infile_contents, infile_pkginfo,
osrel, arch)
spi.IndexAndSave()
elif command == 'import-system-metadata':
if len(args) < 2:
raise UsageError(
"Usage: ... import-system-metadata <osrel> <arch>")
osrel = args[0]
arch = args[1]
importer = system_pkgmap.InstallContentsImporter(osrel,
arch,
debug=options.debug)
importer.Import(show_progress=(not options.debug))
elif (command, subcommand) == ('pkg', 'search'):
logging.debug("Searching for %s", args)
sqo_osrel = m.OsRelease.selectBy(short_name=options.osrel).getOne()
sqo_arch = m.Architecture.selectBy(name=options.arch).getOne()
sqo_catrel = m.CatalogRelease.selectBy(name=options.catrel).getOne()
if len(args) < 1:
logging.fatal("Wrong number of arguments: %s", len(args))
raise SystemExit
for catalogname in args:
join = [
sqlbuilder.INNERJOINOn(
None, m.Srv4FileInCatalog,
m.Srv4FileInCatalog.q.srv4file == m.Srv4FileStats.q.id),
]
res = m.Srv4FileStats.select(
sqlobject.AND(
m.Srv4FileInCatalog.q.osrel == sqo_osrel,
m.Srv4FileInCatalog.q.arch == sqo_arch,
m.Srv4FileInCatalog.q.catrel == sqo_catrel,
m.Srv4FileStats.q.catalogname.contains(catalogname),
m.Srv4FileStats.q.use_to_generate_catalogs == True),
join=join,
).orderBy("catalogname")
for sqo_srv4 in res:
print "%s %s" % (sqo_srv4.basename, sqo_srv4.md5_sum)
elif command == 'sync-cat-from-file':
if len(args) != 4:
raise UsageError("Wrong number of arguments, see usage.")
osrel, arch, catrel, catalog_file = args
ci = CatalogImporter(debug=options.debug)
ci.SyncFromCatalogFile(osrel, arch, catrel, catalog_file)
elif command == 'sync-catalogs-from-tree':
if len(args) != 2:
raise UsageError("Wrong number of arguments, see usage.")
ci = CatalogImporter(debug=options.debug)
catrel, base_dir = args
ci.SyncFromCatalogTree(catrel, base_dir, options.force_unpack)
elif (command, subcommand) == ('show', 'cat'):
sqo_osrel, sqo_arch, sqo_catrel = m.GetSqoTriad(
options.osrel, options.arch, options.catrel)
res = m.GetCatPackagesResult(sqo_osrel, sqo_arch, sqo_catrel)
for obj in res:
print obj.catalogname, obj.basename, obj.md5_sum
elif (command, subcommand) == ('show', 'files'):
md5_sum = args[0]
join = [
sqlbuilder.INNERJOINOn(
None, m.Srv4FileStats,
m.CswFile.q.srv4_file == m.Srv4FileStats.q.id),
]
res = m.CswFile.select(
m.Srv4FileStats.q.md5_sum == md5_sum,
join=join,
)
for obj in res:
print os.path.join(obj.path, obj.basename)
elif (command, subcommand) == ('show', 'basename'):
db_catalog = checkpkg_lib.Catalog()
for arg in args:
pkgs_by_path = db_catalog.GetPathsAndPkgnamesByBasename(
arg, options.osrel, options.arch, options.catrel)
for file_path in pkgs_by_path:
print os.path.join(file_path,
arg), ", ".join(pkgs_by_path[file_path])
elif (command, subcommand) == ('show', 'filename'):
db_catalog = checkpkg_lib.Catalog()
for arg in args:
pkgs = db_catalog.GetPkgByPath(arg, options.osrel, options.arch,
options.catrel)
print " ".join(pkgs)
else:
raise UsageError("Command unrecognized: %s" % command)