class PkgnamesAndPathsByBasename(object): def GET(self, catrel, arch, osrel): user_data = web.input() try: basename = user_data.basename except AttributeError, e: raise web.badrequest() send_filename = ( '%s-%s-%s-%s-packages.txt' % (catrel, arch, osrel, basename.replace('/', '-'))) db_catalog = checkpkg_lib.Catalog() try: data = db_catalog.GetPathsAndPkgnamesByBasename( basename, osrel, arch, catrel) except sqlobject.main.SQLObjectNotFound: raise web.notfound() web.header( 'Content-type', 'application/x-vnd.opencsw.pkg;type=pkgname-list') web.header('Content-Disposition', 'attachment; filename=%s' % send_filename) response = cjson.encode(data) web.header('Content-Length', str(len(response))) return response
def PUT(self, catrel_name, arch_name, osrel_name, md5_sum): """Adds package to a catalog. When pycurl calls this function, it often hangs, waiting. A fix for that is to add the 'Content-Length' header. However, it sometimes still gets stuck and I don't know why. """ configuration.SetUpSqlobjectConnection() if catrel_name != 'unstable': # Updates via web are allowed only for the unstable catalog. # We should return an error message instead. raise web.notfound() try: if arch_name == 'all': raise checkpkg_lib.CatalogDatabaseError( "Cannot add to 'all' catalog.") srv4 = models.Srv4FileStats.selectBy(md5_sum=md5_sum).getOne() parsed_basename = opencsw.ParsePackageFileName(srv4.basename) if parsed_basename["vendortag"] != "CSW": raise checkpkg_lib.CatalogDatabaseError( "Package vendor tag is %s instead of CSW." % parsed_basename["vendortag"]) if not srv4.registered: # Package needs to be registered for releases stats = srv4.GetStatsStruct() # This can throw CatalogDatabaseError if the db user doesn't have # enough permissions. package_stats.PackageStats.ImportPkg(stats, True) srv4 = models.Srv4FileStats.selectBy(md5_sum=md5_sum).getOne() c = checkpkg_lib.Catalog() # See if there already is a package with that catalogname. sqo_osrel, sqo_arch, sqo_catrel = pkgdb.GetSqoTriad( osrel_name, arch_name, catrel_name) res = c.GetConflictingSrv4ByCatalognameResult( srv4, srv4.catalogname, sqo_osrel, sqo_arch, sqo_catrel) if res.count() == 1: # Removing old version of the package from the catalog for pkg_in_catalog in res: srv4_to_remove = pkg_in_catalog.srv4file c.RemoveSrv4(srv4_to_remove, osrel_name, arch_name, catrel_name) c.AddSrv4ToCatalog(srv4, osrel_name, arch_name, catrel_name) web.header('Content-type', 'application/x-vnd.opencsw.pkg;type=catalog-update') response = json.dumps([ u"Added to catalog %s %s %s" % (catrel_name, arch_name, osrel_name), u"%s" % srv4.basename, ]) web.header('Content-Length', len(response)) return response except (checkpkg_lib.CatalogDatabaseError, sqlobject.dberrors.OperationalError), e: web.header('Content-type', 'application/x-vnd.opencsw.pkg;type=error-message') response = json.dumps({ "error_message": unicode(e), }) web.header('Content-Length', len(response)) return response
def GET(self, catrel, arch, osrel): user_data = web.input() filename = user_data.filename send_filename = ('%s-%s-%s-%s-packages.txt' % (catrel, arch, osrel, filename.replace('/', '-'))) db_catalog = checkpkg_lib.Catalog() try: pkgs = db_catalog.GetPkgByPath(filename, osrel, arch, catrel) except sqlobject.main.SQLObjectNotFound, e: raise web.notfound()
def DELETE(self, catrel_name, arch_name, osrel_name, md5_sum): configuration.SetUpSqlobjectConnection() try: srv4_to_remove = models.Srv4FileStats.selectBy( md5_sum=md5_sum).getOne() c = checkpkg_lib.Catalog() c.RemoveSrv4(srv4_to_remove, osrel_name, arch_name, catrel_name) except (sqlobject.main.SQLObjectNotFound, sqlobject.dberrors.OperationalError), e: # Some better error reporting would be good here. raise web.internalerror()
class PkgnamesAndPathsByBasename(object): def GET(self, catrel, arch, osrel): user_data = web.input() try: basename = user_data.basename except AttributeError, e: raise web.badrequest() send_filename = ('%s-%s-%s-%s-packages.txt' % (catrel, arch, osrel, basename.replace('/', '-'))) db_catalog = checkpkg_lib.Catalog() try: data = db_catalog.GetPathsAndPkgnamesByBasename( basename, osrel, arch, catrel) except sqlobject.main.SQLObjectNotFound, e: raise web.notfound()
def GET(self, catrel, arch, osrel): user_data = web.input() filename = user_data.filename send_filename = ( '%s-%s-%s-%s-packages.txt' % (catrel, arch, osrel, filename.replace('/', '-'))) db_catalog = checkpkg_lib.Catalog() try: pkgs = db_catalog.GetPkgByPath(filename, osrel, arch, catrel) except sqlobject.main.SQLObjectNotFound: raise web.notfound() web.header('Content-type', 'application/x-vnd.opencsw.pkg;type=pkgname-list') web.header('X-Rest-Info', 'I could tell you about the format, but I won\'t') web.header('Content-Disposition', 'attachment; filename=%s' % send_filename) return cjson.encode(sorted(pkgs))
def DELETE(self, catrel_name, arch_name, osrel_name, md5_sum): configuration.SetUpSqlobjectConnection() try: if osrel_name not in common_constants.OS_RELS: self.ReturnError("%s is not one of %s (OS releases)" % (osrel_name, common_constants.OS_RELS)) if osrel_name in common_constants.OBSOLETE_OS_RELS: self.ReturnError( "package deletions from an obsolete OS release such as %s " "are not allowed" % osrel_name) srv4_to_remove = models.Srv4FileStats.selectBy( md5_sum=md5_sum).getOne() c = checkpkg_lib.Catalog() c.RemoveSrv4(srv4_to_remove, osrel_name, arch_name, catrel_name) except (sqlobject.main.SQLObjectNotFound, sqlobject.dberrors.OperationalError), e: self.ReturnError("An error occurred: %s" % e)
def _ComposePkgstats(self, data, include_prefixes=None, show_progress=True): logging.debug("_ComposePkgstats()") osrel = data["osrel"] arch = data["arch"] contents = data["contents"] files_metadata = [representations.FileMetadata._make(x) for x in data["files_metadata"]] logging.debug("Creating an in-memory index of files metadata (mime types)") metadata_by_file_name = dict( (x.path, x) for x in files_metadata) catalog = checkpkg_lib.Catalog() srv4_files_to_catalog = set() cleaned_pkgs = set() pkgs_by_pkgname = {} pbar = self._GetPbar(show_progress) progressbar_divisor = int(len(contents) / 1000) if progressbar_divisor < 1: progressbar_divisor = 1 update_period = 1L pbar.maxval = len(contents) pbar.start() count = itertools.count() plc = fake_pkgstats_composer.PkgstatsListComposer(osrel, arch) # Create a new class responsible for creating pkgstats lists. # We might potentially run out of memory here. for pkgmap_tuple in contents: pkgmap_entry = representations.PkgmapEntry._make(pkgmap_tuple) logging.debug("Pkgmap entry: %r", pkgmap_entry) for pkgname in pkgmap_entry.pkgnames: pkgname = self.SanitizeInstallContentsPkgname(pkgname) if not self._SkipPrefix(pkgname, include_prefixes): # This is a little messy. Some pkgmap_entry objects are the # /opt/csw ones which don't have the metadata collected. # We have to skip them. if pkgmap_entry.path not in metadata_by_file_name: continue plc.AddPkgname(pkgname) plc.AddFile(pkgname, pkgmap_entry, metadata_by_file_name[pkgmap_entry.path], None, None) i = count.next() if not i % update_period and (i / progressbar_divisor) <= pbar.maxval: pbar.update(i / progressbar_divisor) pbar.finish() return plc.GetPkgstats()
def main(): parser = optparse.OptionParser() parser.add_option("--debug", dest="debug", default=False, action="store_true") parser.add_option("--catalog-release", dest="catrel", default="unstable") parser.add_option("--arch", dest="arch", default="i386") parser.add_option("--os-release", dest="osrel", default="SunOS5.10") parser.add_option("--md5", dest="md5") parser.add_option("--check-name", dest="check_name") options, args = parser.parse_args() if options.debug: logging.basicConfig(level=logging.DEBUG) else: logging.basicConfig(level=logging.INFO) configuration.SetUpSqlobjectConnection() osrel = options.osrel catrel = options.catrel arch = options.arch md5 = options.md5 config = configuration.GetConfig() rest_client = rest.RestClient(pkgdb_url=config.get('rest', 'pkgdb'), releases_url=config.get('rest', 'releases')) logging.info("Fetching %s", md5) pkgstats = rest_client.GetBlob('pkgstats', md5) catalog = checkpkg_lib.Catalog() check_function = getattr(package_checks, options.check_name) examined_files_by_pkg = {} check_interface = LoggingCheckInterface(osrel, arch, catrel, catalog, examined_files_by_pkg) messenger = checkpkg_lib.CheckpkgMessenger() pkgstats['elfdump_info'] = checkpkg_lib.LazyElfinfo(rest_client) check_function([pkgstats], check_interface, logger=logging, messenger=messenger)
def main(): parser = optparse.OptionParser(USAGE) parser.add_option("-d", "--debug", dest="debug", default=False, action="store_true", help="Turn on debugging messages") parser.add_option( "-t", "--pkg-review-template", dest="pkg_review_template", help="A Cheetah template used for package review reports.") parser.add_option("-r", "--os-release", dest="osrel", default="SunOS5.10", help="E.g. SunOS5.10") parser.add_option("-a", "--arch", dest="arch", default="sparc", help="'i386' or 'sparc'") parser.add_option("-c", "--catalog-release", dest="catrel", default="unstable", help="E.g. unstable, dublin") parser.add_option("--replace", dest="replace", default=False, action="store_true", help="Replace packages when importing (importpkg)") parser.add_option("--profile", dest="profile", default=False, action="store_true", help="Turn on profiling") parser.add_option("--force-unpack", dest="force_unpack", default=False, action="store_true", help="Force unpacking of packages") options, args = parser.parse_args() logging_level = logging.INFO if options.debug: logging_level = logging.DEBUG fmt = '%(levelname)s %(asctime)s %(filename)s:%(lineno)d %(message)s' logging.basicConfig(format=fmt, level=logging_level) if not args: raise UsageError("Please specify a command. See --help.") # SetUpSqlobjectConnection needs to be called after # logging.basicConfig configuration.SetUpSqlobjectConnection() command = args[0] args = args[1:] if command == 'show': subcommand = args[0] args = args[1:] elif command == 'pkg': subcommand = args[0] args = args[1:] else: subcommand = None md5_sums = args if (command, subcommand) == ('show', 'errors'): for md5_sum in md5_sums: srv4 = GetPkg(md5_sum) res = m.CheckpkgErrorTag.select( m.CheckpkgErrorTag.q.srv4_file == srv4) for row in res: print 'overridden' if row.overridden else 'active', print row.pkgname, row.tag_name, row.tag_info, row.catrel.name, row.arch.name, print row.os_rel.short_name elif (command, subcommand) == ('show', 'overrides'): for md5_sum in md5_sums: srv4 = GetPkg(md5_sum) res = m.CheckpkgOverride.select( m.CheckpkgOverride.q.srv4_file == srv4) for row in res: print row.pkgname, row.tag_name, row.tag_info elif (command, subcommand) == ('show', 'pkg'): for md5_sum in md5_sums: srv4 = GetPkg(md5_sum) t = Template(SHOW_PKG_TMPL, searchList=[srv4]) sys.stdout.write(unicode(t)) elif command == 'gen-html': config = configuration.GetConfig() username, password = rest.GetUsernameAndPassword() rest_client = rest.RestClient(pkgdb_url=config.get('rest', 'pkgdb'), releases_url=config.get( 'rest', 'releases'), username=username, password=password, debug=options.debug) g = HtmlGenerator(md5_sums, options.pkg_review_template, rest_client, options.debug) sys.stdout.write(g.GenerateHtml()) elif command == 'initdb': config = configuration.GetConfig() database.InitDB(config) elif command == 'importpkg': collector = package_stats.StatsCollector(logger=logging, debug=options.debug) file_list = args catalog_entries = [] for file_name in file_list: file_hash = hashlib.md5() chunk_size = 2 * 1024 * 1024 with open(file_name, 'rb') as fd: data = fd.read(chunk_size) while data: file_hash.update(data) data = fd.read(chunk_size) data_md5_sum = file_hash.hexdigest() catalog_entry = { 'md5sum': data_md5_sum, 'file_basename': os.path.basename(file_name), 'pkg_path': file_name, } catalog_entries.append(catalog_entry) md5_list = collector.CollectStatsFromCatalogEntries( catalog_entries, force_unpack=options.force_unpack) config = configuration.GetConfig() rest_client = rest.RestClient(pkgdb_url=config.get('rest', 'pkgdb'), releases_url=config.get( 'rest', 'releases'), debug=options.debug) for md5_sum in md5_list: logging.debug("Importing %s", md5_sum) rest_client.RegisterLevelTwo(md5_sum) elif command == 'removepkg': for md5_sum in md5_sums: srv4 = GetPkg(md5_sum) in_catalogs = list(srv4.in_catalogs) if in_catalogs: for in_catalog in in_catalogs: logging.warning("%s", in_catalog) logging.warning( "Not removing from the database, because the package " "in question is part of at least one catalog.") else: logging.info("Removing %s", srv4) srv4.DeleteAllDependentObjects() srv4.destroySelf() elif command == 'add-to-cat': if len(args) < 4: raise UsageError("Not enough arguments, see usage.") user = getpass.getuser() osrel, arch, catrel = args[:3] username, password = rest.GetUsernameAndPassword() rest_client = rest.RestClient(username=username, password=password) md5_sums = args[3:] for md5_sum in md5_sums: rest_client.AddSvr4ToCatalog(catrel, arch, osrel, md5_sum) elif command == 'del-from-cat': if len(args) < 4: raise UsageError("Not enough arguments, see usage.") osrel, arch, catrel = args[:3] md5_sums = args[3:] username, password = rest.GetUsernameAndPassword() rest_client = rest.RestClient(username=username, password=password) for md5_sum in md5_sums: rest_client.RemoveSvr4FromCatalog(catrel, arch, osrel, md5_sum) elif command == 'system-metadata-to-disk': logging.debug("Args: %s", args) outfile = None infile_contents = common_constants.DEFAULT_INSTALL_CONTENTS_FILE infile_pkginfo = None osrel, arch = (None, None) if len(args) >= 2: infile_contents = args[0] infile_pkginfo = args[1] if len(args) >= 3: outfile = args[2] if len(args) >= 4: if len(args) == 5: osrel, arch = args[3:5] else: raise UsageError("Wrong number of arguments (%s), see usage." % len(args)) spi = system_pkgmap.Indexer(outfile, infile_contents, infile_pkginfo, osrel, arch) spi.IndexAndSave() elif command == 'import-system-metadata': if len(args) < 2: raise UsageError( "Usage: ... import-system-metadata <osrel> <arch>") osrel = args[0] arch = args[1] importer = system_pkgmap.InstallContentsImporter(osrel, arch, debug=options.debug) importer.Import(show_progress=(not options.debug)) elif (command, subcommand) == ('pkg', 'search'): logging.debug("Searching for %s", args) sqo_osrel = m.OsRelease.selectBy(short_name=options.osrel).getOne() sqo_arch = m.Architecture.selectBy(name=options.arch).getOne() sqo_catrel = m.CatalogRelease.selectBy(name=options.catrel).getOne() if len(args) < 1: logging.fatal("Wrong number of arguments: %s", len(args)) raise SystemExit for catalogname in args: join = [ sqlbuilder.INNERJOINOn( None, m.Srv4FileInCatalog, m.Srv4FileInCatalog.q.srv4file == m.Srv4FileStats.q.id), ] res = m.Srv4FileStats.select( sqlobject.AND( m.Srv4FileInCatalog.q.osrel == sqo_osrel, m.Srv4FileInCatalog.q.arch == sqo_arch, m.Srv4FileInCatalog.q.catrel == sqo_catrel, m.Srv4FileStats.q.catalogname.contains(catalogname), m.Srv4FileStats.q.use_to_generate_catalogs == True), join=join, ).orderBy("catalogname") for sqo_srv4 in res: print "%s %s" % (sqo_srv4.basename, sqo_srv4.md5_sum) elif command == 'sync-cat-from-file': if len(args) != 4: raise UsageError("Wrong number of arguments, see usage.") osrel, arch, catrel, catalog_file = args ci = CatalogImporter(debug=options.debug) ci.SyncFromCatalogFile(osrel, arch, catrel, catalog_file) elif command == 'sync-catalogs-from-tree': if len(args) != 2: raise UsageError("Wrong number of arguments, see usage.") ci = CatalogImporter(debug=options.debug) catrel, base_dir = args ci.SyncFromCatalogTree(catrel, base_dir, options.force_unpack) elif (command, subcommand) == ('show', 'cat'): sqo_osrel, sqo_arch, sqo_catrel = m.GetSqoTriad( options.osrel, options.arch, options.catrel) res = m.GetCatPackagesResult(sqo_osrel, sqo_arch, sqo_catrel) for obj in res: print obj.catalogname, obj.basename, obj.md5_sum elif (command, subcommand) == ('show', 'files'): md5_sum = args[0] join = [ sqlbuilder.INNERJOINOn( None, m.Srv4FileStats, m.CswFile.q.srv4_file == m.Srv4FileStats.q.id), ] res = m.CswFile.select( m.Srv4FileStats.q.md5_sum == md5_sum, join=join, ) for obj in res: print os.path.join(obj.path, obj.basename) elif (command, subcommand) == ('show', 'basename'): db_catalog = checkpkg_lib.Catalog() for arg in args: pkgs_by_path = db_catalog.GetPathsAndPkgnamesByBasename( arg, options.osrel, options.arch, options.catrel) for file_path in pkgs_by_path: print os.path.join(file_path, arg), ", ".join(pkgs_by_path[file_path]) elif (command, subcommand) == ('show', 'filename'): db_catalog = checkpkg_lib.Catalog() for arg in args: pkgs = db_catalog.GetPkgByPath(arg, options.osrel, options.arch, options.catrel) print " ".join(pkgs) else: raise UsageError("Command unrecognized: %s" % command)