def GetPkgByPath(self, full_file_path, osrel, arch, catrel): """Returns a list of packages.""" # Memoization for performance # Memoization won't buy us much. Perhaps we can fetch all the files # belonging to the same package, so that we quickly prepopulate the cache. key = (full_file_path, osrel, arch, catrel) if key not in self.pkgs_by_path_cache: file_path, basename = os.path.split(full_file_path) sqo_osrel, sqo_arch, sqo_catrel = self.GetSqlobjectTriad( osrel, arch, catrel) oac = sqlobject.AND(m.Srv4FileInCatalog.q.osrel == sqo_osrel, m.Srv4FileInCatalog.q.arch == sqo_arch, m.Srv4FileInCatalog.q.catrel == sqo_catrel) path_filter = sqlobject.AND(oac, m.CswFile.q.path == file_path, m.CswFile.q.basename == basename, m.Srv4FileStats.q.registered == True) join = [ sqlbuilder.INNERJOINOn( None, m.Srv4FileStats, m.Pkginst.q.id == m.Srv4FileStats.q.pkginst), sqlbuilder.INNERJOINOn( None, m.Srv4FileInCatalog, m.Srv4FileStats.q.id == m.Srv4FileInCatalog.q.srv4file), sqlbuilder.INNERJOINOn( None, m.CswFile, m.Srv4FileStats.q.id == m.CswFile.q.srv4_file), ] res = m.Pkginst.select(path_filter, join=join) pkgs = [x.pkgname for x in res] self.pkgs_by_path_cache[key] = frozenset(pkgs) return self.pkgs_by_path_cache[key]
def GetPkgByPath(self, full_file_path, osrel, arch, catrel): """Returns a list of packages.""" # TODO(maciej): Move this to models.py and have pkgdb_web return the JSON # structure. This is a step towards RESTification. key = (full_file_path, osrel, arch, catrel) if key not in self.pkgs_by_path_cache: file_path, basename = os.path.split(full_file_path) sqo_osrel, sqo_arch, sqo_catrel = self.GetSqlobjectTriad( osrel, arch, catrel) oac = sqlobject.AND( m.Srv4FileInCatalog.q.osrel==sqo_osrel, m.Srv4FileInCatalog.q.arch==sqo_arch, m.Srv4FileInCatalog.q.catrel==sqo_catrel) path_filter = sqlobject.AND( oac, m.CswFile.q.path==file_path, m.CswFile.q.basename==basename, m.Srv4FileStats.q.registered_level_two==True) join = [ sqlbuilder.INNERJOINOn(None, m.Srv4FileStats, m.Pkginst.q.id==m.Srv4FileStats.q.pkginst), sqlbuilder.INNERJOINOn(None, m.Srv4FileInCatalog, m.Srv4FileStats.q.id==m.Srv4FileInCatalog.q.srv4file), sqlbuilder.INNERJOINOn(None, m.CswFile, m.Srv4FileStats.q.id==m.CswFile.q.srv4_file), ] res = m.Pkginst.select(path_filter, join=join) pkgs = [x.pkgname for x in res] self.pkgs_by_path_cache[key] = frozenset(pkgs) return self.pkgs_by_path_cache[key]
def GET(self, catrel_name, arch_name, osrel_name, pkgname): """Get a srv4 reference by catalog ane pkgname.""" configuration.SetUpSqlobjectConnection() sqo_osrel, sqo_arch, sqo_catrel = pkgdb.GetSqoTriad( osrel_name, arch_name, catrel_name) join = [ sqlbuilder.INNERJOINOn( None, models.Srv4FileInCatalog, models.Srv4FileInCatalog.q.srv4file == models.Srv4FileStats.q.id), sqlbuilder.INNERJOINOn( None, models.Pkginst, models.Pkginst.q.id == models.Srv4FileStats.q.pkginst), ] res = models.Srv4FileStats.select( sqlobject.AND( models.Srv4FileInCatalog.q.osrel == sqo_osrel, models.Srv4FileInCatalog.q.arch == sqo_arch, models.Srv4FileInCatalog.q.catrel == sqo_catrel, models.Pkginst.q.pkgname == pkgname, models.Srv4FileStats.q.use_to_generate_catalogs == True), join=join, ) try: srv4 = res.getOne() mimetype, data = srv4.GetRestRepr() web.header('Content-type', mimetype) web.header('Access-Control-Allow-Origin', '*') return cjson.encode(data) except sqlobject.main.SQLObjectNotFound: return cjson.encode(None) except sqlobject.dberrors.OperationalError, e: raise web.internalerror(e)
def GetPathsAndPkgnamesByBasedir(self, basedir, osrel, arch, catrel): sqo_osrel, sqo_arch, sqo_catrel = self.GetSqlobjectTriad( osrel, arch, catrel) connection = m.CswFile._connection join = [ sqlbuilder.INNERJOINOn(None, m.Pkginst, m.CswFile.q.pkginst==m.Pkginst.q.id), sqlbuilder.INNERJOINOn(None, m.Srv4FileStats, m.CswFile.q.srv4_file==m.Srv4FileStats.q.id), sqlbuilder.INNERJOINOn(None, m.Srv4FileInCatalog, m.Srv4FileStats.q.id==m.Srv4FileInCatalog.q.srv4file), ] where = sqlobject.AND( m.CswFile.q.path==basedir, m.Srv4FileInCatalog.q.osrel==sqo_osrel, m.Srv4FileInCatalog.q.arch==sqo_arch, m.Srv4FileInCatalog.q.catrel==sqo_catrel, ) query = connection.sqlrepr( sqlbuilder.Select( [m.CswFile.q.basename, m.Pkginst.q.pkgname], where=where, join=join)) rows = connection.queryAll(query) pkgs = {} for row in rows: basename, pkginst = row pkgs.setdefault(pkginst, []).append(basename) return pkgs
def GetPathsAndPkgnamesByBasename(self, basename, osrel, arch, catrel): """Retrieves pkginst names of packages that have certain files. Since it needs to match a specific catalog, a table join is required: - CswFile (basename) - related Srv4FileStats - related Srv4FileInCatalog Args: basename: u'libfoo.so.1' osrel: u'5.9' arch: 'sparc', 'x86' catrel: 'stable' Returns: {"/opt/csw/lib": ["CSWfoo", "CSWbar"], "/opt/csw/1/lib": ["CSWfoomore"]} """ pkgs = {} sqo_osrel, sqo_arch, sqo_catrel = self.GetSqlobjectTriad( osrel, arch, catrel) connection = m.CswFile._connection join = [ sqlbuilder.INNERJOINOn(None, m.Pkginst, m.CswFile.q.pkginst==m.Pkginst.q.id), sqlbuilder.INNERJOINOn(None, m.Srv4FileStats, m.CswFile.q.srv4_file==m.Srv4FileStats.q.id), sqlbuilder.INNERJOINOn(None, m.Srv4FileInCatalog, m.Srv4FileStats.q.id==m.Srv4FileInCatalog.q.srv4file), ] where = sqlobject.AND( m.CswFile.q.basename==basename, m.Srv4FileInCatalog.q.osrel==sqo_osrel, m.Srv4FileInCatalog.q.arch==sqo_arch, m.Srv4FileInCatalog.q.catrel==sqo_catrel, ) query = connection.sqlrepr( sqlbuilder.Select( [m.CswFile.q.path, m.Pkginst.q.pkgname], where=where, join=join)) rows = connection.queryAll(query) pkgs = {} for row in rows: file_path, pkginst = row pkgs.setdefault(file_path, []).append(pkginst) logging.debug("self.error_mgr_mock.GetPathsAndPkgnamesByBasename(%s)" ".AndReturn(%s)", repr(basename), pprint.pformat(pkgs)) return pkgs
def GetCatalogGenerationResult(sqo_osrel, sqo_arch, sqo_catrel): """Get rows with catalog results. CatalogEntry catalogname version pkgname basename md5_sum size deps category i_deps """ join = [ sqlbuilder.INNERJOINOn(None, Srv4FileInCatalog, Srv4FileInCatalog.q.srv4file==Srv4FileStats.q.id), sqlbuilder.INNERJOINOn(None, CatalogGenData, Srv4FileStats.q.md5_sum==CatalogGenData.q.md5_sum), sqlbuilder.INNERJOINOn(None, Maintainer, Srv4FileStats.q.maintainer==Maintainer.q.id), ] where = sqlbuilder.AND( Srv4FileInCatalog.q.osrel==sqo_osrel, Srv4FileInCatalog.q.arch==sqo_arch, Srv4FileInCatalog.q.catrel==sqo_catrel, Srv4FileStats.q.use_to_generate_catalogs==True, ) select = sqlbuilder.Select( ['catalogname', # 0 'version_string', # 1 'pkgname', # 2 'basename', # 3 'srv4_file_stats.md5_sum', # Hardcoded table name, is it portable? 'size', # 5 'deps', # 6 'i_deps', # 7 'pkginfo_name', # 8 # The above columns are used to generate catalogs. # Additional columns can be added blow. # 'maintainer.id', # 9 'maintainer.email', # 9 'mtime', # 10 'created_on', # 11 'created_by', # 12 'bundle', # 13 ], where=where, orderBy='catalogname', join=join) query = sqlobject.sqlhub.processConnection.sqlrepr(select) rows = sqlobject.sqlhub.processConnection.queryAll(query) return rows
def GET(self, catrel_name, arch_name, osrel_name, catalogname): """Get a srv4 reference by catalog ane catalogname.""" web.header('Access-Control-Allow-Origin', '*') try: sqo_osrel, sqo_arch, sqo_catrel = models.GetSqoTriad( osrel_name, arch_name, catrel_name) except sqlobject.main.SQLObjectNotFound: raise web.notfound( cjson.encode({'message': 'catalog %s %s %s not found' % (osrel_name, arch_name, catrel_name)})) join = [ sqlbuilder.INNERJOINOn(None, models.Srv4FileInCatalog, models.Srv4FileInCatalog.q.srv4file==models.Srv4FileStats.q.id), ] res = models.Srv4FileStats.select( sqlobject.AND( models.Srv4FileInCatalog.q.osrel==sqo_osrel, models.Srv4FileInCatalog.q.arch==sqo_arch, models.Srv4FileInCatalog.q.catrel==sqo_catrel, models.Srv4FileStats.q.catalogname==catalogname, models.Srv4FileStats.q.use_to_generate_catalogs==True), join=join, ) try: srv4 = res.getOne() mimetype, data = srv4.GetRestRepr(quick=True) web.header('Content-type', mimetype) return cjson.encode(data) except sqlobject.main.SQLObjectNotFound: data = {'message': 'no %s %s %s %s packages found' % (catrel_name, arch_name, osrel_name, catalogname)} raise web.notfound(cjson.encode(data)) except sqlobject.dberrors.OperationalError as exc: raise web.internalerror(exc)
def GetConflictingSrv4ByPkgnameResult(self, sqo_srv4, pkgname, sqo_osrel, sqo_arch, sqo_catrel): join = [ sqlbuilder.INNERJOINOn( None, m.Pkginst, m.Srv4FileStats.q.pkginst == m.Pkginst.q.id), ] res = m.Srv4FileStats.select( m.Pkginst.q.pkgname == pkgname, join=join).throughTo.in_catalogs.filter( sqlobject.AND(m.Srv4FileInCatalog.q.osrel == sqo_osrel, m.Srv4FileInCatalog.q.arch == sqo_arch, m.Srv4FileInCatalog.q.catrel == sqo_catrel, m.Srv4FileInCatalog.q.srv4file != sqo_srv4)) return res
def GetInstalledPackages(self, osrel, arch, catrel): sqo_osrel, sqo_arch, sqo_catrel = self.GetSqlobjectTriad( osrel, arch, catrel) logging.debug("GetInstalledPackages(%s, %s, %s)" % (osrel, arch, catrel)) # Object defining a filter for our OS release, architecture and catalog # release. oac = sqlobject.AND(m.Srv4FileInCatalog.q.osrel == sqo_osrel, m.Srv4FileInCatalog.q.arch == sqo_arch, m.Srv4FileInCatalog.q.catrel == sqo_catrel) # Join object, responsible for connecting the tables the right way. join = [ sqlbuilder.INNERJOINOn( None, m.Srv4FileStats, m.Pkginst.q.id == m.Srv4FileStats.q.pkginst), sqlbuilder.INNERJOINOn( None, m.Srv4FileInCatalog, m.Srv4FileStats.q.id == m.Srv4FileInCatalog.q.srv4file), ] res = m.Pkginst.select(oac, join=join) pkgs = [x.pkgname for x in res] return pkgs
def GET(self, tag_name): join = [ sqlbuilder.INNERJOINOn(None, models.Srv4FileStats, models.CheckpkgErrorTag.q.srv4_file==models.Srv4FileStats.q.id), ] tags = models.CheckpkgErrorTag.select( sqlobject.AND( models.CheckpkgErrorTag.q.tag_name==tag_name, models.Srv4FileStats.q.registered_level_two==True, models.Srv4FileStats.q.use_to_generate_catalogs==True, ), join=join, ).orderBy(('basename', 'tag_info', 'os_rel_id', 'arch_id', 'catrel_id')) return render.ErrorTagDetail(tag_name, tags)
def GetCatPackagesResult(sqo_osrel, sqo_arch, sqo_catrel): join = [ sqlbuilder.INNERJOINOn(None, Srv4FileInCatalog, Srv4FileInCatalog.q.srv4file==Srv4FileStats.q.id), ] res = Srv4FileStats.select( sqlobject.AND( Srv4FileInCatalog.q.osrel==sqo_osrel, Srv4FileInCatalog.q.arch==sqo_arch, Srv4FileInCatalog.q.catrel==sqo_catrel, Srv4FileStats.q.use_to_generate_catalogs==True, ), join=join, ).orderBy('catalogname') return res
infile = args[0] importer = system_pkgmap.InstallContentsImporter() infile_fd = open(infile, "r") importer.ImportFromFile(infile_fd, show_progress=True) elif (command, subcommand) == ('pkg', 'search'): logging.debug("Searching for %s", args) sqo_osrel = m.OsRelease.selectBy(short_name=options.osrel).getOne() sqo_arch = m.Architecture.selectBy(name=options.arch).getOne() sqo_catrel = m.CatalogRelease.selectBy(name=options.catrel).getOne() if len(args) < 1: logging.fatal("Wrong number of arguments: %s", len(args)) raise SystemExit for catalogname in args: join = [ sqlbuilder.INNERJOINOn( None, m.Srv4FileInCatalog, m.Srv4FileInCatalog.q.srv4file == m.Srv4FileStats.q.id), ] res = m.Srv4FileStats.select( sqlobject.AND( m.Srv4FileInCatalog.q.osrel == sqo_osrel, m.Srv4FileInCatalog.q.arch == sqo_arch, m.Srv4FileInCatalog.q.catrel == sqo_catrel, m.Srv4FileStats.q.catalogname.contains(catalogname), m.Srv4FileStats.q.use_to_generate_catalogs == True), join=join, ).orderBy("catalogname") for sqo_srv4 in res: print "%s %s" % (sqo_srv4.basename, sqo_srv4.md5_sum) elif command == 'sync-cat-from-file': if len(args) != 4:
def main(): parser = optparse.OptionParser(USAGE) parser.add_option("-d", "--debug", dest="debug", default=False, action="store_true", help="Turn on debugging messages") parser.add_option( "-t", "--pkg-review-template", dest="pkg_review_template", help="A Cheetah template used for package review reports.") parser.add_option("-r", "--os-release", dest="osrel", default="SunOS5.10", help="E.g. SunOS5.10") parser.add_option("-a", "--arch", dest="arch", default="sparc", help="'i386' or 'sparc'") parser.add_option("-c", "--catalog-release", dest="catrel", default="unstable", help="E.g. unstable, dublin") parser.add_option("--replace", dest="replace", default=False, action="store_true", help="Replace packages when importing (importpkg)") parser.add_option("--profile", dest="profile", default=False, action="store_true", help="Turn on profiling") parser.add_option("--force-unpack", dest="force_unpack", default=False, action="store_true", help="Force unpacking of packages") options, args = parser.parse_args() logging_level = logging.INFO if options.debug: logging_level = logging.DEBUG fmt = '%(levelname)s %(asctime)s %(filename)s:%(lineno)d %(message)s' logging.basicConfig(format=fmt, level=logging_level) if not args: raise UsageError("Please specify a command. See --help.") # SetUpSqlobjectConnection needs to be called after # logging.basicConfig configuration.SetUpSqlobjectConnection() command = args[0] args = args[1:] if command == 'show': subcommand = args[0] args = args[1:] elif command == 'pkg': subcommand = args[0] args = args[1:] else: subcommand = None md5_sums = args if (command, subcommand) == ('show', 'errors'): for md5_sum in md5_sums: srv4 = GetPkg(md5_sum) res = m.CheckpkgErrorTag.select( m.CheckpkgErrorTag.q.srv4_file == srv4) for row in res: print 'overridden' if row.overridden else 'active', print row.pkgname, row.tag_name, row.tag_info, row.catrel.name, row.arch.name, print row.os_rel.short_name elif (command, subcommand) == ('show', 'overrides'): for md5_sum in md5_sums: srv4 = GetPkg(md5_sum) res = m.CheckpkgOverride.select( m.CheckpkgOverride.q.srv4_file == srv4) for row in res: print row.pkgname, row.tag_name, row.tag_info elif (command, subcommand) == ('show', 'pkg'): for md5_sum in md5_sums: srv4 = GetPkg(md5_sum) t = Template(SHOW_PKG_TMPL, searchList=[srv4]) sys.stdout.write(unicode(t)) elif command == 'gen-html': config = configuration.GetConfig() username, password = rest.GetUsernameAndPassword() rest_client = rest.RestClient(pkgdb_url=config.get('rest', 'pkgdb'), releases_url=config.get( 'rest', 'releases'), username=username, password=password, debug=options.debug) g = HtmlGenerator(md5_sums, options.pkg_review_template, rest_client, options.debug) sys.stdout.write(g.GenerateHtml()) elif command == 'initdb': config = configuration.GetConfig() database.InitDB(config) elif command == 'importpkg': collector = package_stats.StatsCollector(logger=logging, debug=options.debug) file_list = args catalog_entries = [] for file_name in file_list: file_hash = hashlib.md5() chunk_size = 2 * 1024 * 1024 with open(file_name, 'rb') as fd: data = fd.read(chunk_size) while data: file_hash.update(data) data = fd.read(chunk_size) data_md5_sum = file_hash.hexdigest() catalog_entry = { 'md5sum': data_md5_sum, 'file_basename': os.path.basename(file_name), 'pkg_path': file_name, } catalog_entries.append(catalog_entry) md5_list = collector.CollectStatsFromCatalogEntries( catalog_entries, force_unpack=options.force_unpack) config = configuration.GetConfig() rest_client = rest.RestClient(pkgdb_url=config.get('rest', 'pkgdb'), releases_url=config.get( 'rest', 'releases'), debug=options.debug) for md5_sum in md5_list: logging.debug("Importing %s", md5_sum) rest_client.RegisterLevelTwo(md5_sum) elif command == 'removepkg': for md5_sum in md5_sums: srv4 = GetPkg(md5_sum) in_catalogs = list(srv4.in_catalogs) if in_catalogs: for in_catalog in in_catalogs: logging.warning("%s", in_catalog) logging.warning( "Not removing from the database, because the package " "in question is part of at least one catalog.") else: logging.info("Removing %s", srv4) srv4.DeleteAllDependentObjects() srv4.destroySelf() elif command == 'add-to-cat': if len(args) < 4: raise UsageError("Not enough arguments, see usage.") user = getpass.getuser() osrel, arch, catrel = args[:3] username, password = rest.GetUsernameAndPassword() rest_client = rest.RestClient(username=username, password=password) md5_sums = args[3:] for md5_sum in md5_sums: rest_client.AddSvr4ToCatalog(catrel, arch, osrel, md5_sum) elif command == 'del-from-cat': if len(args) < 4: raise UsageError("Not enough arguments, see usage.") osrel, arch, catrel = args[:3] md5_sums = args[3:] username, password = rest.GetUsernameAndPassword() rest_client = rest.RestClient(username=username, password=password) for md5_sum in md5_sums: rest_client.RemoveSvr4FromCatalog(catrel, arch, osrel, md5_sum) elif command == 'system-metadata-to-disk': logging.debug("Args: %s", args) outfile = None infile_contents = common_constants.DEFAULT_INSTALL_CONTENTS_FILE infile_pkginfo = None osrel, arch = (None, None) if len(args) >= 2: infile_contents = args[0] infile_pkginfo = args[1] if len(args) >= 3: outfile = args[2] if len(args) >= 4: if len(args) == 5: osrel, arch = args[3:5] else: raise UsageError("Wrong number of arguments (%s), see usage." % len(args)) spi = system_pkgmap.Indexer(outfile, infile_contents, infile_pkginfo, osrel, arch) spi.IndexAndSave() elif command == 'import-system-metadata': if len(args) < 2: raise UsageError( "Usage: ... import-system-metadata <osrel> <arch>") osrel = args[0] arch = args[1] importer = system_pkgmap.InstallContentsImporter(osrel, arch, debug=options.debug) importer.Import(show_progress=(not options.debug)) elif (command, subcommand) == ('pkg', 'search'): logging.debug("Searching for %s", args) sqo_osrel = m.OsRelease.selectBy(short_name=options.osrel).getOne() sqo_arch = m.Architecture.selectBy(name=options.arch).getOne() sqo_catrel = m.CatalogRelease.selectBy(name=options.catrel).getOne() if len(args) < 1: logging.fatal("Wrong number of arguments: %s", len(args)) raise SystemExit for catalogname in args: join = [ sqlbuilder.INNERJOINOn( None, m.Srv4FileInCatalog, m.Srv4FileInCatalog.q.srv4file == m.Srv4FileStats.q.id), ] res = m.Srv4FileStats.select( sqlobject.AND( m.Srv4FileInCatalog.q.osrel == sqo_osrel, m.Srv4FileInCatalog.q.arch == sqo_arch, m.Srv4FileInCatalog.q.catrel == sqo_catrel, m.Srv4FileStats.q.catalogname.contains(catalogname), m.Srv4FileStats.q.use_to_generate_catalogs == True), join=join, ).orderBy("catalogname") for sqo_srv4 in res: print "%s %s" % (sqo_srv4.basename, sqo_srv4.md5_sum) elif command == 'sync-cat-from-file': if len(args) != 4: raise UsageError("Wrong number of arguments, see usage.") osrel, arch, catrel, catalog_file = args ci = CatalogImporter(debug=options.debug) ci.SyncFromCatalogFile(osrel, arch, catrel, catalog_file) elif command == 'sync-catalogs-from-tree': if len(args) != 2: raise UsageError("Wrong number of arguments, see usage.") ci = CatalogImporter(debug=options.debug) catrel, base_dir = args ci.SyncFromCatalogTree(catrel, base_dir, options.force_unpack) elif (command, subcommand) == ('show', 'cat'): sqo_osrel, sqo_arch, sqo_catrel = m.GetSqoTriad( options.osrel, options.arch, options.catrel) res = m.GetCatPackagesResult(sqo_osrel, sqo_arch, sqo_catrel) for obj in res: print obj.catalogname, obj.basename, obj.md5_sum elif (command, subcommand) == ('show', 'files'): md5_sum = args[0] join = [ sqlbuilder.INNERJOINOn( None, m.Srv4FileStats, m.CswFile.q.srv4_file == m.Srv4FileStats.q.id), ] res = m.CswFile.select( m.Srv4FileStats.q.md5_sum == md5_sum, join=join, ) for obj in res: print os.path.join(obj.path, obj.basename) elif (command, subcommand) == ('show', 'basename'): db_catalog = checkpkg_lib.Catalog() for arg in args: pkgs_by_path = db_catalog.GetPathsAndPkgnamesByBasename( arg, options.osrel, options.arch, options.catrel) for file_path in pkgs_by_path: print os.path.join(file_path, arg), ", ".join(pkgs_by_path[file_path]) elif (command, subcommand) == ('show', 'filename'): db_catalog = checkpkg_lib.Catalog() for arg in args: pkgs = db_catalog.GetPkgByPath(arg, options.osrel, options.arch, options.catrel) print " ".join(pkgs) else: raise UsageError("Command unrecognized: %s" % command)