def test_versionclass_single_branch1(self): packages = [ # here we only have default branch (Package(repo='1', family='1', name='a', version='2.2.20990101', flags=PackageFlags.ignore), VersionClass.ignored), (Package(repo='1', family='1', name='a', version='2.1'), VersionClass.newest), (Package(repo='1', family='1', name='a', version='2.0.20990101', flags=PackageFlags.ignore), VersionClass.legacy), (Package(repo='1', family='1', name='a', version='2.0'), VersionClass.legacy), (Package(repo='2', family='2', name='a', version='2.1'), VersionClass.newest), (Package(repo='2', family='2', name='a', version='2.0'), VersionClass.legacy), (Package(repo='3', family='3', name='a', version='2.0'), VersionClass.outdated), ] FillPackagesetVersions([package for package, _ in packages]) for package, expectedclass in packages: self.assertEqual(package.versionclass, expectedclass, msg='repo {}, pkg {}, ver {}'.format( package.repo, package.name, package.version))
def main(): options = parse_arguments() logger = StderrLogger() if options.logfile: logger = FileLogger(options.logfile) if options.fields == 'all': options.fields = sorted(Package().__dict__.keys()) else: options.fields = options.fields.split(',') # Set up filters filters = [] if options.maintainer: filters.append(MaintainerFilter(options.maintainer)) if options.category: filters.append(CategoryFilter(options.maintainer)) if options.more_repos is not None or options.less_repos is not None: filters.append(FamilyCountFilter(more=options.more_repos, less=options.less_repos)) if options.in_repository: filters.append(InRepoFilter(options.in_repository)) if options.not_in_repository: filters.append(NotInRepoFilter(options.not_in_repository)) if options.outdated_in_repository: filters.append(OutdatedInRepoFilter(options.not_in_repository)) if not options.no_shadow: filters.append(ShadowFilter()) repomgr = RepositoryManager(options.repos_dir) repoproc = RepositoryProcessor(repomgr, options.statedir) logger.Log('dumping...') for packageset in repoproc.StreamDeserializeMulti(reponames=options.reponames): FillPackagesetVersions(packageset) if not PackagesetCheckFilters(packageset, *filters): continue if options.dump == 'packages': for package in packageset: print( options.field_separator.join( ( format_package_field(field, getattr(package, field)) for field in options.fields ) ) ) if options.dump == 'summaries': print(packageset[0].effname) best_pkg_by_repo = PackagesetToBestByRepo(packageset) for reponame in repomgr.GetNames(options.reponames): if reponame in best_pkg_by_repo: print(' {}: {} ({})'.format( reponame, best_pkg_by_repo[reponame].version, VersionClass.ToString(best_pkg_by_repo[reponame].versionclass) )) return 0
def main() -> int: options = parse_arguments() logger: Logger = StderrLogger() if options.logfile: logger = FileLogger(options.logfile) if options.fields == 'all': options.fields = sorted(Package().__dict__.keys()) else: options.fields = options.fields.split(',') repomgr = RepositoryManager(options.repos_dir) repoproc = RepositoryProcessor(repomgr, options.statedir, options.parseddir) logger.log('dumping...') for packageset in repoproc.iter_parsed(reponames=options.reponames): FillPackagesetVersions(packageset) if not options.all and packageset_is_shadow_only(packageset): continue for package in packageset: print( options.field_separator.join( (format_package_field(field, getattr(package, field)) for field in options.fields))) return 0
def database_update(env): logger = env.get_main_logger() database = env.get_main_database_connection() logger.log('clearing the database') database.update_start() package_queue = [] num_pushed = 0 start_time = timer() logger.log('pushing packages to database') for packageset in env.get_repo_processor().StreamDeserializeMulti( reponames=env.get_enabled_repo_names(), logger=logger): FillPackagesetVersions(packageset) package_queue.extend(packageset) if len(package_queue) >= 10000: database.add_packages(package_queue) num_pushed += len(package_queue) package_queue = [] logger.get_indented().log( 'pushed {} packages, {:.2f} packages/second'.format( num_pushed, num_pushed / (timer() - start_time))) # process what's left in the queue database.add_packages(package_queue) logger.log('updating views') database.update_finish() logger.log('committing changes') database.commit()
def test_versionclass_ignoredignored(self): packages = [ (Package(repo='1', family='1', name='a', version='2.2.99999999', flags=PackageFlags.ignore), VersionClass.ignored), (Package(repo='1', family='1', name='a', version='2.2.9999', flags=PackageFlags.ignore), VersionClass.ignored), # this one should be outdated, not legacy, e.g. ignored's should not be counted # as first packages in the branch (Package(repo='1', family='1', name='a', version='2.1'), VersionClass.outdated), (Package(repo='1', family='1', name='a', version='2.0'), VersionClass.legacy), (Package(repo='2', family='2', name='a', version='2.2'), VersionClass.newest), ] FillPackagesetVersions([package for package, _ in packages]) for package, expectedclass in packages: self.assertEqual(package.versionclass, expectedclass, msg='repo {}, pkg {}, ver {}'.format( package.repo, package.name, package.version))
def test_versionclass_manylegacy(self): packages = [ (Package(repo='2', family='2', name='a', version='2.2'), VersionClass.newest), # one of these packages should not make the other one legacy instead of outdated (Package(repo='1', family='1', name='a', version='2.1'), VersionClass.outdated), (Package(repo='1', family='1', name='a', version='2.0'), VersionClass.legacy), (Package(repo='1', family='1', name='a', version='2.0'), VersionClass.legacy), (Package(repo='1', family='1', name='a', version='1.9'), VersionClass.legacy), (Package(repo='1', family='1', name='a', version='1.9'), VersionClass.legacy), (Package(repo='1', family='1', name='a', version='1.8'), VersionClass.legacy), (Package(repo='1', family='1', name='a', version='1.8'), VersionClass.legacy), ] FillPackagesetVersions([package for package, _ in packages]) for package, expectedclass in packages: self.assertEqual(package.versionclass, expectedclass, msg='repo {}, pkg {}, ver {}'.format( package.repo, package.name, package.version))
def test_versionclass_branch_bounds(self): packages = [ (Package(repo='1', family='1', name='a', version='2.2beta1', flags=PackageFlags.devel), VersionClass.devel), (Package(repo='1', family='1', name='a', version='2.2alpha1.9999', flags=PackageFlags.ignore | PackageFlags.devel), VersionClass.legacy), # see #338. There are multiple possible ways to ignored version between branches, # we go with ignored for now (Package(repo='1', family='1', name='a', version='2.1.9999', flags=PackageFlags.ignore), VersionClass.ignored), (Package(repo='1', family='1', name='a', version='2.1'), VersionClass.newest), (Package(repo='1', family='1', name='a', version='2.0'), VersionClass.legacy), (Package(repo='2', family='2', name='a', version='2.1'), VersionClass.newest), ] FillPackagesetVersions([package for package, _ in packages]) for package, expectedclass in packages: self.assertEqual(package.versionclass, expectedclass, msg='repo {}, pkg {}, ver {}'.format( package.repo, package.name, package.version))
def test_versionclass_devel_lower_than_default(self): packages = [ # devel package < normal package (Package(repo='1', family='1', name='a', version='2.1'), VersionClass.newest), (Package(repo='1', family='1', name='a', version='2.0', flags=PackageFlags.devel), VersionClass.legacy), (Package(repo='2', family='2', name='a', version='2.1'), VersionClass.newest), (Package(repo='2', family='2', name='a', version='2.0', flags=PackageFlags.devel), VersionClass.legacy), ] FillPackagesetVersions([package for package, _ in packages]) for package, expectedclass in packages: self.assertEqual(package.versionclass, expectedclass, msg='repo {}, pkg {}, ver {}'.format( package.repo, package.name, package.version))
def test_versionclass_branch_confusion(self): packages = [ # same version is both devel and default in different packages # this should be consistently aggregated (Package(repo='1', family='1', name='a', version='2.1', flags=PackageFlags.devel), VersionClass.devel), (Package(repo='1', family='1', name='a', version='2.0', flags=PackageFlags.devel), VersionClass.legacy), (Package(repo='2', family='2', name='a', version='2.1'), VersionClass.devel), (Package(repo='2', family='2', name='a', version='2.0'), VersionClass.legacy), ] FillPackagesetVersions([package for package, _ in packages]) for package, expectedclass in packages: self.assertEqual(package.versionclass, expectedclass, msg='repo {}, pkg {}, ver {}'.format( package.repo, package.name, package.version))
def package_processor(packageset): FillPackagesetVersions(packageset) if not PackagesetCheckFilters(packageset, *filters): return if options.dump == 'packages': for package in packageset: print( options.field_separator.join( [ VersionClass.ToString(package.versionclass) if field == 'versionclass' else str(getattr(package, field)) for field in options.fields ] ) ) if options.dump == 'summaries': print(packageset[0].effname) best_pkg_by_repo = PackagesetToBestByRepo(packageset) for reponame in repomgr.GetNames(options.reponames): if reponame in best_pkg_by_repo: print(' {}: {} ({})'.format( reponame, best_pkg_by_repo[reponame].version, VersionClass.ToString(best_pkg_by_repo[reponame].versionclass) ))
def test_suppress_ignored_rolling(self): packages = [ (Package(repo='0', family='0', name='a', version='3.0', flags=PackageFlags.rolling), VersionClass.rolling), (Package(repo='1', family='1', name='a', version='2.0', flags=PackageFlags.ignore), VersionClass.newest), (Package(repo='2', family='1', name='a', version='1.0', flags=PackageFlags.ignore), VersionClass.outdated), ] FillPackagesetVersions([package for package, _ in packages]) for package, expectedclass in packages: self.assertEqual(package.versionclass, expectedclass, msg='repo {}, pkg {}, ver {}'.format( package.repo, package.name, package.version))
def PackageProcessor(packageset): nonlocal package_queue, num_pushed, start_time FillPackagesetVersions(packageset) package_queue.extend(packageset) if len(package_queue) >= 10000: database.AddPackages(package_queue) num_pushed += len(package_queue) package_queue = [] db_logger.Log(' pushed {} packages, {:.2f} packages/second'.format(num_pushed, num_pushed / (timer() - start_time)))
def PackageProcessor(packageset): nonlocal package_queue, num_pushed FillPackagesetVersions(packageset) package_queue.extend(packageset) if len(package_queue) >= 1000: database.AddPackages(package_queue) num_pushed += len(package_queue) package_queue = [] db_logger.Log(' pushed {} packages'.format(num_pushed))
def test_versionclass_unique(self): packages = [ (Package(repo='1', family='1', name='a', version='2.0alpha1', devel=True), VersionClass.unique), (Package(repo='2', family='1', name='a', version='1.2'), VersionClass.unique), (Package(repo='3', family='1', name='a', version='1.1'), VersionClass.outdated), (Package(repo='3', family='1', name='a', version='1.0'), VersionClass.legacy), ] FillPackagesetVersions([package for package, _ in packages]) for package, expectedclass in packages: self.assertEqual(package.versionclass, expectedclass, msg='repo {}, pkg {}, ver {}'.format(package.repo, package.name, package.version))
def test_versionclass_unignored_really_unignored(self): packages = [ # ignored package should be fully unignored with the same non-ignored version in another repo (Package(repo='1', family='1', name='a', version='2.1', ignoreversion=True), VersionClass.newest), (Package(repo='1', family='1', name='a', version='2.0'), VersionClass.legacy), (Package(repo='2', family='2', name='a', version='2.1'), VersionClass.newest), (Package(repo='2', family='2', name='a', version='2.0'), VersionClass.legacy), ] FillPackagesetVersions([package for package, _ in packages]) for package, expectedclass in packages: self.assertEqual(package.versionclass, expectedclass, msg='repo {}, pkg {}, ver {}'.format(package.repo, package.name, package.version))
def test_versionclass_single_branch2(self): packages = [ # here we only have devel branch (Package(repo='1', family='1', name='a', version='2.2rc1.20990101', ignoreversion=True, devel=True), VersionClass.ignored), (Package(repo='1', family='1', name='a', version='2.2beta1', devel=True), VersionClass.devel), (Package(repo='1', family='1', name='a', version='2.2alpha1.20990101', ignoreversion=True, devel=True), VersionClass.legacy), (Package(repo='1', family='1', name='a', version='2.2alpha1', devel=True), VersionClass.legacy), (Package(repo='2', family='2', name='a', version='2.2beta1', devel=True), VersionClass.devel), (Package(repo='2', family='2', name='a', version='2.2alpha1', devel=True), VersionClass.legacy), (Package(repo='3', family='3', name='a', version='2.2alpha1', devel=True), VersionClass.outdated), ] FillPackagesetVersions([package for package, _ in packages]) for package, expectedclass in packages: self.assertEqual(package.versionclass, expectedclass, msg='repo {}, pkg {}, ver {}'.format(package.repo, package.name, package.version))
def test_versionclass_legacy(self): packages = [ (Package(repo='1', family='1', name='a', version='2.0'), VersionClass.newest), (Package(repo='2', family='2', name='a', version='1.0'), VersionClass.outdated), (Package(repo='3', family='3', name='a', version='1.0', flags=PackageFlags.legacy), VersionClass.legacy), ] FillPackagesetVersions([package for package, _ in packages]) for package, expectedclass in packages: self.assertEqual(package.versionclass, expectedclass, msg='repo {}, pkg {}, ver {}'.format( package.repo, package.name, package.version))
def main(): options = parse_arguments() logger = StderrLogger() if options.logfile: logger = FileLogger(options.logfile) if options.fields == 'all': options.fields = sorted(Package().__dict__.keys()) else: options.fields = options.fields.split(',') repomgr = RepositoryManager(options.repos_dir) repoproc = RepositoryProcessor(repomgr, options.statedir, options.parseddir) logger.Log('dumping...') for packageset in repoproc.iter_parsed(reponames=options.reponames): FillPackagesetVersions(packageset) if not options.all and packageset_is_shadow_only(packageset): continue if options.dump == 'packages': for package in packageset: print( options.field_separator.join( (format_package_field(field, getattr(package, field)) for field in options.fields))) if options.dump == 'summaries': print(packageset[0].effname) best_pkg_by_repo = PackagesetToBestByRepo(packageset) for reponame in repomgr.GetNames(options.reponames): if reponame in best_pkg_by_repo: print(' {}: {} ({})'.format( reponame, best_pkg_by_repo[reponame].version, VersionClass.ToString( best_pkg_by_repo[reponame].versionclass))) return 0
def test_versionclasses_big(self): packages = [ # Reference repo (Package(repo='1', family='1', name='a', version='2.2.20990101', ignoreversion=True), VersionClass.ignored), (Package(repo='1', family='1', name='a', version='2.2beta1', devel=True), VersionClass.devel), (Package(repo='1', family='1', name='a', version='2.2alpha1.20990101', devel=True, ignoreversion=True), VersionClass.legacy), (Package(repo='1', family='1', name='a', version='2.2alpha1', devel=True), VersionClass.legacy), # see #338. There are multiple possible ways to ignored version between branches, # we go with ignored for now (Package(repo='1', family='1', name='a', version='2.1.20990101', ignoreversion=True), VersionClass.ignored), (Package(repo='1', family='1', name='a', version='2.1'), VersionClass.newest), (Package(repo='1', family='1', name='a', version='2.0.20990101', ignoreversion=True), VersionClass.legacy), (Package(repo='1', family='1', name='a', version='2.0'), VersionClass.legacy), (Package(repo='1', family='1', name='a', version='1.2.20990101', ignoreversion=True), VersionClass.legacy), (Package(repo='1', family='1', name='a', version='1.2beta1', devel=True), VersionClass.legacy), (Package(repo='1', family='1', name='a', version='1.2alpha1.20990101', ignoreversion=True), VersionClass.legacy), (Package(repo='1', family='1', name='a', version='1.2alpha1', devel=True), VersionClass.legacy), (Package(repo='1', family='1', name='a', version='1.1.20990101', ignoreversion=True), VersionClass.legacy), (Package(repo='1', family='1', name='a', version='1.1'), VersionClass.legacy), (Package(repo='1', family='1', name='a', version='1.0.20990101', ignoreversion=True), VersionClass.legacy), (Package(repo='1', family='1', name='a', version='1.0'), VersionClass.legacy), # devel + legacy (Package(repo='2', family='2', name='a', version='2.2beta1', devel=True), VersionClass.devel), (Package(repo='2', family='2', name='a', version='2.0'), VersionClass.outdated), # devel + newest + legacy (Package(repo='3', family='3', name='a', version='2.2beta1', devel=True), VersionClass.devel), (Package(repo='3', family='3', name='a', version='2.1'), VersionClass.newest), (Package(repo='3', family='3', name='a', version='2.0'), VersionClass.legacy), # newest + legacy (Package(repo='4', family='4', name='a', version='2.1'), VersionClass.newest), (Package(repo='4', family='4', name='a', version='2.0'), VersionClass.legacy), # outdated + legacy (Package(repo='5', family='5', name='a', version='1.1'), VersionClass.outdated), (Package(repo='5', family='5', name='a', version='1.0'), VersionClass.legacy), # outdated outdated/ignored + legacy (Package(repo='6', family='6', name='a', version='1.1.20990101', ignoreversion=True), VersionClass.outdated), (Package(repo='6', family='6', name='a', version='1.1'), VersionClass.legacy), (Package(repo='6', family='6', name='a', version='1.0'), VersionClass.legacy), # devel class should be ignored for newest version and below (Package(repo='7', family='7', name='a', version='2.1', devel=True), VersionClass.newest), # ignored classes are unignored when they are backed with real classes (Package(repo='8', family='8', name='a', version='2.2beta1', devel=True, ignoreversion=True), VersionClass.devel), (Package(repo='9', family='9', name='a', version='2.1', ignoreversion=True), VersionClass.newest), (Package(repo='10', family='10', name='a', version='2.0', ignoreversion=True), VersionClass.outdated), (Package(repo='10', family='10', name='a', version='1.9', ignoreversion=True), VersionClass.legacy), # version between newest and devel should be outdated when there's no devel (Package(repo='11', family='11', name='a', version='2.2alpha1', devel=True), VersionClass.outdated), # outdated in devel and normal at the same time (Package(repo='12', family='12', name='a', version='2.2alpha1', devel=True), VersionClass.outdated), (Package(repo='12', family='12', name='a', version='2.0'), VersionClass.outdated), ] FillPackagesetVersions([package for package, _ in packages]) for package, expectedclass in packages: self.assertEqual(package.versionclass, expectedclass, msg='repo {}, pkg {}, ver {}'.format(package.repo, package.name, package.version))