def test_versionclass_single_branch1(self):
packages = [
# here we only have default branch
(Package(repo='1',
family='1',
name='a',
version='2.2.20990101',
flags=PackageFlags.ignore), VersionClass.ignored),
(Package(repo='1', family='1', name='a',
version='2.1'), VersionClass.newest),
(Package(repo='1',
family='1',
name='a',
version='2.0.20990101',
flags=PackageFlags.ignore), VersionClass.legacy),
(Package(repo='1', family='1', name='a',
version='2.0'), VersionClass.legacy),
(Package(repo='2', family='2', name='a',
version='2.1'), VersionClass.newest),
(Package(repo='2', family='2', name='a',
version='2.0'), VersionClass.legacy),
(Package(repo='3', family='3', name='a',
version='2.0'), VersionClass.outdated),
]
FillPackagesetVersions([package for package, _ in packages])
for package, expectedclass in packages:
self.assertEqual(package.versionclass,
expectedclass,
msg='repo {}, pkg {}, ver {}'.format(
package.repo, package.name, package.version))
def main():
options = parse_arguments()
logger = StderrLogger()
if options.logfile:
logger = FileLogger(options.logfile)
if options.fields == 'all':
options.fields = sorted(Package().__dict__.keys())
else:
options.fields = options.fields.split(',')
# Set up filters
filters = []
if options.maintainer:
filters.append(MaintainerFilter(options.maintainer))
if options.category:
filters.append(CategoryFilter(options.maintainer))
if options.more_repos is not None or options.less_repos is not None:
filters.append(FamilyCountFilter(more=options.more_repos, less=options.less_repos))
if options.in_repository:
filters.append(InRepoFilter(options.in_repository))
if options.not_in_repository:
filters.append(NotInRepoFilter(options.not_in_repository))
if options.outdated_in_repository:
filters.append(OutdatedInRepoFilter(options.not_in_repository))
if not options.no_shadow:
filters.append(ShadowFilter())
repomgr = RepositoryManager(options.repos_dir)
repoproc = RepositoryProcessor(repomgr, options.statedir)
logger.Log('dumping...')
for packageset in repoproc.StreamDeserializeMulti(reponames=options.reponames):
FillPackagesetVersions(packageset)
if not PackagesetCheckFilters(packageset, *filters):
continue
if options.dump == 'packages':
for package in packageset:
print(
options.field_separator.join(
(
format_package_field(field, getattr(package, field)) for field in options.fields
)
)
)
if options.dump == 'summaries':
print(packageset[0].effname)
best_pkg_by_repo = PackagesetToBestByRepo(packageset)
for reponame in repomgr.GetNames(options.reponames):
if reponame in best_pkg_by_repo:
print(' {}: {} ({})'.format(
reponame,
best_pkg_by_repo[reponame].version,
VersionClass.ToString(best_pkg_by_repo[reponame].versionclass)
))
return 0
def main() -> int:
options = parse_arguments()
logger: Logger = StderrLogger()
if options.logfile:
logger = FileLogger(options.logfile)
if options.fields == 'all':
options.fields = sorted(Package().__dict__.keys())
else:
options.fields = options.fields.split(',')
repomgr = RepositoryManager(options.repos_dir)
repoproc = RepositoryProcessor(repomgr, options.statedir,
options.parseddir)
logger.log('dumping...')
for packageset in repoproc.iter_parsed(reponames=options.reponames):
FillPackagesetVersions(packageset)
if not options.all and packageset_is_shadow_only(packageset):
continue
for package in packageset:
print(
options.field_separator.join(
(format_package_field(field, getattr(package, field))
for field in options.fields)))
return 0
def database_update(env):
logger = env.get_main_logger()
database = env.get_main_database_connection()
logger.log('clearing the database')
database.update_start()
package_queue = []
num_pushed = 0
start_time = timer()
logger.log('pushing packages to database')
for packageset in env.get_repo_processor().StreamDeserializeMulti(
reponames=env.get_enabled_repo_names(), logger=logger):
FillPackagesetVersions(packageset)
package_queue.extend(packageset)
if len(package_queue) >= 10000:
database.add_packages(package_queue)
num_pushed += len(package_queue)
package_queue = []
logger.get_indented().log(
'pushed {} packages, {:.2f} packages/second'.format(
num_pushed, num_pushed / (timer() - start_time)))
# process what's left in the queue
database.add_packages(package_queue)
logger.log('updating views')
database.update_finish()
logger.log('committing changes')
database.commit()
def test_versionclass_ignoredignored(self):
packages = [
(Package(repo='1',
family='1',
name='a',
version='2.2.99999999',
flags=PackageFlags.ignore), VersionClass.ignored),
(Package(repo='1',
family='1',
name='a',
version='2.2.9999',
flags=PackageFlags.ignore), VersionClass.ignored),
# this one should be outdated, not legacy, e.g. ignored's should not be counted
# as first packages in the branch
(Package(repo='1', family='1', name='a',
version='2.1'), VersionClass.outdated),
(Package(repo='1', family='1', name='a',
version='2.0'), VersionClass.legacy),
(Package(repo='2', family='2', name='a',
version='2.2'), VersionClass.newest),
]
FillPackagesetVersions([package for package, _ in packages])
for package, expectedclass in packages:
self.assertEqual(package.versionclass,
expectedclass,
msg='repo {}, pkg {}, ver {}'.format(
package.repo, package.name, package.version))
def test_versionclass_manylegacy(self):
packages = [
(Package(repo='2', family='2', name='a',
version='2.2'), VersionClass.newest),
# one of these packages should not make the other one legacy instead of outdated
(Package(repo='1', family='1', name='a',
version='2.1'), VersionClass.outdated),
(Package(repo='1', family='1', name='a',
version='2.0'), VersionClass.legacy),
(Package(repo='1', family='1', name='a',
version='2.0'), VersionClass.legacy),
(Package(repo='1', family='1', name='a',
version='1.9'), VersionClass.legacy),
(Package(repo='1', family='1', name='a',
version='1.9'), VersionClass.legacy),
(Package(repo='1', family='1', name='a',
version='1.8'), VersionClass.legacy),
(Package(repo='1', family='1', name='a',
version='1.8'), VersionClass.legacy),
]
FillPackagesetVersions([package for package, _ in packages])
for package, expectedclass in packages:
self.assertEqual(package.versionclass,
expectedclass,
msg='repo {}, pkg {}, ver {}'.format(
package.repo, package.name, package.version))
def test_versionclass_branch_bounds(self):
packages = [
(Package(repo='1',
family='1',
name='a',
version='2.2beta1',
flags=PackageFlags.devel), VersionClass.devel),
(Package(repo='1',
family='1',
name='a',
version='2.2alpha1.9999',
flags=PackageFlags.ignore | PackageFlags.devel),
VersionClass.legacy),
# see #338. There are multiple possible ways to ignored version between branches,
# we go with ignored for now
(Package(repo='1',
family='1',
name='a',
version='2.1.9999',
flags=PackageFlags.ignore), VersionClass.ignored),
(Package(repo='1', family='1', name='a',
version='2.1'), VersionClass.newest),
(Package(repo='1', family='1', name='a',
version='2.0'), VersionClass.legacy),
(Package(repo='2', family='2', name='a',
version='2.1'), VersionClass.newest),
]
FillPackagesetVersions([package for package, _ in packages])
for package, expectedclass in packages:
self.assertEqual(package.versionclass,
expectedclass,
msg='repo {}, pkg {}, ver {}'.format(
package.repo, package.name, package.version))
def test_versionclass_devel_lower_than_default(self):
packages = [
# devel package < normal package
(Package(repo='1', family='1', name='a',
version='2.1'), VersionClass.newest),
(Package(repo='1',
family='1',
name='a',
version='2.0',
flags=PackageFlags.devel), VersionClass.legacy),
(Package(repo='2', family='2', name='a',
version='2.1'), VersionClass.newest),
(Package(repo='2',
family='2',
name='a',
version='2.0',
flags=PackageFlags.devel), VersionClass.legacy),
]
FillPackagesetVersions([package for package, _ in packages])
for package, expectedclass in packages:
self.assertEqual(package.versionclass,
expectedclass,
msg='repo {}, pkg {}, ver {}'.format(
package.repo, package.name, package.version))
def test_versionclass_branch_confusion(self):
packages = [
# same version is both devel and default in different packages
# this should be consistently aggregated
(Package(repo='1',
family='1',
name='a',
version='2.1',
flags=PackageFlags.devel), VersionClass.devel),
(Package(repo='1',
family='1',
name='a',
version='2.0',
flags=PackageFlags.devel), VersionClass.legacy),
(Package(repo='2', family='2', name='a',
version='2.1'), VersionClass.devel),
(Package(repo='2', family='2', name='a',
version='2.0'), VersionClass.legacy),
]
FillPackagesetVersions([package for package, _ in packages])
for package, expectedclass in packages:
self.assertEqual(package.versionclass,
expectedclass,
msg='repo {}, pkg {}, ver {}'.format(
package.repo, package.name, package.version))
def package_processor(packageset):
FillPackagesetVersions(packageset)
if not PackagesetCheckFilters(packageset, *filters):
return
if options.dump == 'packages':
for package in packageset:
print(
options.field_separator.join(
[
VersionClass.ToString(package.versionclass) if field == 'versionclass' else str(getattr(package, field))
for field in options.fields
]
)
)
if options.dump == 'summaries':
print(packageset[0].effname)
best_pkg_by_repo = PackagesetToBestByRepo(packageset)
for reponame in repomgr.GetNames(options.reponames):
if reponame in best_pkg_by_repo:
print(' {}: {} ({})'.format(
reponame,
best_pkg_by_repo[reponame].version,
VersionClass.ToString(best_pkg_by_repo[reponame].versionclass)
))
def test_suppress_ignored_rolling(self):
packages = [
(Package(repo='0',
family='0',
name='a',
version='3.0',
flags=PackageFlags.rolling), VersionClass.rolling),
(Package(repo='1',
family='1',
name='a',
version='2.0',
flags=PackageFlags.ignore), VersionClass.newest),
(Package(repo='2',
family='1',
name='a',
version='1.0',
flags=PackageFlags.ignore), VersionClass.outdated),
]
FillPackagesetVersions([package for package, _ in packages])
for package, expectedclass in packages:
self.assertEqual(package.versionclass,
expectedclass,
msg='repo {}, pkg {}, ver {}'.format(
package.repo, package.name, package.version))
def PackageProcessor(packageset):
nonlocal package_queue, num_pushed, start_time
FillPackagesetVersions(packageset)
package_queue.extend(packageset)
if len(package_queue) >= 10000:
database.AddPackages(package_queue)
num_pushed += len(package_queue)
package_queue = []
db_logger.Log(' pushed {} packages, {:.2f} packages/second'.format(num_pushed, num_pushed / (timer() - start_time)))
def PackageProcessor(packageset):
nonlocal package_queue, num_pushed
FillPackagesetVersions(packageset)
package_queue.extend(packageset)
if len(package_queue) >= 1000:
database.AddPackages(package_queue)
num_pushed += len(package_queue)
package_queue = []
db_logger.Log(' pushed {} packages'.format(num_pushed))
def test_versionclass_unique(self):
packages = [
(Package(repo='1', family='1', name='a', version='2.0alpha1', devel=True), VersionClass.unique),
(Package(repo='2', family='1', name='a', version='1.2'), VersionClass.unique),
(Package(repo='3', family='1', name='a', version='1.1'), VersionClass.outdated),
(Package(repo='3', family='1', name='a', version='1.0'), VersionClass.legacy),
]
FillPackagesetVersions([package for package, _ in packages])
for package, expectedclass in packages:
self.assertEqual(package.versionclass, expectedclass, msg='repo {}, pkg {}, ver {}'.format(package.repo, package.name, package.version))
def test_versionclass_unignored_really_unignored(self):
packages = [
# ignored package should be fully unignored with the same non-ignored version in another repo
(Package(repo='1', family='1', name='a', version='2.1', ignoreversion=True), VersionClass.newest),
(Package(repo='1', family='1', name='a', version='2.0'), VersionClass.legacy),
(Package(repo='2', family='2', name='a', version='2.1'), VersionClass.newest),
(Package(repo='2', family='2', name='a', version='2.0'), VersionClass.legacy),
]
FillPackagesetVersions([package for package, _ in packages])
for package, expectedclass in packages:
self.assertEqual(package.versionclass, expectedclass, msg='repo {}, pkg {}, ver {}'.format(package.repo, package.name, package.version))
def test_versionclass_single_branch2(self):
packages = [
# here we only have devel branch
(Package(repo='1', family='1', name='a', version='2.2rc1.20990101', ignoreversion=True, devel=True), VersionClass.ignored),
(Package(repo='1', family='1', name='a', version='2.2beta1', devel=True), VersionClass.devel),
(Package(repo='1', family='1', name='a', version='2.2alpha1.20990101', ignoreversion=True, devel=True), VersionClass.legacy),
(Package(repo='1', family='1', name='a', version='2.2alpha1', devel=True), VersionClass.legacy),
(Package(repo='2', family='2', name='a', version='2.2beta1', devel=True), VersionClass.devel),
(Package(repo='2', family='2', name='a', version='2.2alpha1', devel=True), VersionClass.legacy),
(Package(repo='3', family='3', name='a', version='2.2alpha1', devel=True), VersionClass.outdated),
]
FillPackagesetVersions([package for package, _ in packages])
for package, expectedclass in packages:
self.assertEqual(package.versionclass, expectedclass, msg='repo {}, pkg {}, ver {}'.format(package.repo, package.name, package.version))
def test_versionclass_legacy(self):
packages = [
(Package(repo='1', family='1', name='a',
version='2.0'), VersionClass.newest),
(Package(repo='2', family='2', name='a',
version='1.0'), VersionClass.outdated),
(Package(repo='3',
family='3',
name='a',
version='1.0',
flags=PackageFlags.legacy), VersionClass.legacy),
]
FillPackagesetVersions([package for package, _ in packages])
for package, expectedclass in packages:
self.assertEqual(package.versionclass,
expectedclass,
msg='repo {}, pkg {}, ver {}'.format(
package.repo, package.name, package.version))
def main():
options = parse_arguments()
logger = StderrLogger()
if options.logfile:
logger = FileLogger(options.logfile)
if options.fields == 'all':
options.fields = sorted(Package().__dict__.keys())
else:
options.fields = options.fields.split(',')
repomgr = RepositoryManager(options.repos_dir)
repoproc = RepositoryProcessor(repomgr, options.statedir,
options.parseddir)
logger.Log('dumping...')
for packageset in repoproc.iter_parsed(reponames=options.reponames):
FillPackagesetVersions(packageset)
if not options.all and packageset_is_shadow_only(packageset):
continue
if options.dump == 'packages':
for package in packageset:
print(
options.field_separator.join(
(format_package_field(field, getattr(package, field))
for field in options.fields)))
if options.dump == 'summaries':
print(packageset[0].effname)
best_pkg_by_repo = PackagesetToBestByRepo(packageset)
for reponame in repomgr.GetNames(options.reponames):
if reponame in best_pkg_by_repo:
print(' {}: {} ({})'.format(
reponame, best_pkg_by_repo[reponame].version,
VersionClass.ToString(
best_pkg_by_repo[reponame].versionclass)))
return 0
def test_versionclasses_big(self):
packages = [
# Reference repo
(Package(repo='1', family='1', name='a', version='2.2.20990101', ignoreversion=True), VersionClass.ignored),
(Package(repo='1', family='1', name='a', version='2.2beta1', devel=True), VersionClass.devel),
(Package(repo='1', family='1', name='a', version='2.2alpha1.20990101', devel=True, ignoreversion=True), VersionClass.legacy),
(Package(repo='1', family='1', name='a', version='2.2alpha1', devel=True), VersionClass.legacy),
# see #338. There are multiple possible ways to ignored version between branches,
# we go with ignored for now
(Package(repo='1', family='1', name='a', version='2.1.20990101', ignoreversion=True), VersionClass.ignored),
(Package(repo='1', family='1', name='a', version='2.1'), VersionClass.newest),
(Package(repo='1', family='1', name='a', version='2.0.20990101', ignoreversion=True), VersionClass.legacy),
(Package(repo='1', family='1', name='a', version='2.0'), VersionClass.legacy),
(Package(repo='1', family='1', name='a', version='1.2.20990101', ignoreversion=True), VersionClass.legacy),
(Package(repo='1', family='1', name='a', version='1.2beta1', devel=True), VersionClass.legacy),
(Package(repo='1', family='1', name='a', version='1.2alpha1.20990101', ignoreversion=True), VersionClass.legacy),
(Package(repo='1', family='1', name='a', version='1.2alpha1', devel=True), VersionClass.legacy),
(Package(repo='1', family='1', name='a', version='1.1.20990101', ignoreversion=True), VersionClass.legacy),
(Package(repo='1', family='1', name='a', version='1.1'), VersionClass.legacy),
(Package(repo='1', family='1', name='a', version='1.0.20990101', ignoreversion=True), VersionClass.legacy),
(Package(repo='1', family='1', name='a', version='1.0'), VersionClass.legacy),
# devel + legacy
(Package(repo='2', family='2', name='a', version='2.2beta1', devel=True), VersionClass.devel),
(Package(repo='2', family='2', name='a', version='2.0'), VersionClass.outdated),
# devel + newest + legacy
(Package(repo='3', family='3', name='a', version='2.2beta1', devel=True), VersionClass.devel),
(Package(repo='3', family='3', name='a', version='2.1'), VersionClass.newest),
(Package(repo='3', family='3', name='a', version='2.0'), VersionClass.legacy),
# newest + legacy
(Package(repo='4', family='4', name='a', version='2.1'), VersionClass.newest),
(Package(repo='4', family='4', name='a', version='2.0'), VersionClass.legacy),
# outdated + legacy
(Package(repo='5', family='5', name='a', version='1.1'), VersionClass.outdated),
(Package(repo='5', family='5', name='a', version='1.0'), VersionClass.legacy),
# outdated outdated/ignored + legacy
(Package(repo='6', family='6', name='a', version='1.1.20990101', ignoreversion=True), VersionClass.outdated),
(Package(repo='6', family='6', name='a', version='1.1'), VersionClass.legacy),
(Package(repo='6', family='6', name='a', version='1.0'), VersionClass.legacy),
# devel class should be ignored for newest version and below
(Package(repo='7', family='7', name='a', version='2.1', devel=True), VersionClass.newest),
# ignored classes are unignored when they are backed with real classes
(Package(repo='8', family='8', name='a', version='2.2beta1', devel=True, ignoreversion=True), VersionClass.devel),
(Package(repo='9', family='9', name='a', version='2.1', ignoreversion=True), VersionClass.newest),
(Package(repo='10', family='10', name='a', version='2.0', ignoreversion=True), VersionClass.outdated),
(Package(repo='10', family='10', name='a', version='1.9', ignoreversion=True), VersionClass.legacy),
# version between newest and devel should be outdated when there's no devel
(Package(repo='11', family='11', name='a', version='2.2alpha1', devel=True), VersionClass.outdated),
# outdated in devel and normal at the same time
(Package(repo='12', family='12', name='a', version='2.2alpha1', devel=True), VersionClass.outdated),
(Package(repo='12', family='12', name='a', version='2.0'), VersionClass.outdated),
]
FillPackagesetVersions([package for package, _ in packages])
for package, expectedclass in packages:
self.assertEqual(package.versionclass, expectedclass, msg='repo {}, pkg {}, ver {}'.format(package.repo, package.name, package.version))