def iter_parse(self, path: str, factory: PackageFactory) -> Iterable[PackageMaker]:
normalize_version = VersionStripper().strip_right(',').strip_right('_')
for pkgname, pkgdata in iter_json_dict(path, ('packages', None)):
with factory.begin() as pkg:
pkg.add_name(pkgname, NameType.BSD_PKGNAME)
pkg.add_name(pkgdata['port'], NameType.BSD_ORIGIN)
pkg.set_version(pkgdata['version'], normalize_version)
pkg.set_summary(pkgdata['summary'])
pkg.add_categories(item['category'] for item in pkgdata['categories'])
pkg.add_licenses(pkgdata['licenses'])
pkg.add_homepages(pkgdata['homepages'])
yield pkg
def iter_parse(self, path: str,
factory: PackageFactory) -> Iterable[PackageMaker]:
normalize_version = VersionStripper().strip_left(':')
for key, pkgdata in iter_json_dict(path, ('packages', None)):
with factory.begin(key) as pkg:
pkg.add_name(pkgdata['name'], NameType.SPACK_NAME)
pkg.add_homepages(pkgdata['homepages'])
pkg.add_maintainers(f'{m}@spack'
for m in pkgdata['maintainers'])
pkg.set_extra_field('patch', [
patch.split()[0]
for patch in pkgdata['patches'] if '://' not in patch
])
# - no usable keywords/categories (yet)
# - summaries are multiline, so ignored
# - dependencies info is available, not used yet
# spack may contain a lot of versions for a single project,
# we don't handle that very well, so pick greatest release
# version and all rolling versions
picked_verdatas: list[dict[str, Any]] = []
latest_release_verdata: dict[str, Any] | None = None
for pkgverdata in pkgdata['version']:
if 'branch' in pkgverdata:
picked_verdatas.append(pkgverdata)
elif latest_release_verdata is None or version_compare(
pkgverdata['version'],
latest_release_verdata['version']) > 0:
latest_release_verdata = pkgverdata
if latest_release_verdata:
picked_verdatas.append(latest_release_verdata)
for pkgverdata in picked_verdatas:
verpkg = pkg.clone()
if 'branch' in pkgverdata:
verpkg.set_flags(PackageFlags.ROLLING)
verpkg.set_version(pkgverdata['version'],
normalize_version)
verpkg.add_downloads(pkgverdata['downloads'])
yield verpkg
def iter_parse(self, path: str, factory: PackageFactory) -> Iterable[PackageMaker]:
for packagename, packagedata in iter_json_dict(path, ('packages', None)):
with factory.begin() as pkg:
pkg.add_name(packagename, NameType.JUSTINSTALL_NAME)
pkg.set_version(packagedata['version'])
for arch in ['x86', 'x86_64']:
if arch in packagedata['installer']:
installer = packagedata['installer'][arch]
# https://github.com/just-install/registry/blob/master/docs/registry.md#placeholders
installer = installer.replace('{{.version}}', packagedata['version'])
if '{{.' in installer:
raise RuntimeError('Unhandled replacement: {}'.format(installer))
pkg.add_downloads(installer)
yield pkg
def iter_parse(self, path: str, factory: PackageFactory,
transformer: PackageTransformer) -> Iterable[PackageMaker]:
for key, packagedata in iter_json_dict(path, ('packages', None),
encoding='utf-8'):
with factory.begin(key) as pkg:
# these should eventually go away as soon as the data is fixed
# in nix (e.g. via manual pnames) and is properly exposed
if 'node-_at' in packagedata['name']:
pkg.log('dropping, garbage name "{}"'.format(
packagedata['name']),
severity=Logger.ERROR)
continue
if '--' in packagedata['name']:
pkg.log('dropping, garbage name "{}"'.format(
packagedata['name']),
severity=Logger.ERROR)
continue
if packagedata['name'].startswith('luajit-2.1.0-beta3-'):
pkg.log('dropping, garbage name "{}"'.format(
packagedata['name']),
severity=Logger.ERROR)
continue
if packagedata['name'].startswith('palp-6d'):
pkg.log('dropping, garbage name "{}"'.format(
packagedata['name']),
severity=Logger.ERROR)
continue
if re.match('cudatoolkit-[0-9.-]+-cudnn', packagedata['name']):
pkg.log('dropping, garbage name "{}"'.format(
packagedata['name']),
severity=Logger.ERROR)
continue
if re.match('nccl-[0-9.-]+-cuda', packagedata['name']):
pkg.log('dropping, garbage name "{}"'.format(
packagedata['name']),
severity=Logger.ERROR)
continue
skip = False
if packagedata['pname'] in _BLACKLIST1:
for verprefix in _BLACKLIST1[packagedata['pname']]:
if packagedata['version'].startswith(verprefix):
pkg.log(
'dropping {}, "{}" does not belong to version'.
format(packagedata['name'], verprefix),
severity=Logger.ERROR)
skip = True
break
if packagedata['pname'] in _BLACKLIST2:
pkg.log('dropping {}, "{}" does not belong to name'.format(
packagedata['name'],
packagedata['pname'].rsplit('-')[-1]),
severity=Logger.ERROR)
skip = True
for verprefix in ['100dpi', '75dpi']:
if packagedata['version'].startswith(verprefix):
pkg.log(
'dropping "{}", "{}" does not belong to version'.
format(packagedata['name'], verprefix),
severity=Logger.ERROR)
skip = True
break
if skip:
continue
if not packagedata['version']:
continue # no version - silently ignore
if re.match('[0-9].*[a-z].*-[0-9]',
packagedata['version'].lower()):
letters = ''.join(c
for c in packagedata['version'].lower()
if c.isalpha())
if letters not in [
'alpha', 'beta', 'rc', 'a', 'b', 'pre', 'post',
'rev', 'q', 'u', 'build', 'unstable'
]:
pkg.log(
'"{}": suspicious version "{}", worth rechecking'.
format(packagedata['name'],
packagedata['version']),
severity=Logger.WARNING)
pname = packagedata['pname']
version = packagedata['version']
# This is temporary solution (see #854) which overrides pname and version with ones
# (ambigiously) parsed from name. That's what nix currently does (instead of exposing
# explicitly set pname and version), and we do the same instead of using pname/version
# provided by them to avoid unexpected change in data when/if they change their logic
# As soon as they do and changed data is verified, this block may be removed
match = re.match('(.+?)-([0-9].*)$', packagedata['name'])
if match is None:
pkg.log('cannot parse name "{}"'.format(
packagedata['name']),
severity=Logger.ERROR)
continue
else:
pname = match.group(1)
version = match.group(2)
pkg.add_name(key, NameType.NIX_ATTRIBUTE_PATH)
pkg.add_name(pname, NameType.NIX_PNAME)
pkg.set_version(version)
meta = packagedata['meta']
keyparts = key.split('.')
if len(keyparts) > 1:
pkg.add_categories(keyparts[0])
# XXX: move to rules
if pname.endswith('-git'):
pkg.add_name(pname[:-4], NameType.NIX_PNAME)
pkg.set_flags(PackageFlags.IGNORE)
# XXX: move to rules
if re.match('.*20[0-9]{2}-[0-9]{2}-[0-9]{2}', pkg.version):
pkg.set_flags(PackageFlags.IGNORE)
# XXX: move to rules
if re.match('[0-9a-f]*[a-f][0-9a-f]*$',
pkg.version) and len(pkg.version) >= 7:
pkg.set_flags(PackageFlags.IGNORE)
pkg.add_homepages(meta.get('homepage'))
if 'description' in meta:
pkg.set_summary(meta['description'].replace('\n', ' '))
if 'maintainers' in meta:
if not isinstance(meta['maintainers'], list):
pkg.log('maintainers "{}" is not a list'.format(
meta['maintainers']),
severity=Logger.ERROR)
else:
pkg.add_maintainers(
extract_nix_maintainers(meta['maintainers']))
if 'license' in meta:
pkg.add_licenses(extract_nix_licenses(meta['license']))
if 'position' in meta:
posfile, posline = meta['position'].rsplit(':', 1)
pkg.set_extra_field('posfile', posfile)
pkg.set_extra_field('posline', posline)
if posfile.startswith('pkgs/development/haskell-modules'):
pkg.set_flags(
PackageFlags.ROLLING
) # XXX: haskell modules are autogenerated in nix: https://github.com/NixOS/nixpkgs/commits/master/pkgs/development/haskell-modules/hackage-packages.nix
if posfile.startswith(
'pkgs/applications/editors/emacs-modes'):
pkg.set_flags(
PackageFlags.UNTRUSTED) # XXX: uses MALPA versions
else:
pkg.log('dropping, no position recorded in meta',
severity=Logger.ERROR)
continue
yield pkg
def iter_parse(self, path: str, factory: PackageFactory,
transformer: PackageTransformer) -> Iterable[PackageMaker]:
for key, packagedata in iter_json_dict(path, ('packages', None),
encoding='utf-8'):
with factory.begin(key) as pkg:
meta = packagedata['meta']
# see #854; first, try new mode which relies on dedicated version field
if 'version' in meta:
if not packagedata['name'].endswith('-' + meta['version']):
pkg.log(
'name "{}" does not end with version "{}"'.format(
packagedata['name'], meta['version']),
severity=Logger.ERROR)
else:
pkg.set_name(
packagedata['name'][:-len(meta['version']) - 1])
pkg.set_version(meta['version'])
# and fallback to old method which splits name-version pair
if not pkg.version:
# matches most of exceptions mentioned below
if re.search('-[0-9]+[a-z]', packagedata['name']):
pkg.log(
'possibly ambiguous package name "{}", consider adding explicit version'
.format(packagedata['name']),
severity=Logger.WARNING)
# useless for now due to too many matches
#if re.search('-[^a-zA-Z].*-[^a-zA-Z]', packagedata['name']) and not re.match('.*20[0-9]{2}-[0-9]{2}-[0-9]{2}', packagedata['name']):
# pkg.log('possibly ambiguous package name/version pair: {}'.format(packagedata['name']), severity=Logger.WARNING)
# see how Nix parses 'derivative' names in
# https://github.com/NixOS src/libexpr/names.cc, DrvName::DrvName
# it just splits on dash followed by non-letter
#
# this doesn't work well on 100% cases, it's an upstream problem
match = re.match('(.+?)-([^a-zA-Z].*)$',
packagedata['name'])
if not match:
pkg.log('cannot extract version from "{}"'.format(
packagedata['name']),
severity=Logger.ERROR)
continue
if 'node-_at_webassemblyjs' in packagedata['name']:
pkg.log('skipping garbage name "{}"'.format(
packagedata['name']),
severity=Logger.ERROR)
continue
pkg.set_name(match.group(1))
pkg.set_version(match.group(2))
# some exceptions
for prefix in ('75dpi', '100dpi'):
if pkg.version.startswith(prefix):
pkg.set_name(pkg.name + '-' + prefix)
pkg.set_version(pkg.version[len(prefix) + 1:])
merged = pkg.name + '-' + pkg.version
for pkgname in [
'liblqr-1', 'python2.7-3to2', 'python3.6-3to2',
'libretro-4do', 'polkit-qt-1-qt5',
'polkit-qt-1-qt4'
]:
if merged.startswith(pkgname):
pkg.set_name(pkgname)
pkg.set_version(merged[len(pkgname) + 1:])
keyparts = key.split('.')
if len(keyparts) > 1:
pkg.add_categories(keyparts[0])
# XXX: mode to rules
if pkg.name.endswith('-git'):
pkg.set_name(pkg.name[:-4])
pkg.set_flags(PackageFlags.IGNORE)
# XXX: mode to rules
if re.match('.*20[0-9]{2}-[0-9]{2}-[0-9]{2}', pkg.version):
pkg.set_flags(PackageFlags.IGNORE)
# XXX: mode to rules
if re.match('[0-9a-f]*[a-f][0-9a-f]*$',
pkg.version) and len(pkg.version) >= 7:
pkg.set_flags(PackageFlags.IGNORE)
pkg.add_homepages(meta.get('homepage'))
if 'description' in meta:
pkg.set_summary(meta['description'].replace('\n', ' '))
if 'maintainers' in meta:
if not isinstance(meta['maintainers'], list):
pkg.log('maintainers "{}" is not a list'.format(
meta['maintainers']),
severity=Logger.ERROR)
else:
pkg.add_maintainers(
extract_nix_maintainers(meta['maintainers']))
if 'license' in meta:
pkg.add_licenses(extract_nix_licenses(meta['license']))
if 'position' in meta:
posfile, posline = meta['position'].rsplit(':', 1)
pkg.set_extra_field('posfile', posfile)
pkg.set_extra_field('posline', posline)
if posfile.startswith('pkgs/development/haskell-modules'):
pkg.set_flags(
PackageFlags.ROLLING
) # XXX: haskell modules are autogenerated in nix: https://github.com/NixOS/nixpkgs/commits/master/pkgs/development/haskell-modules/hackage-packages.nix
yield pkg
