def incorrect_action(package: Package,
package_context: PackageContext,
match_context: MatchContext) -> None:
package.set_flag(PackageFlags.INCORRECT, incorrect_flag)
def ignore_action(package: Package,
package_context: PackageContext,
match_context: MatchContext) -> None:
package.set_flag(PackageFlags.IGNORE, ignore_flag)
def stable_action(package: Package,
package_context: PackageContext,
match_context: MatchContext) -> None:
package.set_flag(PackageFlags.STABLE, stable_flag)
def __init__(self, logger, ident, itemno, skipfailed=False):
super(PackageMaker, self).__init__(logger)
self.package = Package()
self.ident = ident
self.itemno = itemno
self.skipfailed = skipfailed
def is_p_is_patch_matcher(package: Package,
package_context: PackageContext,
match_context: MatchContext) -> bool:
return package.has_flag(
PackageFlags.P_IS_PATCH) == is_p_is_patch
def Parse(self, path):
result = []
for category in os.listdir(path):
category_path = os.path.join(path, category)
if not os.path.isdir(category_path):
continue
if category == 'virtual' or category == 'metadata':
continue
for package in os.listdir(category_path):
package_path = os.path.join(category_path, package)
if not os.path.isdir(package_path):
continue
metadata_path = os.path.join(package_path, 'metadata.xml')
# parse maintainers from metadata.xml
# these are the same for all ebuilds for current package
maintainers = []
if os.path.isfile(metadata_path):
with open(metadata_path, 'r',
encoding='utf-8') as metafile:
meta = xml.etree.ElementTree.parse(metafile)
for entry in meta.findall('maintainer'):
email_node = entry.find('email')
if email_node is not None and email_node.text is not None:
maintainers += GetMaintainers(email_node.text)
if not maintainers:
# If we have no maintainer set, assign Gentoo's default maintainer value
# See https://wiki.gentoo.org/wiki/GLEP:67#Bug_assignment
maintainers = ['*****@*****.**']
for ebuild in os.listdir(package_path):
if not ebuild.endswith('.ebuild'):
continue
pkg = Package()
pkg.name = package
pkg.category = category
pkg.maintainers = maintainers
pkg.version, pkg.origversion = SanitizeVersion(
ebuild[len(package) + 1:-7])
if pkg.version.endswith('9999'):
# ignore versions for snapshots
pkg.ignoreversion = True
metadata_path = os.path.join(
path, 'metadata', 'md5-cache', category,
package + '-' +
(pkg.origversion if pkg.origversion else pkg.version))
if os.path.isfile(metadata_path):
with open(metadata_path, 'r',
encoding='utf-8') as metadata_file:
for line in metadata_file:
line = line.strip()
key, value = line.split('=', 1)
if key == 'DESCRIPTION':
pkg.comment = value
elif key == 'HOMEPAGE':
pkg.homepage = value.split(' ')[
0] # XXX: save all urls
elif key == 'LICENSE':
if value.find('(') != -1:
# XXX: conditionals and OR's: need more
# complex parsing and backend support
pkg.licenses.append(value)
else:
pkg.licenses += value.split(' ')
elif key == 'SRC_URI':
pkg.downloads += ParseConditionalExpr(
value)
result.append(pkg)
return result
def assert_version_compare(self, a: Package, b: Package, res: int) -> None:
self.assertEqual(a.version_compare(b), res)
def debianism_action(package: Package,
package_context: PackageContext,
match_context: MatchContext) -> None:
# XXX: the same as devel for now
package.set_flag(PackageFlags.DEVEL, debianism_flag)
def generated_action(package: Package,
package_context: PackageContext,
match_context: MatchContext) -> None:
# XXX: the same as rolling for now
package.set_flag(PackageFlags.ROLLING, generated_flag)
def snapshot_action(package: Package,
package_context: PackageContext,
match_context: MatchContext) -> None:
# XXX: the same as ignored for now
package.set_flag(PackageFlags.IGNORE, snapshot_flag)
def successor_action(package: Package,
package_context: PackageContext,
match_context: MatchContext) -> None:
# XXX: the same as devel for now
package.set_flag(PackageFlags.DEVEL, successor_flag)
def rolling_action(package: Package,
package_context: PackageContext,
match_context: MatchContext) -> None:
package.set_flag(PackageFlags.ROLLING, rolling_flag)
def noscheme_action(package: Package,
package_context: PackageContext,
match_context: MatchContext) -> None:
package.set_flag(PackageFlags.NOSCHEME, noscheme_flag)
def untrusted_action(package: Package,
package_context: PackageContext,
match_context: MatchContext) -> None:
package.set_flag(PackageFlags.UNTRUSTED, untrusted_flag)
class PackageMaker(PackageMakerBase):
_package: Package
_ident: Optional[str]
_itemno: int
_skipfailed: bool
def __init__(self, logger: Logger, ident: Optional[str], itemno: int, skipfailed: bool = False) -> None:
super(PackageMaker, self).__init__(logger)
self._package = Package()
self._ident = ident
self._itemno = itemno
self._skipfailed = skipfailed
def _get_ident(self) -> str:
return self._ident or self._package.extrafields.get('origin', None) or self._package.name or self._package.basename or 'item #{}'.format(self._itemno)
@PackageMakerBase._simple_setter('origin', str, nzs.strip, nzs.forbid_newlines)
def set_origin(self, origin: str) -> None:
# XXX: convert to dedicated field
self.set_extra_field('origin', origin)
@PackageMakerBase._simple_setter('name', str, nzs.strip, nzs.forbid_newlines)
def set_name(self, name: str) -> None:
self._package.name = name
@PackageMakerBase._simple_setter('basename', str, nzs.strip, nzs.forbid_newlines)
def set_basename(self, basename: str) -> None:
self._package.basename = basename
def prefix_name(self, prefix: str) -> None:
self._package.name = prefix + self._package.name
@PackageMakerBase._simple_setter('version', str, nzs.strip, nzs.forbid_newlines)
def set_version(self, version: str, version_normalizer: Optional[Callable[[str], str]] = None) -> None:
self._package.rawversion = version
self._package.origversion = version if version_normalizer is None else version_normalizer(version)
self._package.version = self._package.origversion
@PackageMakerBase._simple_setter('version', str, nzs.strip, nzs.forbid_newlines)
def set_rawversion(self, rawversion: str) -> None:
if rawversion != self._package.version:
self._package.rawversion = rawversion
def set_name_and_version(self, namever: str, version_normalizer: Optional[Callable[[str], str]] = None) -> None:
name, version = namever.rsplit('-', 1)
self.set_name(name)
self.set_version(version, version_normalizer)
@PackageMakerBase._simple_setter('arch', str, nzs.strip, nzs.forbid_newlines)
def set_arch(self, arch: str) -> None:
self._package.arch = arch
@PackageMakerBase._simple_setter('summary', str, nzs.strip)
def set_summary(self, summary: str) -> None:
self._package.comment = summary
@PackageMakerBase._omnivorous_setter('maintainer', str, nzs.strip, nzs.forbid_newlines, nzs.tolower)
def add_maintainers(self, *args: Any) -> None:
self._package.maintainers.extend(_iter_unique(args, self._package.maintainers))
@PackageMakerBase._omnivorous_setter('category', str, nzs.strip, nzs.forbid_newlines)
def add_categories(self, *args: Any) -> None:
# XXX: convert into array
if not self._package.category:
self._package.category = args[0]
@PackageMakerBase._omnivorous_setter('homepage', str, nzs.strip, nzs.url, nzs.warn_whitespace, nzs.forbid_newlines)
def add_homepages(self, *args: Any) -> None:
# XXX: convert into array
if not self._package.homepage:
self._package.homepage = args[0]
@PackageMakerBase._omnivorous_setter('license', str, nzs.strip, nzs.forbid_newlines)
def add_licenses(self, *args: Any) -> None:
self._package.licenses.extend(args)
@PackageMakerBase._omnivorous_setter('download', str, nzs.strip, nzs.url, nzs.warn_whitespace, nzs.forbid_newlines)
def add_downloads(self, *args: Any) -> None:
self._package.downloads.extend(_iter_unique(args, self._package.downloads))
def set_flags(self, mask: int, is_set: bool = True) -> None:
self._package.SetFlag(mask, is_set)
def set_extra_field(self, key: str, value: str) -> None:
self._package.extrafields[key] = value
def unwrap(self) -> Package:
return self._package
def clone(self, ident: Optional[str] = None, append_ident: Optional[str] = None) -> 'PackageMaker':
offspring_ident = self._ident
if ident is not None:
offspring_ident = ident
elif append_ident is not None:
offspring_ident = (offspring_ident or '') + append_ident
offspring = PackageMaker(self._logger, offspring_ident, self._itemno)
offspring._package = deepcopy(self._package)
return offspring
def check_sanity(self, require_name: bool = True, require_version: bool = True, verbose: bool = False) -> bool:
if require_name and not self._package.name:
if verbose:
self.log('package with no name', severity=Logger.ERROR)
return False
if require_version and not self._package.version:
if verbose:
self.log('package with no version', severity=Logger.ERROR)
return False
return True
def __getattr__(self, key: str) -> Any:
return getattr(self._package, key)
def __enter__(self) -> 'PackageMaker':
return self
def __exit__(self, exc_type: Any, exc_value: Any, traceback: Any) -> Optional[bool]:
if exc_type:
self.log('parsing failed ({}): {}: {}'.format(
'skipped' if self._skipfailed else 'fatal',
exc_type.__name__,
exc_value
), severity=Logger.ERROR)
if self._skipfailed:
return True
return None
def trace_action(package: Package, package_context: PackageContext,
match_context: MatchContext) -> None:
package.set_flag(PackageFlags.TRACE, trace_flag)
def __init__(self, logger: Logger, ident: Optional[str], itemno: int, skipfailed: bool = False) -> None:
super(PackageMaker, self).__init__(logger)
self._package = Package()
self._ident = ident
self._itemno = itemno
self._skipfailed = skipfailed
def resetflavors_action(package: Package,
package_context: PackageContext,
match_context: MatchContext) -> None:
package.flavors = []
def Parse(self, path):
result = []
with open(path, encoding='utf-8', errors='ignore') as file:
current_data = {}
last_key = None
for line in file:
line = line.rstrip('\n')
# empty line, dump package
if line == '':
if not current_data:
continue # may happen on empty package list
pkg = Package()
def GetField(key, type_=str, default=None):
if key in current_data:
if type_ is None or isinstance(
current_data[key], type_):
return current_data[key]
else:
print(
'WARNING: unable to parse field {}'.format(
key),
file=sys.stderr)
return default
else:
return default
pkg.name = GetField('Package')
pkg.version, pkg.origversion = SanitizeVersion(
GetField('Version'))
pkg.maintainers += GetMaintainers(
GetField('Maintainer', default=''))
pkg.maintainers += GetMaintainers(
GetField('Uploaders', default=''))
pkg.category = GetField('Section')
pkg.homepage = GetField('Homepage')
# This is long description
#pkg.comment = GetField('Description', type_=None)
#if isinstance(pkg.comment, list):
# pkg.comment = ' '.join(pkg.comment)
if pkg.name and pkg.version:
result.append(pkg)
else:
print('WARNING: unable to parse package {}'.format(
str(current_data)),
file=sys.stderr)
current_data = {}
last_key = None
continue
# key - value pair
match = re.fullmatch('([A-Za-z0-9-]+):(.*?)', line)
if match:
key = match.group(1)
value = match.group(2).strip()
current_data[key] = value
last_key = key
continue
# continuation of previous key
match = re.fullmatch(' (.*)', line)
if match:
value = match.group(1).strip()
if not isinstance(current_data[last_key], list):
current_data[last_key] = [current_data[last_key]]
current_data[last_key].append(value)
continue
print('WARNING: unable to parse line: {}'.format(line),
file=sys.stderr)
return result
def setname_action(package: Package,
package_context: PackageContext,
match_context: MatchContext) -> None:
package.effname = match_context.sub_name_dollars(
setname, package.effname)
class PackageMaker(PackageMakerBase):
def __init__(self, logger, ident, itemno, skipfailed=False):
super(PackageMaker, self).__init__(logger)
self.package = Package()
self.ident = ident
self.itemno = itemno
self.skipfailed = skipfailed
def _get_ident(self):
return self.ident or self.package.extrafields.get(
'origin', None
) or self.package.name or self.package.basename or 'item #{}'.format(
self.itemno)
@PackageMakerBase._simple_setter('origin', str, nzs.strip,
nzs.forbid_newlines)
def set_origin(self, origin):
# XXX: convert to dedicated field
self.set_extra_field('origin', origin)
@PackageMakerBase._simple_setter('name', str, nzs.strip,
nzs.forbid_newlines)
def set_name(self, name):
self.package.name = name
@PackageMakerBase._simple_setter('name', str, nzs.strip,
nzs.forbid_newlines)
def set_basename(self, basename):
self.package.basename = basename
def prefix_name(self, prefix):
self.package.name = prefix + self.package.name
@PackageMakerBase._simple_setter('version', str, nzs.strip,
nzs.forbid_newlines)
def set_version(self, version, version_normalizer=None):
self.package.rawversion = version
self.package.origversion = version if version_normalizer is None else version_normalizer(
version)
self.package.version = self.package.origversion
@PackageMakerBase._simple_setter('version', str, nzs.strip,
nzs.forbid_newlines)
def set_rawversion(self, rawversion):
if rawversion != self.package.version:
self.package.rawversion = rawversion
def set_name_and_version(self, namever, version_normalizer=None):
name, version = namever.rsplit('-', 1)
self.set_name(name)
self.set_version(version, version_normalizer)
@PackageMakerBase._simple_setter('summary', str, nzs.strip)
def set_summary(self, summary):
self.package.comment = summary
@PackageMakerBase._omnivorous_setter('maintainer', str, nzs.strip,
nzs.forbid_newlines, nzs.tolower)
def add_maintainers(self, *args):
self.package.maintainers.extend(
_iter_unique(args, self.package.maintainers))
@PackageMakerBase._omnivorous_setter('category', str, nzs.strip,
nzs.forbid_newlines)
def add_categories(self, *args):
# XXX: convert into array
if not self.package.category:
self.package.category = args[0]
@PackageMakerBase._omnivorous_setter('homepage', str, nzs.strip, nzs.url,
nzs.warn_whitespace,
nzs.forbid_newlines)
def add_homepages(self, *args):
# XXX: convert into array
if not self.package.homepage:
self.package.homepage = args[0]
@PackageMakerBase._omnivorous_setter('license', str, nzs.strip,
nzs.forbid_newlines)
def add_licenses(self, *args):
self.package.licenses.extend(args)
@PackageMakerBase._omnivorous_setter('download', str, nzs.strip, nzs.url,
nzs.warn_whitespace,
nzs.forbid_newlines)
def add_downloads(self, *args):
self.package.downloads.extend(
_iter_unique(args, self.package.downloads))
def set_flags(self, mask, is_set=True):
assert (isinstance(mask, int))
assert (isinstance(is_set, bool))
self.package.SetFlag(mask, is_set)
def set_extra_field(self, key, value):
assert (isinstance(key, str))
assert (isinstance(value, str))
self.package.extrafields[key] = value
def unwrap(self):
return self.package
def clone(self, ident=None, append_ident=None):
offspring_ident = self.ident
if ident is not None:
offspring_ident = ident
elif append_ident is not None:
offspring_ident += append_ident
offspring = PackageMaker(self.logger, offspring_ident, self.itemno)
offspring.package = deepcopy(self.package)
return offspring
def check_sanity(self,
require_name=True,
require_version=True,
verbose=False):
if require_name and not self.package.name:
if verbose:
self.log('package with no name', severity=Logger.ERROR)
return False
if require_version and not self.package.version:
if verbose:
self.log('package with no version', severity=Logger.ERROR)
return False
return True
def __getattr__(self, key):
return getattr(self.package, key)
def __enter__(self):
return self
def __exit__(self, exc_type, exc_value, traceback):
if exc_type:
self.log('parsing failed ({}): {}: {}'.format(
'skipped' if self.skipfailed else 'fatal', exc_type.__name__,
exc_value),
severity=Logger.ERROR)
if self.skipfailed:
return True
def setver_action(package: Package,
package_context: PackageContext,
match_context: MatchContext) -> None:
package.version = match_context.sub_ver_dollars(
setver, package.version)
def compare(p1: Package, p2: Package) -> int:
return p2.version_compare(p1)
def replaceinname_action(package: Package,
package_context: PackageContext,
match_context: MatchContext) -> None:
for pattern, replacement in replace_items:
package.effname = package.effname.replace(
pattern, replacement)
def remove_action(package: Package,
package_context: PackageContext,
match_context: MatchContext) -> None:
package.set_flag(PackageFlags.REMOVE, remove_flag)
def tolowername_action(package: Package,
package_context: PackageContext,
match_context: MatchContext) -> None:
package.effname = package.effname.lower()
def weak_devel_action(package: Package,
package_context: PackageContext,
match_context: MatchContext) -> None:
# XXX: currently sets ignore; change to set non-viral variant of devel (#654)
package.set_flag(PackageFlags.IGNORE, weak_devel_flag)
def iter_parse(self, path, logger):
with open(path, 'r', encoding='utf-8') as jsonfile:
for key, packagedata in json.load(jsonfile)['packages'].items():
# see how Nix parses 'derivative' names in
# https://github.com/NixOS src/libexpr/names.cc, DrvName::DrvName
# it just splits on dash followed by non-letter
#
# this doesn't work well on 100% cases, it's an upstream problem
match = re.match('(.+?)-([^a-zA-Z].*)$', packagedata['name'])
if not match:
print('cannot extract version: {}/{}'.format(
key, packagedata['name']),
file=sys.stderr)
continue
pkg = Package()
pkg.name = match.group(1)
pkg.version = match.group(2)
# some exceptions
for prefix in ('75dpi', '100dpi'):
if pkg.version.startswith(prefix):
pkg.name += '-' + prefix
pkg.version = pkg.version[len(prefix) + 1:]
merged = pkg.name + '-' + pkg.version
for pkgname in [
'liblqr-1', 'python2.7-3to2', 'python3.6-3to2'
]:
if merged.startswith(pkgname):
pkg.name = pkgname
pkg.version = merged[len(pkgname) + 1:]
keyparts = key.split('.')
if len(keyparts) > 1:
pkg.category = keyparts[0]
if pkg.name.endswith('-git'):
pkg.name = pkg.name[:-4]
pkg.SetFlag(PackageFlags.ignore)
if re.match('.*20[0-9]{2}-[0-9]{2}-[0-9]{2}', pkg.version):
pkg.SetFlag(PackageFlags.ignore)
if re.match('[0-9a-f]*[a-f][0-9a-f]*$',
pkg.version) and len(pkg.version) >= 7:
print(
'ignoring version which looks like commit hash: {}/{}'.
format(key, packagedata['name']),
file=sys.stderr)
pkg.SetFlag(PackageFlags.ignore)
meta = packagedata['meta']
if 'homepage' in meta:
pkg.homepage = meta['homepage']
if isinstance(
pkg.homepage, list
): # XXX: remove after adding support for homepages array
pkg.homepage = pkg.homepage[0]
if 'description' in meta and meta['description']:
pkg.comment = meta['description'].replace('\n',
' ').strip()
if 'maintainers' in meta:
if not isinstance(meta['maintainers'], list):
print('maintainers is not a list: {}/{}'.format(
key, packagedata['name']),
file=sys.stderr)
else:
pkg.maintainers += list(
extract_nix_maintainers(meta['maintainers']))
if 'license' in meta:
pkg.licenses = extract_nix_licenses(meta['license'])
if 'position' in meta:
posfile, posline = meta['position'].rsplit(':', 1)
pkg.extrafields['posfile'] = posfile
pkg.extrafields['posline'] = posline
yield pkg
def devel_action(package: Package, package_context: PackageContext,
match_context: MatchContext) -> None:
package.set_flag(PackageFlags.DEVEL, devel_flag)
def legacy_action(package: Package,
package_context: PackageContext,
match_context: MatchContext) -> None:
package.set_flag(PackageFlags.LEGACY, legacy_flag)
