def incorrect_action(package: Package, package_context: PackageContext, match_context: MatchContext) -> None: package.set_flag(PackageFlags.INCORRECT, incorrect_flag)
def ignore_action(package: Package, package_context: PackageContext, match_context: MatchContext) -> None: package.set_flag(PackageFlags.IGNORE, ignore_flag)
def stable_action(package: Package, package_context: PackageContext, match_context: MatchContext) -> None: package.set_flag(PackageFlags.STABLE, stable_flag)
def __init__(self, logger, ident, itemno, skipfailed=False): super(PackageMaker, self).__init__(logger) self.package = Package() self.ident = ident self.itemno = itemno self.skipfailed = skipfailed
def is_p_is_patch_matcher(package: Package, package_context: PackageContext, match_context: MatchContext) -> bool: return package.has_flag( PackageFlags.P_IS_PATCH) == is_p_is_patch
def Parse(self, path): result = [] for category in os.listdir(path): category_path = os.path.join(path, category) if not os.path.isdir(category_path): continue if category == 'virtual' or category == 'metadata': continue for package in os.listdir(category_path): package_path = os.path.join(category_path, package) if not os.path.isdir(package_path): continue metadata_path = os.path.join(package_path, 'metadata.xml') # parse maintainers from metadata.xml # these are the same for all ebuilds for current package maintainers = [] if os.path.isfile(metadata_path): with open(metadata_path, 'r', encoding='utf-8') as metafile: meta = xml.etree.ElementTree.parse(metafile) for entry in meta.findall('maintainer'): email_node = entry.find('email') if email_node is not None and email_node.text is not None: maintainers += GetMaintainers(email_node.text) if not maintainers: # If we have no maintainer set, assign Gentoo's default maintainer value # See https://wiki.gentoo.org/wiki/GLEP:67#Bug_assignment maintainers = ['*****@*****.**'] for ebuild in os.listdir(package_path): if not ebuild.endswith('.ebuild'): continue pkg = Package() pkg.name = package pkg.category = category pkg.maintainers = maintainers pkg.version, pkg.origversion = SanitizeVersion( ebuild[len(package) + 1:-7]) if pkg.version.endswith('9999'): # ignore versions for snapshots pkg.ignoreversion = True metadata_path = os.path.join( path, 'metadata', 'md5-cache', category, package + '-' + (pkg.origversion if pkg.origversion else pkg.version)) if os.path.isfile(metadata_path): with open(metadata_path, 'r', encoding='utf-8') as metadata_file: for line in metadata_file: line = line.strip() key, value = line.split('=', 1) if key == 'DESCRIPTION': pkg.comment = value elif key == 'HOMEPAGE': pkg.homepage = value.split(' ')[ 0] # XXX: save all urls elif key == 'LICENSE': if value.find('(') != -1: # XXX: conditionals and OR's: need more # complex parsing and backend support pkg.licenses.append(value) else: pkg.licenses += value.split(' ') elif key == 'SRC_URI': pkg.downloads += ParseConditionalExpr( value) result.append(pkg) return result
def assert_version_compare(self, a: Package, b: Package, res: int) -> None: self.assertEqual(a.version_compare(b), res)
def debianism_action(package: Package, package_context: PackageContext, match_context: MatchContext) -> None: # XXX: the same as devel for now package.set_flag(PackageFlags.DEVEL, debianism_flag)
def generated_action(package: Package, package_context: PackageContext, match_context: MatchContext) -> None: # XXX: the same as rolling for now package.set_flag(PackageFlags.ROLLING, generated_flag)
def snapshot_action(package: Package, package_context: PackageContext, match_context: MatchContext) -> None: # XXX: the same as ignored for now package.set_flag(PackageFlags.IGNORE, snapshot_flag)
def successor_action(package: Package, package_context: PackageContext, match_context: MatchContext) -> None: # XXX: the same as devel for now package.set_flag(PackageFlags.DEVEL, successor_flag)
def rolling_action(package: Package, package_context: PackageContext, match_context: MatchContext) -> None: package.set_flag(PackageFlags.ROLLING, rolling_flag)
def noscheme_action(package: Package, package_context: PackageContext, match_context: MatchContext) -> None: package.set_flag(PackageFlags.NOSCHEME, noscheme_flag)
def untrusted_action(package: Package, package_context: PackageContext, match_context: MatchContext) -> None: package.set_flag(PackageFlags.UNTRUSTED, untrusted_flag)
class PackageMaker(PackageMakerBase): _package: Package _ident: Optional[str] _itemno: int _skipfailed: bool def __init__(self, logger: Logger, ident: Optional[str], itemno: int, skipfailed: bool = False) -> None: super(PackageMaker, self).__init__(logger) self._package = Package() self._ident = ident self._itemno = itemno self._skipfailed = skipfailed def _get_ident(self) -> str: return self._ident or self._package.extrafields.get('origin', None) or self._package.name or self._package.basename or 'item #{}'.format(self._itemno) @PackageMakerBase._simple_setter('origin', str, nzs.strip, nzs.forbid_newlines) def set_origin(self, origin: str) -> None: # XXX: convert to dedicated field self.set_extra_field('origin', origin) @PackageMakerBase._simple_setter('name', str, nzs.strip, nzs.forbid_newlines) def set_name(self, name: str) -> None: self._package.name = name @PackageMakerBase._simple_setter('basename', str, nzs.strip, nzs.forbid_newlines) def set_basename(self, basename: str) -> None: self._package.basename = basename def prefix_name(self, prefix: str) -> None: self._package.name = prefix + self._package.name @PackageMakerBase._simple_setter('version', str, nzs.strip, nzs.forbid_newlines) def set_version(self, version: str, version_normalizer: Optional[Callable[[str], str]] = None) -> None: self._package.rawversion = version self._package.origversion = version if version_normalizer is None else version_normalizer(version) self._package.version = self._package.origversion @PackageMakerBase._simple_setter('version', str, nzs.strip, nzs.forbid_newlines) def set_rawversion(self, rawversion: str) -> None: if rawversion != self._package.version: self._package.rawversion = rawversion def set_name_and_version(self, namever: str, version_normalizer: Optional[Callable[[str], str]] = None) -> None: name, version = namever.rsplit('-', 1) self.set_name(name) self.set_version(version, version_normalizer) @PackageMakerBase._simple_setter('arch', str, nzs.strip, nzs.forbid_newlines) def set_arch(self, arch: str) -> None: self._package.arch = arch @PackageMakerBase._simple_setter('summary', str, nzs.strip) def set_summary(self, summary: str) -> None: self._package.comment = summary @PackageMakerBase._omnivorous_setter('maintainer', str, nzs.strip, nzs.forbid_newlines, nzs.tolower) def add_maintainers(self, *args: Any) -> None: self._package.maintainers.extend(_iter_unique(args, self._package.maintainers)) @PackageMakerBase._omnivorous_setter('category', str, nzs.strip, nzs.forbid_newlines) def add_categories(self, *args: Any) -> None: # XXX: convert into array if not self._package.category: self._package.category = args[0] @PackageMakerBase._omnivorous_setter('homepage', str, nzs.strip, nzs.url, nzs.warn_whitespace, nzs.forbid_newlines) def add_homepages(self, *args: Any) -> None: # XXX: convert into array if not self._package.homepage: self._package.homepage = args[0] @PackageMakerBase._omnivorous_setter('license', str, nzs.strip, nzs.forbid_newlines) def add_licenses(self, *args: Any) -> None: self._package.licenses.extend(args) @PackageMakerBase._omnivorous_setter('download', str, nzs.strip, nzs.url, nzs.warn_whitespace, nzs.forbid_newlines) def add_downloads(self, *args: Any) -> None: self._package.downloads.extend(_iter_unique(args, self._package.downloads)) def set_flags(self, mask: int, is_set: bool = True) -> None: self._package.SetFlag(mask, is_set) def set_extra_field(self, key: str, value: str) -> None: self._package.extrafields[key] = value def unwrap(self) -> Package: return self._package def clone(self, ident: Optional[str] = None, append_ident: Optional[str] = None) -> 'PackageMaker': offspring_ident = self._ident if ident is not None: offspring_ident = ident elif append_ident is not None: offspring_ident = (offspring_ident or '') + append_ident offspring = PackageMaker(self._logger, offspring_ident, self._itemno) offspring._package = deepcopy(self._package) return offspring def check_sanity(self, require_name: bool = True, require_version: bool = True, verbose: bool = False) -> bool: if require_name and not self._package.name: if verbose: self.log('package with no name', severity=Logger.ERROR) return False if require_version and not self._package.version: if verbose: self.log('package with no version', severity=Logger.ERROR) return False return True def __getattr__(self, key: str) -> Any: return getattr(self._package, key) def __enter__(self) -> 'PackageMaker': return self def __exit__(self, exc_type: Any, exc_value: Any, traceback: Any) -> Optional[bool]: if exc_type: self.log('parsing failed ({}): {}: {}'.format( 'skipped' if self._skipfailed else 'fatal', exc_type.__name__, exc_value ), severity=Logger.ERROR) if self._skipfailed: return True return None
def trace_action(package: Package, package_context: PackageContext, match_context: MatchContext) -> None: package.set_flag(PackageFlags.TRACE, trace_flag)
def __init__(self, logger: Logger, ident: Optional[str], itemno: int, skipfailed: bool = False) -> None: super(PackageMaker, self).__init__(logger) self._package = Package() self._ident = ident self._itemno = itemno self._skipfailed = skipfailed
def resetflavors_action(package: Package, package_context: PackageContext, match_context: MatchContext) -> None: package.flavors = []
def Parse(self, path): result = [] with open(path, encoding='utf-8', errors='ignore') as file: current_data = {} last_key = None for line in file: line = line.rstrip('\n') # empty line, dump package if line == '': if not current_data: continue # may happen on empty package list pkg = Package() def GetField(key, type_=str, default=None): if key in current_data: if type_ is None or isinstance( current_data[key], type_): return current_data[key] else: print( 'WARNING: unable to parse field {}'.format( key), file=sys.stderr) return default else: return default pkg.name = GetField('Package') pkg.version, pkg.origversion = SanitizeVersion( GetField('Version')) pkg.maintainers += GetMaintainers( GetField('Maintainer', default='')) pkg.maintainers += GetMaintainers( GetField('Uploaders', default='')) pkg.category = GetField('Section') pkg.homepage = GetField('Homepage') # This is long description #pkg.comment = GetField('Description', type_=None) #if isinstance(pkg.comment, list): # pkg.comment = ' '.join(pkg.comment) if pkg.name and pkg.version: result.append(pkg) else: print('WARNING: unable to parse package {}'.format( str(current_data)), file=sys.stderr) current_data = {} last_key = None continue # key - value pair match = re.fullmatch('([A-Za-z0-9-]+):(.*?)', line) if match: key = match.group(1) value = match.group(2).strip() current_data[key] = value last_key = key continue # continuation of previous key match = re.fullmatch(' (.*)', line) if match: value = match.group(1).strip() if not isinstance(current_data[last_key], list): current_data[last_key] = [current_data[last_key]] current_data[last_key].append(value) continue print('WARNING: unable to parse line: {}'.format(line), file=sys.stderr) return result
def setname_action(package: Package, package_context: PackageContext, match_context: MatchContext) -> None: package.effname = match_context.sub_name_dollars( setname, package.effname)
class PackageMaker(PackageMakerBase): def __init__(self, logger, ident, itemno, skipfailed=False): super(PackageMaker, self).__init__(logger) self.package = Package() self.ident = ident self.itemno = itemno self.skipfailed = skipfailed def _get_ident(self): return self.ident or self.package.extrafields.get( 'origin', None ) or self.package.name or self.package.basename or 'item #{}'.format( self.itemno) @PackageMakerBase._simple_setter('origin', str, nzs.strip, nzs.forbid_newlines) def set_origin(self, origin): # XXX: convert to dedicated field self.set_extra_field('origin', origin) @PackageMakerBase._simple_setter('name', str, nzs.strip, nzs.forbid_newlines) def set_name(self, name): self.package.name = name @PackageMakerBase._simple_setter('name', str, nzs.strip, nzs.forbid_newlines) def set_basename(self, basename): self.package.basename = basename def prefix_name(self, prefix): self.package.name = prefix + self.package.name @PackageMakerBase._simple_setter('version', str, nzs.strip, nzs.forbid_newlines) def set_version(self, version, version_normalizer=None): self.package.rawversion = version self.package.origversion = version if version_normalizer is None else version_normalizer( version) self.package.version = self.package.origversion @PackageMakerBase._simple_setter('version', str, nzs.strip, nzs.forbid_newlines) def set_rawversion(self, rawversion): if rawversion != self.package.version: self.package.rawversion = rawversion def set_name_and_version(self, namever, version_normalizer=None): name, version = namever.rsplit('-', 1) self.set_name(name) self.set_version(version, version_normalizer) @PackageMakerBase._simple_setter('summary', str, nzs.strip) def set_summary(self, summary): self.package.comment = summary @PackageMakerBase._omnivorous_setter('maintainer', str, nzs.strip, nzs.forbid_newlines, nzs.tolower) def add_maintainers(self, *args): self.package.maintainers.extend( _iter_unique(args, self.package.maintainers)) @PackageMakerBase._omnivorous_setter('category', str, nzs.strip, nzs.forbid_newlines) def add_categories(self, *args): # XXX: convert into array if not self.package.category: self.package.category = args[0] @PackageMakerBase._omnivorous_setter('homepage', str, nzs.strip, nzs.url, nzs.warn_whitespace, nzs.forbid_newlines) def add_homepages(self, *args): # XXX: convert into array if not self.package.homepage: self.package.homepage = args[0] @PackageMakerBase._omnivorous_setter('license', str, nzs.strip, nzs.forbid_newlines) def add_licenses(self, *args): self.package.licenses.extend(args) @PackageMakerBase._omnivorous_setter('download', str, nzs.strip, nzs.url, nzs.warn_whitespace, nzs.forbid_newlines) def add_downloads(self, *args): self.package.downloads.extend( _iter_unique(args, self.package.downloads)) def set_flags(self, mask, is_set=True): assert (isinstance(mask, int)) assert (isinstance(is_set, bool)) self.package.SetFlag(mask, is_set) def set_extra_field(self, key, value): assert (isinstance(key, str)) assert (isinstance(value, str)) self.package.extrafields[key] = value def unwrap(self): return self.package def clone(self, ident=None, append_ident=None): offspring_ident = self.ident if ident is not None: offspring_ident = ident elif append_ident is not None: offspring_ident += append_ident offspring = PackageMaker(self.logger, offspring_ident, self.itemno) offspring.package = deepcopy(self.package) return offspring def check_sanity(self, require_name=True, require_version=True, verbose=False): if require_name and not self.package.name: if verbose: self.log('package with no name', severity=Logger.ERROR) return False if require_version and not self.package.version: if verbose: self.log('package with no version', severity=Logger.ERROR) return False return True def __getattr__(self, key): return getattr(self.package, key) def __enter__(self): return self def __exit__(self, exc_type, exc_value, traceback): if exc_type: self.log('parsing failed ({}): {}: {}'.format( 'skipped' if self.skipfailed else 'fatal', exc_type.__name__, exc_value), severity=Logger.ERROR) if self.skipfailed: return True
def setver_action(package: Package, package_context: PackageContext, match_context: MatchContext) -> None: package.version = match_context.sub_ver_dollars( setver, package.version)
def compare(p1: Package, p2: Package) -> int: return p2.version_compare(p1)
def replaceinname_action(package: Package, package_context: PackageContext, match_context: MatchContext) -> None: for pattern, replacement in replace_items: package.effname = package.effname.replace( pattern, replacement)
def remove_action(package: Package, package_context: PackageContext, match_context: MatchContext) -> None: package.set_flag(PackageFlags.REMOVE, remove_flag)
def tolowername_action(package: Package, package_context: PackageContext, match_context: MatchContext) -> None: package.effname = package.effname.lower()
def weak_devel_action(package: Package, package_context: PackageContext, match_context: MatchContext) -> None: # XXX: currently sets ignore; change to set non-viral variant of devel (#654) package.set_flag(PackageFlags.IGNORE, weak_devel_flag)
def iter_parse(self, path, logger): with open(path, 'r', encoding='utf-8') as jsonfile: for key, packagedata in json.load(jsonfile)['packages'].items(): # see how Nix parses 'derivative' names in # https://github.com/NixOS src/libexpr/names.cc, DrvName::DrvName # it just splits on dash followed by non-letter # # this doesn't work well on 100% cases, it's an upstream problem match = re.match('(.+?)-([^a-zA-Z].*)$', packagedata['name']) if not match: print('cannot extract version: {}/{}'.format( key, packagedata['name']), file=sys.stderr) continue pkg = Package() pkg.name = match.group(1) pkg.version = match.group(2) # some exceptions for prefix in ('75dpi', '100dpi'): if pkg.version.startswith(prefix): pkg.name += '-' + prefix pkg.version = pkg.version[len(prefix) + 1:] merged = pkg.name + '-' + pkg.version for pkgname in [ 'liblqr-1', 'python2.7-3to2', 'python3.6-3to2' ]: if merged.startswith(pkgname): pkg.name = pkgname pkg.version = merged[len(pkgname) + 1:] keyparts = key.split('.') if len(keyparts) > 1: pkg.category = keyparts[0] if pkg.name.endswith('-git'): pkg.name = pkg.name[:-4] pkg.SetFlag(PackageFlags.ignore) if re.match('.*20[0-9]{2}-[0-9]{2}-[0-9]{2}', pkg.version): pkg.SetFlag(PackageFlags.ignore) if re.match('[0-9a-f]*[a-f][0-9a-f]*$', pkg.version) and len(pkg.version) >= 7: print( 'ignoring version which looks like commit hash: {}/{}'. format(key, packagedata['name']), file=sys.stderr) pkg.SetFlag(PackageFlags.ignore) meta = packagedata['meta'] if 'homepage' in meta: pkg.homepage = meta['homepage'] if isinstance( pkg.homepage, list ): # XXX: remove after adding support for homepages array pkg.homepage = pkg.homepage[0] if 'description' in meta and meta['description']: pkg.comment = meta['description'].replace('\n', ' ').strip() if 'maintainers' in meta: if not isinstance(meta['maintainers'], list): print('maintainers is not a list: {}/{}'.format( key, packagedata['name']), file=sys.stderr) else: pkg.maintainers += list( extract_nix_maintainers(meta['maintainers'])) if 'license' in meta: pkg.licenses = extract_nix_licenses(meta['license']) if 'position' in meta: posfile, posline = meta['position'].rsplit(':', 1) pkg.extrafields['posfile'] = posfile pkg.extrafields['posline'] = posline yield pkg
def devel_action(package: Package, package_context: PackageContext, match_context: MatchContext) -> None: package.set_flag(PackageFlags.DEVEL, devel_flag)
def legacy_action(package: Package, package_context: PackageContext, match_context: MatchContext) -> None: package.set_flag(PackageFlags.LEGACY, legacy_flag)