def Parse(self, path):
packages = []
for pkgdir in os.listdir(os.path.join(path, 'ports')):
controlpath = os.path.join(path, 'ports', pkgdir, 'CONTROL')
if not os.path.exists(controlpath):
continue
pkg = Package(name=pkgdir)
with open(controlpath, 'r', encoding='utf-8',
errors='ignore') as controlfile:
for line in controlfile:
line = line.strip()
if line.startswith('Version:'):
version = line[8:].strip()
match = re.match(
'[0-9]{4}[.-][0-9]{1,2}[.-][0-9]{1,2}', version)
if match:
pkg.version = version
pkg.SetFlag(PackageFlags.ignore)
else:
pkg.version, pkg.origversion = SanitizeVersion(
line[8:].strip())
elif line.startswith('Description:'):
comment = line[12:].strip()
if comment:
pkg.comment = comment
# pretty much a hack to shut a bunch of fake versions up
portfilepath = os.path.join(path, 'ports', pkgdir,
'portfile.cmake')
if os.path.exists(portfilepath):
with open(portfilepath, 'r', encoding='utf-8',
errors='ignore') as portfile:
for line in portfile:
if 'libimobiledevice-win32' in line:
print(
'WARNING: marking version for {} as untrusted, https://github.com/libimobiledevice-win32 accused of version faking'
.format(pkg.name),
file=sys.stderr)
pkg.SetFlag(PackageFlags.untrusted)
break
if not pkg.version:
print('WARNING: unable to parse port {}: no version'.format(
pkgdir),
file=sys.stderr)
continue
packages.append(pkg)
return packages
def Parse(self, path):
result = []
with open(path, 'r', encoding='utf-8') as jsonfile:
for package in json.load(jsonfile)['packages']:
pkg = Package()
pkg.name = package['name']
if package['version'] is None:
print('no version: {}'.format(pkg.name), file=sys.stderr)
continue
pkg.version, _ = SanitizeVersion(package['version'])
pkg.origversion = package['full_version']
pkg.category = package['pkg_section'] or package['section']
pkg.comment = package['description']
pkg.maintainers = GetMaintainers(package['committer'])
if pkg.version == '999':
pkg.SetFlag(PackageFlags.ignore) # XXX: rolling? revisit
result.append(pkg)
return result
def Parse(self, path):
result = []
jsondata = None
with open(path, 'r', encoding='utf-8') as jsonfile:
jsondata = json.load(jsonfile)
for packagedata in SimplifyResult(jsondata):
entity = packagedata['project'].rsplit(
'/', 1)[-1] # this is URL, take only the ID from it
# use Arch and AUR package names as a name, as they are most non-ambigous
names = []
for field in ['arch_packages', 'aur_packages']:
if packagedata[field]:
names = packagedata[field].split(', ')
break
# generate a package for each package name; these will be merged anyway
for name in set(names):
# generate a package for each version
for version in packagedata['versions'].split(', '):
version, *flags = version.split('|')
is_devel = 'U' in flags
is_foreign_os_release = 'O' in flags and 'L' not in flags
if is_foreign_os_release:
print(
'WARNING: {} ({}) version {} skipped as non-linux release'
.format(packagedata['projectLabel'], entity,
version),
file=sys.stderr)
continue
pkg = Package()
pkg.SetFlag(PackageFlags.devel, is_devel)
pkg.name = entity
pkg.effname = name
pkg.version = version
if 'projectDescription' in packagedata:
pkg.comment = packagedata['projectDescription']
else:
pkg.comment = packagedata['projectLabel']
if packagedata['licenses']:
pkg.licenses = packagedata['licenses'].split(', ')
if packagedata['websites']:
pkg.homepage = packagedata['websites'].split(', ')[
0] # XXX: use all websites when supported
result.append(pkg)
return result
def Parse(self, path):
result = []
maintainers = []
with open(os.path.join(path, 'metadata/about.conf'),
'r',
encoding='utf-8') as metadata:
for line in metadata:
if '=' in line:
key, value = map(lambda s: s.strip(), line.split('=', 1))
if key == 'owner':
maintainers = extract_maintainers(value)
packages_path = os.path.join(path, 'packages')
for category in os.listdir(packages_path):
category_path = os.path.join(packages_path, category)
if not os.path.isdir(category_path):
continue
if category == 'virtual' or category == 'metadata':
continue
for package in os.listdir(category_path):
package_path = os.path.join(category_path, package)
if not os.path.isdir(package_path):
continue
for exheres in os.listdir(package_path):
if not exheres.startswith(package +
'-') and not exheres.endswith(
'.exheres-0'):
continue
pkg = Package()
pkg.category = category
pkg.name = package
pkg.version, pkg.origversion = SanitizeVersion(
exheres[len(package) + 1:-10])
pkg.maintainers = maintainers
if pkg.version == 'scm' or pkg.version.endswith('-scm'):
pkg.SetFlag(PackageFlags.rolling)
result.append(pkg)
return result
def Parse(self, path):
result = []
skipped_archs = {}
for entry in self.ParsePackagesEntriesFromXml(path):
pkg = Package()
arch = entry.find('{http://linux.duke.edu/metadata/common}arch').text
if self.allowed_archs and arch not in self.allowed_archs:
skipped_archs[arch] = skipped_archs.get(arch, 0) + 1
continue
pkg.name = entry.find('{http://linux.duke.edu/metadata/common}name').text
version = entry.find('{http://linux.duke.edu/metadata/common}version').attrib['ver']
release = entry.find('{http://linux.duke.edu/metadata/common}version').attrib['rel']
match = re.match('0\\.[0-9]+\\.((?:alpha|beta|rc)[0-9]+)\\.', release)
if match:
# known pre-release schema: https://fedoraproject.org/wiki/Packaging:Versioning#Prerelease_versions
version += '-' + match.group(1)
elif release < '1':
# unknown pre-release schema: https://fedoraproject.org/wiki/Packaging:Versioning#Some_definitions
# most likely a snapshot
pkg.SetFlag(PackageFlags.ignore)
pkg.version, pkg.origversion = SanitizeVersion(version)
# XXX: append origversion with release
pkg.comment = entry.find('{http://linux.duke.edu/metadata/common}summary').text
pkg.homepage = entry.find('{http://linux.duke.edu/metadata/common}url').text
pkg.category = entry.find('{http://linux.duke.edu/metadata/common}format/'
'{http://linux.duke.edu/metadata/rpm}group').text
pkg.licenses.append(entry.find('{http://linux.duke.edu/metadata/common}format/'
'{http://linux.duke.edu/metadata/rpm}license').text)
packager = entry.find('{http://linux.duke.edu/metadata/common}packager').text
if packager:
pkg.maintainers = extract_maintainers(packager)
result.append(pkg)
for arch, numpackages in sorted(skipped_archs.items()):
print('WARNING: skipping {} packages(s) with disallowed architecture {}'.format(numpackages, arch), file=sys.stderr)
return result
def Parse(self, path):
result = []
root = xml.etree.ElementTree.parse(path)
for info in root.findall('./info'):
pkg = Package()
# derive names and versions from fn field
fn = info.attrib['fn'].rsplit('-', 2)
if len(fn) < 3:
print('WARNING: unable to parse fn: {}'.format(fn),
file=sys.stderr)
continue
pkg.name = fn[0]
pkg.origversion = '-'.join(fn[1:])
pkg.version = fn[1]
# Rosa packages are named like PKGNAME-PKGVER-ROSAREV
# where ROSAREV is most commonly in the form of N.src, but
# may contain other components, such as prerelease stuff
# like alpha/beta/rc/pre/... and snapshot revisions/dates
#
# What we do here is we try to extract prerelease part
# and mark version as ignored with non-trivial ROSAREV,
# as it it likely a snapshot and trus cannot be trusted
if not re.fullmatch('[0-9]+\\.src', fn[2]):
pkg.SetFlag(PackageFlags.ignore)
match = re.search('\\b(a|alpha|b|beta|pre|rc)[0-9]+',
fn[2].lower())
if match:
pkg.version += match.group(0)
# process url and license
url = info.attrib['url']
if url:
pkg.homepage = url
license_ = info.attrib['license']
pkg.licenses = [license_]
result.append(pkg)
return result
def Parse(self, path):
result = []
root = xml.etree.ElementTree.parse(path)
repository = root.find(
'{http://www.openpkg.org/xml-rdf-index/0.9}Repository')
for item in repository.findall(
'{http://www.w3.org/1999/02/22-rdf-syntax-ns#}Description'):
pkg = Package()
pkg.name = item.find(
'{http://www.openpkg.org/xml-rdf-index/0.9}Name').text
pkg.version = item.find(
'{http://www.openpkg.org/xml-rdf-index/0.9}Version').text
pkg.licenses = [
item.find(
'{http://www.openpkg.org/xml-rdf-index/0.9}License').text
]
pkg.comment = item.find(
'{http://www.openpkg.org/xml-rdf-index/0.9}Summary').text
pkg.category = item.find(
'{http://www.openpkg.org/xml-rdf-index/0.9}Group').text
pkg.homepage = item.find(
'{http://www.openpkg.org/xml-rdf-index/0.9}URL').text
for source in item.findall(
'./{http://www.openpkg.org/xml-rdf-index/0.9}Source/{http://www.w3.org/1999/02/22-rdf-syntax-ns#}bag/{http://www.w3.org/1999/02/22-rdf-syntax-ns#}li'
):
text = source.text
if (text.startswith('https://') or text.startswith('http://')
or text.startswith('ftp://')
) and 'openpkg.org' not in text:
pkg.downloads.append(text)
release = item.find(
'{http://www.openpkg.org/xml-rdf-index/0.9}Release').text
if pkg.version.endswith(release):
pkg.SetFlag(PackageFlags.untrusted)
result.append(pkg)
return result
def Parse(self, path):
packages = []
for pkgdir in os.listdir(os.path.join(path, 'ports')):
controlpath = os.path.join(path, 'ports', pkgdir, 'CONTROL')
if not os.path.exists(controlpath):
continue
pkg = Package(name=pkgdir)
with open(controlpath, 'r', encoding='utf-8',
errors='ignore') as controlfile:
for line in controlfile:
line = line.strip()
if line.startswith('Version:'):
version = line[8:].strip()
match = re.match(
'[0-9]{4}[.-][0-9]{1,2}[.-][0-9]{1,2}', version)
if match:
pkg.version = version
pkg.SetFlag(PackageFlags.ignore)
else:
pkg.version, pkg.origversion = SanitizeVersion(
line[8:].strip())
elif line.startswith('Description:'):
comment = line[12:].strip()
if comment:
pkg.comment = comment
if not pkg.version:
print('WARNING: unable to parse port {}: no version'.format(
pkgdir),
file=sys.stderr)
continue
packages.append(pkg)
return packages
def Parse(self, path):
result = []
for category in os.listdir(path):
category_path = os.path.join(path, category)
if not os.path.isdir(category_path):
continue
if category == 'virtual' or category == 'metadata':
continue
for package in os.listdir(category_path):
package_path = os.path.join(category_path, package)
if not os.path.isdir(package_path):
continue
metadata_path = os.path.join(package_path, 'metadata.xml')
# parse maintainers from metadata.xml
# these are the same for all ebuilds for current package
maintainers = []
if os.path.isfile(metadata_path):
with open(metadata_path, 'r',
encoding='utf-8') as metafile:
meta = xml.etree.ElementTree.parse(metafile)
for entry in meta.findall('maintainer'):
email_node = entry.find('email')
if email_node is not None and email_node.text is not None:
maintainers += extract_maintainers(
email_node.text)
for ebuild in os.listdir(package_path):
if not ebuild.endswith('.ebuild'):
continue
pkg = Package()
pkg.name = package
pkg.category = category
pkg.maintainers = maintainers
pkg.version, pkg.origversion = SanitizeVersion(
ebuild[len(package) + 1:-7])
if pkg.version.endswith('9999'):
# ignore versions for snapshots
pkg.SetFlag(PackageFlags.rolling)
metadata_path = os.path.join(
path, 'metadata', 'md5-cache', category,
package + '-' +
(pkg.origversion if pkg.origversion else pkg.version))
if os.path.isfile(metadata_path):
with open(metadata_path, 'r',
encoding='utf-8') as metadata_file:
for line in metadata_file:
line = line.strip()
key, value = line.split('=', 1)
if key == 'DESCRIPTION':
pkg.comment = value
elif key == 'HOMEPAGE':
pkg.homepage = value.split(' ')[
0] # XXX: save all urls
elif key == 'LICENSE':
if '(' in value:
# XXX: conditionals and OR's: need more
# complex parsing and backend support
pkg.licenses.append(value)
else:
pkg.licenses += value.split(' ')
elif key == 'SRC_URI':
pkg.downloads += ParseConditionalExpr(
value)
result.append(pkg)
return result
def Parse(self, path):
result = []
with open(path, 'r', encoding='utf-8') as jsonfile:
for key, packagedata in sorted(
json.load(jsonfile)['packages'].items()):
# see how Nix parses 'derivative' names in
# https://github.com/NixOS src/libexpr/names.cc, DrvName::DrvName
# it just splits on dash followed by non-letter
#
# this doesn't work well on 100% cases, it's an upstream problem
match = re.match('(.+?)-([^a-zA-Z].*)$', packagedata['name'])
if not match:
print('cannot extract version: {}/{}'.format(
key, packagedata['name']),
file=sys.stderr)
continue
pkg = Package()
pkg.name = match.group(1)
pkg.version = match.group(2)
# some exceptions
for prefix in ('75dpi', '100dpi'):
if pkg.version.startswith(prefix):
pkg.name += '-' + prefix
pkg.version = pkg.version[len(prefix) + 1:]
for pkgname in ('liblqr', ):
if pkg.name == pkgname:
dashpos = pkg.version.find('-')
pkg.name = pkg.name + '-' + pkg.version[0:dashpos]
pkg.version = pkg.version[dashpos + 1:]
if pkg.name.endswith('-git'):
pkg.name = pkg.name[:-4]
pkg.SetFlag(PackageFlags.ignore)
if re.match('.*20[0-9]{2}-[0-9]{2}-[0-9]{2}', pkg.version):
pkg.SetFlag(PackageFlags.ignore)
if re.match('[0-9a-f]*[a-f][0-9a-f]*$',
pkg.version) and len(pkg.version) >= 7:
print(
'ignoring version which looks like commit hash: {}/{}'.
format(key, packagedata['name']),
file=sys.stderr)
pkg.SetFlag(PackageFlags.ignore)
meta = packagedata['meta']
if 'homepage' in meta:
pkg.homepage = meta['homepage']
if isinstance(
pkg.homepage, list
): # XXX: remove after adding support for homepages array
pkg.homepage = pkg.homepage[0]
if 'description' in meta:
pkg.comment = meta['description']
if 'maintainers' in meta:
maintainers = meta['maintainers']
if not isinstance(meta['maintainers'], list):
print('maintainers is not a list: {}/{}'.format(
key, packagedata['name']),
file=sys.stderr)
else:
maintainers = ', '.join(maintainers)
pkg.maintainers = GetMaintainers(maintainers)
if 'license' in meta:
pkg.licenses = ExtractLicenses(meta['license'])
result.append(pkg)
return result
def ignore_action(package: Package,
package_context: PackageContext,
match_context: MatchContext) -> None:
package.SetFlag(PackageFlags.ignore, ignore_flag)
def generated_action(package: Package,
package_context: PackageContext,
match_context: MatchContext) -> None:
# XXX: the same as rolling for now
package.SetFlag(PackageFlags.rolling, generated_flag)
def any_is_patch_action(package: Package,
package_context: PackageContext,
match_context: MatchContext) -> None:
package.SetFlag(PackageFlags.any_is_patch, any_is_patch_flag)
class PackageMaker(PackageMakerBase):
def __init__(self, logger, ident):
super(PackageMaker, self).__init__(logger)
self.package = Package()
self.ident = ident
def _get_ident(self):
return self.package.extrafields.get('origin', None) or self.package.name or self.ident or self.package.effname
@PackageMakerBase._simple_setter('origin', str, nzs.strip, nzs.forbid_newlines)
def set_origin(self, origin):
# XXX: convert to dedicated field
self.set_extra_field('origin', origin)
@PackageMakerBase._simple_setter('name', str, nzs.strip, nzs.forbid_newlines)
def set_effname(self, effname):
# XXX: this should be refactored
# It's OK for parsers to set both effname and name, the example case
# is setting effname to <project> name and <name>s to package names
# However this erases strict boundary between fields parsed from
# data and fields calculated based on them. It should be fixed.
self.package.effname = effname
@PackageMakerBase._simple_setter('name', str, nzs.strip, nzs.forbid_newlines)
def set_name(self, name):
self.package.name = name
@PackageMakerBase._simple_setter('version', str, nzs.strip, nzs.forbid_newlines)
def set_version(self, version, version_normalizer=None):
if version_normalizer is None:
self.package.version = version
else:
normalized_version = version_normalizer(version)
# XXX: compatibility shim to allow old style SanitizeVesion returning version and origversion
if isinstance(normalized_version, tuple):
normalized_version = normalized_version[0]
if normalized_version == version:
self.package.version = version
else:
self.package.version = normalized_version
self.package.origversion = version
@PackageMakerBase._simple_setter('version', str, nzs.strip, nzs.forbid_newlines)
def set_origversion(self, origversion):
if origversion != self.package.version:
self.package.origversion = origversion
def set_name_and_version(self, namever, version_normalizer=None):
name, version = namever.rsplit('-', 1)
self.set_name(name)
self.set_version(version, version_normalizer)
@PackageMakerBase._simple_setter('summary', str, nzs.strip)
def set_summary(self, summary):
self.package.comment = summary
@PackageMakerBase._omnivorous_setter('maintainer', str, nzs.strip, nzs.forbid_newlines)
def add_maintainers(self, *args):
self.package.maintainers.extend(args)
@PackageMakerBase._omnivorous_setter('category', str, nzs.strip, nzs.forbid_newlines)
def add_categories(self, *args):
# XXX: convert into array
if not self.package.category:
self.package.category = args[0]
@PackageMakerBase._omnivorous_setter('homepage', str, nzs.strip, nzs.require_url, nzs.warn_whitespace, nzs.forbid_newlines)
def add_homepages(self, *args):
# XXX: convert into array
if not self.package.homepage:
self.package.homepage = args[0]
@PackageMakerBase._omnivorous_setter('license', str, nzs.strip, nzs.forbid_newlines)
def add_licenses(self, *args):
self.package.licenses.extend(args)
@PackageMakerBase._omnivorous_setter('download', str, nzs.strip, nzs.require_url, nzs.warn_whitespace, nzs.forbid_newlines)
def add_downloads(self, *args):
self.package.downloads.extend(args)
def set_flags(self, mask, is_set=True):
assert(isinstance(mask, int))
assert(isinstance(is_set, bool))
self.package.SetFlag(mask, is_set)
def set_extra_field(self, key, value):
assert(isinstance(key, str))
assert(isinstance(value, str))
self.package.extrafields[key] = value
def unwrap(self):
return self.package
def clone(self):
offspring = PackageMaker(self.logger, self.ident)
offspring.package = deepcopy(self.package)
return offspring
def check_sanity(self, verbose=False):
if not self.package.name:
if verbose:
self.log('package with no name', severity=Logger.ERROR)
return False
if not self.package.version:
if verbose:
self.log('package with no version', severity=Logger.ERROR)
return False
return True
def __getattr__(self, key):
return getattr(self.package, key)
# XXX: compatibility shim
def __setattr__(self, key, value):
if key in ['package', 'logger', 'ident']:
return super(PackageMaker, self).__setattr__(key, value)
return setattr(self.package, key, value)
def weak_devel_action(package: Package,
package_context: PackageContext,
match_context: MatchContext) -> None:
# XXX: currently sets ignore; change to set non-viral variant of devel (#654)
package.SetFlag(PackageFlags.ignore, weak_devel_flag)
def noscheme_action(package: Package,
package_context: PackageContext,
match_context: MatchContext) -> None:
package.SetFlag(PackageFlags.noscheme, noscheme_flag)
def untrusted_action(package: Package,
package_context: PackageContext,
match_context: MatchContext) -> None:
package.SetFlag(PackageFlags.untrusted, untrusted_flag)
def incorrect_action(package: Package,
package_context: PackageContext,
match_context: MatchContext) -> None:
package.SetFlag(PackageFlags.incorrect, incorrect_flag)
def legacy_action(package: Package,
package_context: PackageContext,
match_context: MatchContext) -> None:
package.SetFlag(PackageFlags.legacy, legacy_flag)
def outdated_action(package: Package,
package_context: PackageContext,
match_context: MatchContext) -> None:
package.SetFlag(PackageFlags.outdated, outdated_flag)
def remove_action(package: Package,
package_context: PackageContext,
match_context: MatchContext) -> None:
package.SetFlag(PackageFlags.remove, remove_flag)
class PackageMaker(PackageMakerBase):
def __init__(self, logger, ident, itemno, skipfailed=False):
super(PackageMaker, self).__init__(logger)
self.package = Package()
self.ident = ident
self.itemno = itemno
self.skipfailed = skipfailed
def _get_ident(self):
return self.ident or self.package.extrafields.get(
'origin', None
) or self.package.name or self.package.basename or 'item #{}'.format(
self.itemno)
@PackageMakerBase._simple_setter('origin', str, nzs.strip,
nzs.forbid_newlines)
def set_origin(self, origin):
# XXX: convert to dedicated field
self.set_extra_field('origin', origin)
@PackageMakerBase._simple_setter('name', str, nzs.strip,
nzs.forbid_newlines)
def set_name(self, name):
self.package.name = name
@PackageMakerBase._simple_setter('name', str, nzs.strip,
nzs.forbid_newlines)
def set_basename(self, basename):
self.package.basename = basename
def prefix_name(self, prefix):
self.package.name = prefix + self.package.name
@PackageMakerBase._simple_setter('version', str, nzs.strip,
nzs.forbid_newlines)
def set_version(self, version, version_normalizer=None):
self.package.rawversion = version
self.package.origversion = version if version_normalizer is None else version_normalizer(
version)
self.package.version = self.package.origversion
@PackageMakerBase._simple_setter('version', str, nzs.strip,
nzs.forbid_newlines)
def set_rawversion(self, rawversion):
if rawversion != self.package.version:
self.package.rawversion = rawversion
def set_name_and_version(self, namever, version_normalizer=None):
name, version = namever.rsplit('-', 1)
self.set_name(name)
self.set_version(version, version_normalizer)
@PackageMakerBase._simple_setter('summary', str, nzs.strip)
def set_summary(self, summary):
self.package.comment = summary
@PackageMakerBase._omnivorous_setter('maintainer', str, nzs.strip,
nzs.forbid_newlines, nzs.tolower)
def add_maintainers(self, *args):
self.package.maintainers.extend(
_iter_unique(args, self.package.maintainers))
@PackageMakerBase._omnivorous_setter('category', str, nzs.strip,
nzs.forbid_newlines)
def add_categories(self, *args):
# XXX: convert into array
if not self.package.category:
self.package.category = args[0]
@PackageMakerBase._omnivorous_setter('homepage', str, nzs.strip, nzs.url,
nzs.warn_whitespace,
nzs.forbid_newlines)
def add_homepages(self, *args):
# XXX: convert into array
if not self.package.homepage:
self.package.homepage = args[0]
@PackageMakerBase._omnivorous_setter('license', str, nzs.strip,
nzs.forbid_newlines)
def add_licenses(self, *args):
self.package.licenses.extend(args)
@PackageMakerBase._omnivorous_setter('download', str, nzs.strip, nzs.url,
nzs.warn_whitespace,
nzs.forbid_newlines)
def add_downloads(self, *args):
self.package.downloads.extend(
_iter_unique(args, self.package.downloads))
def set_flags(self, mask, is_set=True):
assert (isinstance(mask, int))
assert (isinstance(is_set, bool))
self.package.SetFlag(mask, is_set)
def set_extra_field(self, key, value):
assert (isinstance(key, str))
assert (isinstance(value, str))
self.package.extrafields[key] = value
def unwrap(self):
return self.package
def clone(self, ident=None, append_ident=None):
offspring_ident = self.ident
if ident is not None:
offspring_ident = ident
elif append_ident is not None:
offspring_ident += append_ident
offspring = PackageMaker(self.logger, offspring_ident, self.itemno)
offspring.package = deepcopy(self.package)
return offspring
def check_sanity(self,
require_name=True,
require_version=True,
verbose=False):
if require_name and not self.package.name:
if verbose:
self.log('package with no name', severity=Logger.ERROR)
return False
if require_version and not self.package.version:
if verbose:
self.log('package with no version', severity=Logger.ERROR)
return False
return True
def __getattr__(self, key):
return getattr(self.package, key)
def __enter__(self):
return self
def __exit__(self, exc_type, exc_value, traceback):
if exc_type:
self.log('parsing failed ({}): {}: {}'.format(
'skipped' if self.skipfailed else 'fatal', exc_type.__name__,
exc_value),
severity=Logger.ERROR)
if self.skipfailed:
return True
def successor_action(package: Package,
package_context: PackageContext,
match_context: MatchContext) -> None:
# XXX: the same as devel for now
package.SetFlag(PackageFlags.devel, successor_flag)
def iter_parse(self, path):
with open(path, 'r', encoding='utf-8') as jsonfile:
for key, packagedata in json.load(jsonfile)['packages'].items():
# see how Nix parses 'derivative' names in
# https://github.com/NixOS src/libexpr/names.cc, DrvName::DrvName
# it just splits on dash followed by non-letter
#
# this doesn't work well on 100% cases, it's an upstream problem
match = re.match('(.+?)-([^a-zA-Z].*)$', packagedata['name'])
if not match:
print('cannot extract version: {}/{}'.format(
key, packagedata['name']),
file=sys.stderr)
continue
pkg = Package()
pkg.name = match.group(1)
pkg.version = match.group(2)
# some exceptions
for prefix in ('75dpi', '100dpi'):
if pkg.version.startswith(prefix):
pkg.name += '-' + prefix
pkg.version = pkg.version[len(prefix) + 1:]
merged = pkg.name + '-' + pkg.version
for pkgname in [
'liblqr-1', 'python2.7-3to2', 'python3.6-3to2'
]:
if merged.startswith(pkgname):
pkg.name = pkgname
pkg.version = merged[len(pkgname) + 1:]
keyparts = key.split('.')
if len(keyparts) > 1:
pkg.category = keyparts[0]
if pkg.name.endswith('-git'):
pkg.name = pkg.name[:-4]
pkg.SetFlag(PackageFlags.ignore)
if re.match('.*20[0-9]{2}-[0-9]{2}-[0-9]{2}', pkg.version):
pkg.SetFlag(PackageFlags.ignore)
if re.match('[0-9a-f]*[a-f][0-9a-f]*$',
pkg.version) and len(pkg.version) >= 7:
print(
'ignoring version which looks like commit hash: {}/{}'.
format(key, packagedata['name']),
file=sys.stderr)
pkg.SetFlag(PackageFlags.ignore)
meta = packagedata['meta']
if 'homepage' in meta:
pkg.homepage = meta['homepage']
if isinstance(
pkg.homepage, list
): # XXX: remove after adding support for homepages array
pkg.homepage = pkg.homepage[0]
if 'description' in meta and meta['description']:
pkg.comment = meta['description'].replace('\n',
' ').strip()
if 'maintainers' in meta:
if not isinstance(meta['maintainers'], list):
print('maintainers is not a list: {}/{}'.format(
key, packagedata['name']),
file=sys.stderr)
else:
pkg.maintainers += list(
extract_nix_maintainers(meta['maintainers']))
if 'license' in meta:
pkg.licenses = extract_nix_licenses(meta['license'])
if 'position' in meta:
posfile, posline = meta['position'].rsplit(':', 1)
pkg.extrafields['posfile'] = posfile
pkg.extrafields['posline'] = posline
yield pkg
def debianism_action(package: Package,
package_context: PackageContext,
match_context: MatchContext) -> None:
# XXX: the same as devel for now
package.SetFlag(PackageFlags.devel, debianism_flag)
def rolling_action(package: Package,
package_context: PackageContext,
match_context: MatchContext) -> None:
package.SetFlag(PackageFlags.rolling, rolling_flag)
def devel_action(package: Package, package_context: PackageContext,
match_context: MatchContext) -> None:
package.SetFlag(PackageFlags.devel, devel_flag)
def snapshot_action(package: Package,
package_context: PackageContext,
match_context: MatchContext) -> None:
# XXX: the same as ignored for now
package.SetFlag(PackageFlags.ignore, snapshot_flag)
class PackageMaker(PackageMakerBase):
_package: Package
_ident: Optional[str]
_itemno: int
_skipfailed: bool
def __init__(self, logger: Logger, ident: Optional[str], itemno: int, skipfailed: bool = False) -> None:
super(PackageMaker, self).__init__(logger)
self._package = Package()
self._ident = ident
self._itemno = itemno
self._skipfailed = skipfailed
def _get_ident(self) -> str:
return self._ident or self._package.extrafields.get('origin', None) or self._package.name or self._package.basename or 'item #{}'.format(self._itemno)
@PackageMakerBase._simple_setter('origin', str, nzs.strip, nzs.forbid_newlines)
def set_origin(self, origin: str) -> None:
# XXX: convert to dedicated field
self.set_extra_field('origin', origin)
@PackageMakerBase._simple_setter('name', str, nzs.strip, nzs.forbid_newlines)
def set_name(self, name: str) -> None:
self._package.name = name
@PackageMakerBase._simple_setter('basename', str, nzs.strip, nzs.forbid_newlines)
def set_basename(self, basename: str) -> None:
self._package.basename = basename
def prefix_name(self, prefix: str) -> None:
self._package.name = prefix + self._package.name
@PackageMakerBase._simple_setter('version', str, nzs.strip, nzs.forbid_newlines)
def set_version(self, version: str, version_normalizer: Optional[Callable[[str], str]] = None) -> None:
self._package.rawversion = version
self._package.origversion = version if version_normalizer is None else version_normalizer(version)
self._package.version = self._package.origversion
@PackageMakerBase._simple_setter('version', str, nzs.strip, nzs.forbid_newlines)
def set_rawversion(self, rawversion: str) -> None:
if rawversion != self._package.version:
self._package.rawversion = rawversion
def set_name_and_version(self, namever: str, version_normalizer: Optional[Callable[[str], str]] = None) -> None:
name, version = namever.rsplit('-', 1)
self.set_name(name)
self.set_version(version, version_normalizer)
@PackageMakerBase._simple_setter('arch', str, nzs.strip, nzs.forbid_newlines)
def set_arch(self, arch: str) -> None:
self._package.arch = arch
@PackageMakerBase._simple_setter('summary', str, nzs.strip)
def set_summary(self, summary: str) -> None:
self._package.comment = summary
@PackageMakerBase._omnivorous_setter('maintainer', str, nzs.strip, nzs.forbid_newlines, nzs.tolower)
def add_maintainers(self, *args: Any) -> None:
self._package.maintainers.extend(_iter_unique(args, self._package.maintainers))
@PackageMakerBase._omnivorous_setter('category', str, nzs.strip, nzs.forbid_newlines)
def add_categories(self, *args: Any) -> None:
# XXX: convert into array
if not self._package.category:
self._package.category = args[0]
@PackageMakerBase._omnivorous_setter('homepage', str, nzs.strip, nzs.url, nzs.warn_whitespace, nzs.forbid_newlines)
def add_homepages(self, *args: Any) -> None:
# XXX: convert into array
if not self._package.homepage:
self._package.homepage = args[0]
@PackageMakerBase._omnivorous_setter('license', str, nzs.strip, nzs.forbid_newlines)
def add_licenses(self, *args: Any) -> None:
self._package.licenses.extend(args)
@PackageMakerBase._omnivorous_setter('download', str, nzs.strip, nzs.url, nzs.warn_whitespace, nzs.forbid_newlines)
def add_downloads(self, *args: Any) -> None:
self._package.downloads.extend(_iter_unique(args, self._package.downloads))
def set_flags(self, mask: int, is_set: bool = True) -> None:
self._package.SetFlag(mask, is_set)
def set_extra_field(self, key: str, value: str) -> None:
self._package.extrafields[key] = value
def unwrap(self) -> Package:
return self._package
def clone(self, ident: Optional[str] = None, append_ident: Optional[str] = None) -> 'PackageMaker':
offspring_ident = self._ident
if ident is not None:
offspring_ident = ident
elif append_ident is not None:
offspring_ident = (offspring_ident or '') + append_ident
offspring = PackageMaker(self._logger, offspring_ident, self._itemno)
offspring._package = deepcopy(self._package)
return offspring
def check_sanity(self, require_name: bool = True, require_version: bool = True, verbose: bool = False) -> bool:
if require_name and not self._package.name:
if verbose:
self.log('package with no name', severity=Logger.ERROR)
return False
if require_version and not self._package.version:
if verbose:
self.log('package with no version', severity=Logger.ERROR)
return False
return True
def __getattr__(self, key: str) -> Any:
return getattr(self._package, key)
def __enter__(self) -> 'PackageMaker':
return self
def __exit__(self, exc_type: Any, exc_value: Any, traceback: Any) -> Optional[bool]:
if exc_type:
self.log('parsing failed ({}): {}: {}'.format(
'skipped' if self._skipfailed else 'fatal',
exc_type.__name__,
exc_value
), severity=Logger.ERROR)
if self._skipfailed:
return True
return None
