def test_validate_urls(self):
logger = AccumulatingLogger()
factory = PackageFactory(logger)
maker = factory.begin()
maker.add_downloads('http://www.valid/', 'https://www.valid/some',
'ftp://ftp.valid/', 'invalid')
pkg = maker.unwrap()
self.assertEqual(pkg.downloads, [
'http://www.valid/', 'https://www.valid/some', 'ftp://ftp.valid/'
])
self.assertEqual(len(logger.get()), 1)
self.assertTrue('invalid' in logger.get()[0])
def iter_parse(self, path: str, factory: PackageFactory) -> Iterable[PackageMaker]:
normalize_version = VersionStripper().strip_right(',').strip_right('_')
for pkgname, pkgdata in iter_json_dict(path, ('packages', None)):
with factory.begin() as pkg:
pkg.add_name(pkgname, NameType.BSD_PKGNAME)
pkg.add_name(pkgdata['port'], NameType.BSD_ORIGIN)
pkg.set_version(pkgdata['version'], normalize_version)
pkg.set_summary(pkgdata['summary'])
pkg.add_categories(item['category'] for item in pkgdata['categories'])
pkg.add_licenses(pkgdata['licenses'])
pkg.add_homepages(pkgdata['homepages'])
yield pkg
def iter_parse(self, path: str, factory: PackageFactory, transformer: PackageTransformer) -> Generator[PackageMaker, None, None]:
normalize_version = VersionStripper().strip_right_greedy('+')
skipped_archs: Dict[str, int] = {}
for entry in _iter_package_entries(path):
with factory.begin() as pkg:
arch = entry.findtext('{http://linux.duke.edu/metadata/common}arch')
if self.allowed_archs and arch not in self.allowed_archs:
skipped_archs[arch] = skipped_archs.get(arch, 0) + 1
continue
pkg.set_name(entry.findtext('{http://linux.duke.edu/metadata/common}name'))
version_elt = entry.find('{http://linux.duke.edu/metadata/common}version')
if version_elt is None:
raise RuntimeError('Cannot find <version> element')
epoch = version_elt.attrib['epoch']
version = version_elt.attrib['ver']
release = version_elt.attrib['rel']
match = re.match('0\\.[0-9]+\\.((?:alpha|beta|rc)[0-9]+)\\.', release)
if match:
# known pre-release schema: https://fedoraproject.org/wiki/Packaging:Versioning#Prerelease_versions
version += '-' + match.group(1)
elif release < '1':
# unknown pre-release schema: https://fedoraproject.org/wiki/Packaging:Versioning#Some_definitions
# most likely a snapshot
pkg.set_flags(PackageFlags.ignore)
pkg.set_version(version, normalize_version)
pkg.set_rawversion(nevra_construct(None, epoch, version, release))
pkg.set_summary(entry.findtext('{http://linux.duke.edu/metadata/common}summary'))
pkg.add_homepages(entry.findtext('{http://linux.duke.edu/metadata/common}url'))
pkg.add_categories(entry.findtext('{http://linux.duke.edu/metadata/common}format/'
'{http://linux.duke.edu/metadata/rpm}group'))
pkg.add_licenses(entry.findtext('{http://linux.duke.edu/metadata/common}format/'
'{http://linux.duke.edu/metadata/rpm}license'))
pkg.set_arch(entry.findtext('{http://linux.duke.edu/metadata/common}arch'))
packager = entry.findtext('{http://linux.duke.edu/metadata/common}packager')
if packager:
pkg.add_maintainers(extract_maintainers(packager))
yield pkg
for arch, numpackages in sorted(skipped_archs.items()):
factory.log('skipped {} packages(s) with disallowed architecture {}'.format(numpackages, arch))
def iter_parse(self, path: str, factory: PackageFactory,
transformer: PackageTransformer) -> Iterable[PackageMaker]:
root = xml.etree.ElementTree.parse(path)
repository = root.find(
'{http://www.openpkg.org/xml-rdf-index/0.9}Repository')
assert (repository is not None)
for item in repository.findall(
'{http://www.w3.org/1999/02/22-rdf-syntax-ns#}Description'):
with factory.begin() as pkg:
pkg.add_name(
safe_findtext(
item,
'{http://www.openpkg.org/xml-rdf-index/0.9}Name'),
NameType.SRCRPM_NAME)
pkg.set_version(
safe_findtext(
item,
'{http://www.openpkg.org/xml-rdf-index/0.9}Version'))
pkg.add_licenses(
item.findtext(
'{http://www.openpkg.org/xml-rdf-index/0.9}License'))
pkg.set_summary(
item.findtext(
'{http://www.openpkg.org/xml-rdf-index/0.9}Summary'))
pkg.add_categories(
item.findtext(
'{http://www.openpkg.org/xml-rdf-index/0.9}Group'))
pkg.add_homepages(
item.findtext(
'{http://www.openpkg.org/xml-rdf-index/0.9}URL'))
for source in safe_findalltexts(
item,
'./{http://www.openpkg.org/xml-rdf-index/0.9}Source/{http://www.w3.org/1999/02/22-rdf-syntax-ns#}bag/{http://www.w3.org/1999/02/22-rdf-syntax-ns#}li'
):
if (source.startswith('https://')
or source.startswith('http://')
or source.startswith('ftp://')
) and 'openpkg.org' not in source:
pkg.add_downloads(source)
release = safe_findtext(
item, '{http://www.openpkg.org/xml-rdf-index/0.9}Release')
if pkg.version.endswith(release):
pkg.set_flags(PackageFlags.UNTRUSTED)
yield pkg
def iter_parse(self, path: str, factory: PackageFactory,
transformer: PackageTransformer) -> Iterable[PackageMaker]:
for item in lxml.html.parse(path).getroot().xpath(
'.//div[@id="rpms_list"]/ul/li/a'):
nevra = nevra_parse(item.text)
pkg = factory.begin()
pkg.set_name(nevra[0])
pkg.set_version(nevra[2])
pkg.set_rawversion(
nevra_construct(None, nevra[1], nevra[2], nevra[3]))
yield pkg
def iter_parse(self, path: str,
factory: PackageFactory) -> Iterable[PackageMaker]:
for row in lxml.html.parse(path).getroot().xpath(
'.//div[@class="packages"]/table/tbody/tr'):
pkg = factory.begin()
name, version = row.xpath('./td[1]/a')[0].text.rsplit('-', 1)
pkg.add_name(name, NameType.STACKAGE_NAME)
pkg.set_version(version)
pkg.set_summary((row.xpath('./td[2]')[0].text
or '').replace('\n', ' '))
yield pkg
def iter_parse(self, path: str, factory: PackageFactory) -> Iterable[PackageMaker]:
for row in lxml.html.parse(path).getroot().xpath('.//table[@class="ctable"]')[0].xpath('./form/tr[position()>3 and position()<last()-3]'):
pkg = factory.begin()
name, version, revision = row.xpath('./td[1]/a')[0].text.rsplit('-', 2)
# entries on kaosx.tk inconsistently have .pkg.tar.zst suffix, get rid of it
revision = revision.split('.pkg.')[0]
pkg.add_name(name, NameType.KAOS_NAME)
pkg.set_version(version.split(':', 1)[-1])
pkg.set_rawversion(version + '-' + revision)
yield pkg
def iter_parse(
self, path: str, factory: PackageFactory,
transformer: PackageTransformer
) -> Generator[PackageMaker, None, None]:
with open(path, 'rb') as jsonfile:
for package in JsonSlicer(jsonfile, (None, )):
pkg = factory.begin()
pkg.set_name(package['name'].split('@', 1)[0])
pkg.set_version(package['versions']['stable'])
pkg.set_summary(package['desc'])
pkg.add_homepages(package['homepage'])
yield pkg
def test_sanity(self):
factory = PackageFactory(NoopLogger())
maker = factory.begin()
self.assertFalse(maker.check_sanity())
maker.set_name('foo')
self.assertFalse(maker.check_sanity())
maker.set_version('1.0')
self.assertTrue(maker.check_sanity())
def iter_parse(self, path: str, factory: PackageFactory, transformer: PackageTransformer) -> Iterable[PackageMaker]:
for port in iter_json_list(path, ('ports', None)):
with factory.begin() as pkg:
pkg.add_name(port['name'], NameType.CRUX_NAME)
pkg.set_summary(port['description'])
pkg.set_version(port['version'])
if port['maintainer'] == '':
pkg.log('Missing maintainer for port "{}"'.format(port['name']), severity=Logger.ERROR)
else:
pkg.add_maintainers(extract_maintainers(port['maintainer']))
pkg.add_homepages(port['url'])
pkg.set_subrepo(port['repository'])
yield pkg
def iter_parse(
self, path: str, factory: PackageFactory,
transformer: PackageTransformer
) -> Generator[PackageMaker, None, None]:
root = xml.etree.ElementTree.parse(path)
repository = root.find(
'{http://www.openpkg.org/xml-rdf-index/0.9}Repository')
for item in repository.findall(
'{http://www.w3.org/1999/02/22-rdf-syntax-ns#}Description'
): # type: ignore
pkg = factory.begin()
pkg.set_name(
item.find('{http://www.openpkg.org/xml-rdf-index/0.9}Name').
text) # type: ignore
pkg.set_version(
item.find('{http://www.openpkg.org/xml-rdf-index/0.9}Version').
text) # type: ignore
pkg.add_licenses(
item.find('{http://www.openpkg.org/xml-rdf-index/0.9}License').
text) # type: ignore
pkg.set_summary(
item.find('{http://www.openpkg.org/xml-rdf-index/0.9}Summary').
text) # type: ignore
pkg.add_categories(
item.find('{http://www.openpkg.org/xml-rdf-index/0.9}Group').
text) # type: ignore
pkg.add_homepages(
item.find('{http://www.openpkg.org/xml-rdf-index/0.9}URL').text
) # type: ignore
for source in item.findall(
'./{http://www.openpkg.org/xml-rdf-index/0.9}Source/{http://www.w3.org/1999/02/22-rdf-syntax-ns#}bag/{http://www.w3.org/1999/02/22-rdf-syntax-ns#}li'
):
text = source.text
if (text.startswith('https://') or text.startswith('http://')
or text.startswith('ftp://')
) and 'openpkg.org' not in text: # type: ignore
pkg.add_downloads(text)
release = item.find(
'{http://www.openpkg.org/xml-rdf-index/0.9}Release'
).text # type: ignore
if pkg.version.endswith(release):
pkg.set_flags(PackageFlags.untrusted)
yield pkg
def iter_parse(self, path: str, factory: PackageFactory,
transformer: PackageTransformer) -> Iterable[PackageMaker]:
for versionpath in walk_tree(path, name='version'):
rootdir = os.path.dirname(versionpath)
with factory.begin(os.path.relpath(rootdir, path)) as pkg:
pkg.add_name(os.path.basename(rootdir), NameType.KISS_NAME)
with open(versionpath) as f:
version, revision = f.read().strip().split()
pkg.set_version(version)
with open(os.path.join(rootdir, 'sources')) as f:
for line in f:
line = line.strip()
if not line or line.startswith('#'):
continue
url, *rest = line.split()
if '://' in url:
pkg.add_downloads(url)
pkgpath = os.path.relpath(rootdir, path)
subrepo = os.path.split(pkgpath)[0]
pkg.set_extra_field('path', pkgpath)
pkg.set_subrepo(subrepo)
if self._maintainer_from_git:
command = [
'git', 'log', '-1', '--format=tformat:%ae',
os.path.relpath(versionpath, path)
]
with subprocess.Popen(command,
stdout=subprocess.PIPE,
encoding='utf-8',
errors='ignore',
cwd=path) as git:
lastauthor, _ = git.communicate()
pkg.add_maintainers(extract_maintainers(lastauthor))
patchesdir_abs = os.path.join(rootdir, 'patches')
if os.path.exists(patchesdir_abs):
pkg.set_extra_field('patch',
sorted(os.listdir(patchesdir_abs)))
yield pkg
def iter_parse(
self, path: str, factory: PackageFactory,
transformer: PackageTransformer
) -> Generator[PackageMaker, None, None]:
for category in os.listdir(path):
category_path = os.path.join(path, category)
if not os.path.isdir(category_path):
continue
for package in os.listdir(category_path):
package_path = os.path.join(category_path, package)
if not os.path.isdir(package_path):
continue
for recipe in os.listdir(package_path):
if not recipe.endswith('.recipe'):
continue
pkg = factory.begin()
pkg.set_name(package)
pkg.add_categories(category)
# may want to shadow haiku-only ports
#if pkg.category.startswith('haiku-'):
# pkg.shadow = True
# it seems to be guaranteed there's only one hyphen in recipe filename
name, version = recipe[:-7].split('-', 1)
if package.replace('-', '_') != name:
pkg.log(
'mismatch for package directory and recipe name: {} != {}'
.format(package, name),
severity=Logger.WARNING)
pkg.set_version(version)
# XXX: we rely on the fact that no substitutions happen in these
# variables. That's true as of 2018-05-14.
with open(os.path.join(category_path, package, recipe),
'r',
encoding='utf-8') as recipefile:
match = re.search('^HOMEPAGE="([^"]+)"',
recipefile.read(), re.MULTILINE)
if match:
pkg.add_homepages(match.group(1).split())
yield pkg
def iter_parse(self, path: str, factory: PackageFactory,
transformer: PackageTransformer) -> Iterable[PackageMaker]:
for pkgloc in _iter_packages(path):
with factory.begin(pkgloc.yamlpath_rel) as pkg:
try:
with open(pkgloc.yamlpath_abs, 'r') as fd:
pkgdata = yaml.safe_load(fd)
except UnicodeDecodeError:
pkg.log('Cannot read file, probably UTF-16 garbage',
Logger.ERROR)
continue
except yaml.MarkedYAMLError as e:
pkg.log(
f'YAML error at line {e.problem_mark.line}: {e.problem}',
Logger.ERROR)
continue
if 'PackageName' not in pkgdata:
pkg.log('No PackageName defined', Logger.ERROR)
continue
pkg.add_name(pkgdata['PackageIdentifier'], NameType.WINGET_ID)
pkg.add_name(pkgdata['PackageIdentifier'].split('.', 1)[-1],
NameType.WINGET_ID_NAME)
pkg.add_name(pkgdata['PackageName'], NameType.WINGET_NAME)
pkg.add_name(pkgloc.relevant_path, NameType.WINGET_PATH)
# Moniker field is optional and mosty useless
version = pkgdata['PackageVersion']
if isinstance(version, float):
pkg.log(
f'PackageVersion "{version}" is a floating point, should be quoted in YAML',
Logger.WARNING)
pkg.set_version(str(version))
pkg.add_homepages(pkgdata.get('PackageUrl'))
# pkg.set_summary(pkgdata.get('Description')) # may be long
# pkg.add_licenses(pkgdata['License']) # long garbage
pkg.add_categories(map(str, pkgdata.get('Tags', [])))
if 'Installers' in pkgdata:
pkg.add_downloads(installer['InstallerUrl']
for installer in pkgdata['Installers'])
pkg.set_extra_field('yamlpath', pkgloc.yamlpath_rel)
yield pkg
def test_redefine(self):
factory = PackageFactory(NoopLogger())
maker = factory.begin()
maker.set_name('foo')
maker.set_name('bar')
maker.set_version('1.0')
maker.set_version('1.1')
maker.set_summary('Foo')
maker.set_summary('Bar')
pkg = maker.unwrap()
self.assertEqual(pkg.name, 'bar')
self.assertEqual(pkg.version, '1.1')
self.assertEqual(pkg.comment, 'Bar')
def iter_parse(
self, path: str, factory: PackageFactory,
transformer: PackageTransformer
) -> Generator[PackageMaker, None, None]:
normalize_version = VersionStripper().strip_right('-')
for row in lxml.html.parse(path).getroot().xpath(
'.//table[@class="pdb"]')[0].xpath('./tr[@class="package"]'):
with factory.begin() as pkg:
pkg.set_name(row.xpath('./td[1]/a')[0].text)
pkg.set_version(
row.xpath('./td[2]')[0].text, normalize_version)
pkg.set_summary(row.xpath('./td[3]')[0].text)
yield pkg
def test_iter(self):
def iter_maintainers():
yield 'a@com'
yield ['b@com', None, '', 'c@com']
yield None
yield ''
yield 'd@com'
factory = PackageFactory(NoopLogger())
maker = factory.begin()
maker.add_maintainers(iter_maintainers())
pkg = maker.unwrap()
self.assertEqual(pkg.maintainers, ['a@com', 'b@com', 'c@com', 'd@com'])
def test_clone():
factory = PackageFactory(NoopLogger())
pkg1 = factory.begin('pkg1')
pkg1.add_maintainers('foo')
pkg2 = pkg1.clone('pkg2')
pkg2.add_maintainers('bar')
pkg3 = pkg1.clone(append_ident='pkg3')
pkg3.add_maintainers('baz')
assert pkg1.maintainers == ['foo']
assert pkg2.maintainers == ['foo', 'bar']
assert pkg3.maintainers == ['foo', 'baz']
def iter_parse(
self, path: str, factory: PackageFactory,
transformer: PackageTransformer
) -> Generator[PackageMaker, None, None]:
with open(path, 'rb') as jsonfile:
for item in JsonSlicer(jsonfile, ('packages', None)):
with factory.begin() as pkg:
pkg.set_name(item['name'])
pkg.set_version(item['ver'])
pkg.set_summary(item['descs'])
pkg.set_arch(item['arch'])
pkg.set_extra_field('location', item['loc'])
yield pkg
def test_clone(self) -> None:
factory = PackageFactory(NoopLogger())
pkg1 = factory.begin('pkg1')
pkg1.add_maintainers('foo')
pkg2 = pkg1.clone('pkg2')
pkg2.add_maintainers('bar')
pkg3 = pkg1.clone(append_ident='pkg3')
pkg3.add_maintainers('baz')
self.assertEqual(pkg1.maintainers, ['foo'])
self.assertEqual(pkg2.maintainers, ['foo', 'bar'])
self.assertEqual(pkg3.maintainers, ['foo', 'baz'])
def iter_parse(self, path: str, factory: PackageFactory,
transformer: PackageTransformer) -> Iterable[PackageMaker]:
with open(path, encoding='utf-8') as indexfile:
for line in indexfile:
pkg = factory.begin()
pkgname, pkgpath = line.strip().split('|')[:2]
name, version = pkgname.rsplit('-', 1)
pkg.add_name(name, NameType.HPUX_NAME)
pkg.set_version(version)
pkg.set_extra_field('path', pkgpath)
yield pkg
def iter_parse(self, path: str, factory: PackageFactory,
transformer: PackageTransformer) -> Iterable[PackageMaker]:
for pkgdata in _iter_packages(path):
with factory.begin(pkgdata['Package']) as pkg:
pkg.set_version(pkgdata['Version'], _normalize_version)
pkg.add_maintainers(
extract_maintainers(pkgdata.get('Maintainer', '')))
pkg.add_maintainers(
extract_maintainers(pkgdata.get('Uploaders', '')))
pkg.add_categories(pkgdata.get('Section'))
pkg.add_homepages(pkgdata.get('Homepage'))
self._extra_handling(pkg, pkgdata)
yield pkg
def iter_parse(self, path: str, factory: PackageFactory, transformer: PackageTransformer) -> Iterable[PackageMaker]:
for category, filename, size, date, version, description in _iter_index(path):
with factory.begin(filename) as pkg:
pkg.set_extra_field('filename', filename)
pkg.set_name(filename.rsplit('.', 1)[0])
if not version:
pkg.log('skipping, no version', Logger.ERROR)
continue
pkg.set_version(version)
pkg.set_summary(description)
pkg.add_categories(category)
yield pkg
def iter_parse(self, path: str,
factory: PackageFactory) -> Iterable[PackageMaker]:
for item in iter_json_list(path, ('packages', None)):
with factory.begin() as pkg:
pkg.add_name(item['name'], NameType.SALIX_NAME)
pkg.set_version(item['ver'])
pkg.set_summary(item['descs'])
pkg.set_arch(item['arch'])
# May be potentially useful for packagelinks, but not used now
# as there's no way to generate package-specific link to
# https://packages.salixos.org/#!
#pkg.set_extra_field('location', item['loc'])
yield pkg
def iter_parse(self, path: str, factory: PackageFactory,
transformer: PackageTransformer) -> Iterable[PackageMaker]:
for row in lxml.html.parse(path).getroot().xpath(
'.//table[@class="ctable"]')[0].xpath(
'./form/tr[position()>3 and position()<last()-3]'):
pkg = factory.begin()
name, version, revision = row.xpath('./td[1]/a')[0].text.rsplit(
'-', 2)
pkg.add_name(name, NameType.KAOS_NAME)
pkg.set_version(version.split(':', 1)[-1])
pkg.set_rawversion(version + '-' + revision)
yield pkg
def iter_parse(self, path: str, factory: PackageFactory,
transformer: PackageTransformer) -> Iterable[PackageMaker]:
for packagedata in iter_json_list(path, (None, )):
with factory.begin() as pkg:
pkg.add_name(packagedata['name'], NameType.HOMEBREW_NAME)
pkg.add_name(packagedata['name'].split('@', 1)[0],
NameType.HOMEBREW_NAME_PRE_AT)
pkg.add_name(packagedata['oldname'], NameType.HOMEBREW_OLDNAME)
pkg.add_name(packagedata['full_name'],
NameType.HOMEBREW_FULL_NAME)
pkg.set_version(packagedata['versions']['stable'])
pkg.set_summary(packagedata['desc'])
pkg.add_homepages(packagedata['homepage'])
yield pkg
def iter_parse(self, path: str, factory: PackageFactory,
transformer: PackageTransformer) -> Iterable[PackageMaker]:
normalize_version = VersionStripper().strip_right_greedy(
'-').strip_left(':').strip_right_greedy('+')
for package in os.listdir(path):
desc_path = os.path.join(path, package, 'desc')
if not os.path.isfile(desc_path):
continue
with open(desc_path, encoding='utf-8') as file:
pkg = factory.begin(package)
comment = None
tag = None
for line in file:
line = line.strip()
if line == '':
tag = None
elif tag == 'NAME':
pkg.set_name(line)
elif tag == 'VERSION':
pkg.set_version(line, normalize_version)
elif tag == 'ARCH':
pkg.set_arch(line)
elif tag == 'DESC':
if comment is None:
comment = ''
if comment != '':
comment += '\n'
comment += line
elif tag == 'URL':
pkg.add_homepages(line)
elif tag == 'LICENSE':
pkg.add_licenses(line)
#elif tag == 'PACKAGER':
# pkg.add_maintainers(extract_maintainers(line))
elif tag == 'BASE':
pkg.set_extra_field('base', line)
pkg.set_basename(line)
elif line.startswith('%') and line.endswith('%'):
tag = line[1:-1]
pkg.set_summary(comment)
yield pkg
def iter_parse(self, path: str, factory: PackageFactory, transformer: PackageTransformer) -> Generator[PackageMaker, None, None]:
with open(path) as listfile:
for line in listfile:
pkg = factory.begin()
filename = line.strip().split()[-1]
nevra = nevra_parse(filename)
pkg.set_name(nevra[0])
pkg.set_version(nevra[2])
pkg.set_rawversion(nevra_construct(None, nevra[1], nevra[2], nevra[3]))
pkg.set_extra_field('nevr', filename.rsplit('.', 2)[0])
yield pkg
def iter_parse(self, path: str, factory: PackageFactory,
transformer: PackageTransformer) -> Iterable[PackageMaker]:
for pkgdata in _iter_apkindex(os.path.join(path, 'APKINDEX')):
with factory.begin(pkgdata['P']) as pkg:
pkg.add_name(pkgdata['P'], NameType.APK_BIG_P)
pkg.add_name(pkgdata['o'], NameType.APK_SMALL_O)
pkg.set_version(pkgdata['V'], _normalize_version)
pkg.set_summary(pkgdata['T'])
pkg.add_homepages(pkgdata['U']) # XXX: split?
pkg.add_licenses(pkgdata['L'])
pkg.set_arch(pkgdata['A'])
pkg.add_maintainers(extract_maintainers(pkgdata.get('m')))
yield pkg
def _parse_latest_packages(packages: Iterable[dict[str, Any]],
latest_versions: dict[str, str],
factory: PackageFactory) -> Iterable[PackageMaker]:
for fields in packages:
# only take latest versions (there's only one of them per distribution)
if _as_str(fields['status']) != 'latest':
continue
distribution, pkg = _parse_package(factory.begin(), fields)
if not pkg.version:
pkg.log('empty version', severity=Logger.ERROR)
continue
latest_versions[distribution] = pkg.version
yield pkg
