def iter_parse(self, path: str, factory: PackageFactory, transformer: PackageTransformer) -> Iterable[PackageMaker]:
for recipe_name in os.listdir(path):
if recipe_name.startswith('.'):
continue
pkg = factory.begin()
pkg.add_name(recipe_name, NameType.GOBOLINUX_RECIPE)
package_path = os.path.join(path, recipe_name)
maxversion: Optional[str] = None
for version_name in os.listdir(package_path):
if maxversion is None or version_compare(version_name, maxversion) > 0:
maxversion = version_name
if maxversion is None:
pkg.log('no usable versions found', severity=Logger.ERROR)
continue
pkg.set_version(maxversion)
recipe_path = os.path.join(package_path, maxversion, 'Recipe')
description_path = os.path.join(package_path, maxversion, 'Resources', 'Description')
if os.path.isfile(recipe_path):
with open(recipe_path, 'r', encoding='utf-8', errors='ignore') as recipe:
for line in recipe:
line = line.strip()
if line.startswith('url='):
download = _expand_mirrors(line[4:])
if '$' not in download:
pkg.add_downloads(download.strip('"'))
else:
factory.log('Recipe for {}/{} skipped, unhandled URL substitute found'.format(recipe_name, maxversion), severity=Logger.ERROR)
if os.path.isfile(description_path):
with open(description_path, 'r', encoding='utf-8', errors='ignore') as description:
data = {}
current_tag = None
for line in description:
line = line.strip()
match = re.match('^\\[([A-Z][a-z]+)\\] *(.*?)$', line)
if match:
current_tag = match.group(1)
data[current_tag] = match.group(2)
elif current_tag is None:
factory.log('Description for {}/{} skipped, dumb format'.format(recipe_name, maxversion), severity=Logger.ERROR)
break
elif line:
if data[current_tag]:
data[current_tag] += ' '
data[current_tag] += line
pkg.set_summary(data.get('Summary'))
pkg.add_licenses(data.get('License'))
pkg.add_homepages(data.get('Homepage', '').strip('"'))
yield pkg
def iter_parse(self, path: str, factory: PackageFactory,
transformer: PackageTransformer) -> Iterable[PackageMaker]:
for fmri, pkgdata in _iter_packages(path):
pkg = factory.begin('{} {}'.format(fmri, pkgdata['version']))
pkg.set_extra_field('fmri', fmri)
variables = {}
for action in pkgdata['actions']:
tokens = shlex.split(action)
if not tokens or tokens.pop(0) != 'set':
factory.log('unrecognized action ' + action,
severity=Logger.ERROR)
continue
key = None
value = []
for token in tokens:
if token.startswith('name='):
key = token[5:]
elif token.startswith('value='):
value.append(token[6:])
elif token.startswith('last-fmri='):
pass
else:
factory.log('unrecognized token ' + token,
severity=Logger.ERROR)
continue
if key and value:
variables[key] = value
# these are entries without name, likely not really packages
# skip these early to avoid parsing other stuff and polluting logs with warnings
if 'com.oracle.info.name' not in variables or 'com.oracle.info.version' not in variables:
continue
# Regarding comment requirement: there are some packages which lack it,
# however for ALL of them have counterparts with comment and some
# additional fields (category, homepage, downloads). Packages without
# comment look like legacy, and it's OK and desirable to drop them here
if 'pkg.summary' not in variables:
continue
pkg.add_name(variables['com.oracle.info.name'][0],
NameType.GENERIC_PKGNAME)
pkg.set_version(variables['com.oracle.info.version'][0])
pkg.set_summary(variables['pkg.summary'][0])
for category in variables.get('info.classification', []):
if category.startswith('org.opensolaris.category.2008:'):
pkg.add_categories(category.split(':', 1)[1])
pkg.add_homepages(variables.get('info.upstream-url'))
pkg.add_downloads(variables.get('info.source-url'))
yield pkg
def iter_parse(self, path: str, factory: PackageFactory,
transformer: PackageTransformer) -> Iterable[PackageMaker]:
total_entries = 0
for packagedata in _iter_packages(path):
entity = packagedata['project'].rsplit(
'/', 1)[-1] # this is URL, take only the ID from it
label = packagedata['projectLabel']
with factory.begin('{} ({})'.format(entity, label)) as pkg:
pkg.set_extra_field('entity', entity)
pkg.set_name(label)
pkg.set_summary(packagedata.get('projectDescription'))
pkg.add_licenses(packagedata.get('licenses', '').split(', '))
pkg.add_homepages(packagedata.get('websites', '').split(', '))
names = set(packagedata['repology_projects'].split(
', ')) if packagedata['repology_projects'] else set()
if len(names) > 1:
pkg.log('multiple Repology project names: {}'.format(
','.join(sorted(names))),
severity=Logger.WARNING)
total_entries += 1
# generate a package for each version
for version in sorted(packagedata['versions'].split(', ')):
version, *flags = version.split('|')
verpkg = pkg.clone()
is_devel = 'U' in flags
is_foreign_os_release = 'o' in flags and 'O' not in flags
is_foreign_platform_release = 'p' in flags and 'P' not in flags
if is_foreign_os_release:
verpkg.log(
'version {} skipped due to bad OS'.format(version),
severity=Logger.NOTICE)
continue
if is_foreign_platform_release:
verpkg.log(
'version {} skipped due to bad Platform'.format(
version),
severity=Logger.NOTICE)
continue
verpkg.set_flags(PackageFlags.DEVEL, is_devel)
verpkg.set_version(version)
# generate package for each guessed name; it most cases, these will be merged anyway
for name in names:
namepkg = verpkg.clone()
namepkg.set_basename(name)
yield namepkg
factory.log('{} total entries'.format(total_entries))
def iter_parse(self, path: str, factory: PackageFactory, transformer: PackageTransformer) -> Iterable[PackageMaker]:
normalize_version = VersionStripper().strip_right_greedy('+')
skipped_archs: Dict[str, int] = {}
for entry in iter_xml_elements_at_level(path, 1, ['{http://linux.duke.edu/metadata/common}package']):
with factory.begin() as pkg:
arch = safe_findtext(entry, '{http://linux.duke.edu/metadata/common}arch')
if self.allowed_archs and arch not in self.allowed_archs:
skipped_archs[arch] = skipped_archs.get(arch, 0) + 1
continue
name = safe_findtext(entry, '{http://linux.duke.edu/metadata/common}name')
if '%{' in name:
pkg.log('incorrect package name (unexpanded substitution)', severity=Logger.ERROR)
continue
pkg.add_name(name, NameType.GENERIC_PKGNAME)
version_elt = entry.find('{http://linux.duke.edu/metadata/common}version')
if version_elt is None:
raise RuntimeError('Cannot find <version> element')
epoch = version_elt.attrib['epoch']
version = version_elt.attrib['ver']
release = version_elt.attrib['rel']
match = re.match('0\\.[0-9]+\\.((?:alpha|beta|rc)[0-9]+)\\.', release)
if match:
# known pre-release schema: https://fedoraproject.org/wiki/Packaging:Versioning#Prerelease_versions
version += '-' + match.group(1)
elif release < '1':
# unknown pre-release schema: https://fedoraproject.org/wiki/Packaging:Versioning#Some_definitions
# most likely a snapshot
pkg.set_flags(PackageFlags.IGNORE)
pkg.set_version(version, normalize_version)
pkg.set_rawversion(nevra_construct(None, epoch, version, release))
pkg.set_summary(entry.findtext('{http://linux.duke.edu/metadata/common}summary'))
pkg.add_homepages(entry.findtext('{http://linux.duke.edu/metadata/common}url'))
pkg.add_categories(entry.findtext('{http://linux.duke.edu/metadata/common}format/'
'{http://linux.duke.edu/metadata/rpm}group'))
pkg.add_licenses(entry.findtext('{http://linux.duke.edu/metadata/common}format/'
'{http://linux.duke.edu/metadata/rpm}license'))
pkg.set_arch(entry.findtext('{http://linux.duke.edu/metadata/common}arch'))
packager = entry.findtext('{http://linux.duke.edu/metadata/common}packager')
if packager:
pkg.add_maintainers(extract_maintainers(packager))
yield pkg
for arch, numpackages in sorted(skipped_archs.items()):
factory.log('skipped {} packages(s) with disallowed architecture {}'.format(numpackages, arch))
def iter_parse(self, path: str, factory: PackageFactory) -> Iterable[PackageMaker]:
num_manifests_without_packagename = 0
for pkgloc in _iter_packages(path):
with factory.begin(pkgloc.yamlpath_rel) as pkg:
try:
with open(pkgloc.yamlpath_abs, 'r') as fd:
pkgdata = yaml.safe_load(fd)
except UnicodeDecodeError:
pkg.log('Cannot read file, probably UTF-16 garbage', Logger.ERROR)
continue
except yaml.MarkedYAMLError as e:
if e.problem_mark:
pkg.log(f'YAML error at line {e.problem_mark.line}: {e.problem}', Logger.ERROR)
else:
pkg.log(f'YAML error: {e.problem}', Logger.ERROR)
continue
if 'PackageName' not in pkgdata:
num_manifests_without_packagename += 1
continue
pkg.add_name(pkgdata['PackageIdentifier'], NameType.WINGET_ID)
pkg.add_name(pkgdata['PackageIdentifier'].split('.', 1)[-1], NameType.WINGET_ID_NAME)
pkg.add_name(pkgdata['PackageName'], NameType.WINGET_NAME)
pkg.add_name(pkgloc.relevant_path, NameType.WINGET_PATH)
# Moniker field is optional and mosty useless
version = pkgdata['PackageVersion']
if isinstance(version, float):
pkg.log(f'PackageVersion "{version}" is a floating point, should be quoted in YAML', Logger.WARNING)
pkg.set_version(str(version))
pkg.add_homepages(pkgdata.get('PackageUrl'))
# pkg.set_summary(pkgdata.get('Description')) # may be long
# pkg.add_licenses(pkgdata['License']) # long garbage
pkg.add_categories(map(str, pkgdata.get('Tags', [])))
if 'Installers' in pkgdata:
pkg.add_downloads(installer['InstallerUrl'] for installer in pkgdata['Installers'])
pkg.set_extra_field('yamlpath', pkgloc.yamlpath_rel)
yield pkg
factory.log(f'Skipped manifests without PackageName: {num_manifests_without_packagename}', Logger.WARNING)
def iter_parse(self, path: str, factory: PackageFactory, transformer: PackageTransformer) -> Iterable[PackageMaker]:
skipped_gemplats: Dict[str, int] = Counter()
with open(path, 'rb') as fd:
for gemname, gemversion, gemplat in rubymarshal.reader.load(fd):
gemname = str(gemname)
with factory.begin(gemname) as pkg:
if gemplat != 'ruby':
skipped_gemplats[gemplat] += 1
continue
gemversion = str(gemversion.marshal_dump()[0])
pkg.add_name(gemname, NameType.RUBYGEMS_NAME)
pkg.set_version(gemversion)
yield pkg
for gemplat, count in sorted(skipped_gemplats.items()):
factory.log(f'skipped {count} occurrence(s) of gemplat {gemplat}')
def iter_parse(self, path: str,
factory: PackageFactory) -> Iterable[PackageMaker]:
has_control_files = False
for pkgdir in os.listdir(os.path.join(path, 'ports')):
with factory.begin(pkgdir) as pkg:
package_path_abs = os.path.join(path, 'ports', pkgdir)
controlpath = os.path.join(package_path_abs, 'CONTROL')
manifestpath = os.path.join(package_path_abs, 'vcpkg.json')
# read either of old-style control (CONTROL) or new-style manifest (vcpkg.json) file
if os.path.exists(manifestpath):
pkgdata = _read_manifest_file(manifestpath)
elif os.path.exists(controlpath):
has_control_files = True
pkgdata = _read_control_file(controlpath)
else:
pkg.log('neither control nor manifest file found',
Logger.ERROR)
continue
if pkgdata['name'] != pkgdir:
raise RuntimeError(
f'sanity check failed: source {pkgdata["name"]} != directory {pkgdir}'
)
pkg.add_name(pkgdata['name'], NameType.VCPKG_SOURCE)
for key in [
'version', 'version-string', 'version-semver',
'version-date'
]:
if key in pkgdata:
version = pkgdata[key]
break
else:
raise RuntimeError(
'none of expected version schemes found')
if re.match('[0-9]{4}[.-][0-9]{1,2}[.-][0-9]{1,2}', version):
pkg.set_version(version)
pkg.set_flags(PackageFlags.UNTRUSTED)
else:
pkg.set_version(version, _normalize_version)
# handle description which may be either a string or a list of strings
description = pkgdata.get('description')
if isinstance(description, str):
pkg.set_summary(description)
elif isinstance(description, list):
pkg.set_summary(description[0])
pkg.add_homepages(pkgdata.get('homepage'))
for maintainer in pkgdata.get('maintainers', []):
pkg.add_maintainers(extract_maintainers(maintainer))
# pretty much a hack to shut a bunch of fake versions up
portfilepath = os.path.join(package_path_abs, 'portfile.cmake')
if _grep_file(portfilepath, 'libimobiledevice-win32'):
pkg.log(
'marking as untrusted, https://github.com/libimobiledevice-win32 accused of version faking',
severity=Logger.WARNING)
pkg.set_flags(PackageFlags.UNTRUSTED)
add_patch_files(pkg, package_path_abs, '*.patch')
yield pkg
if not has_control_files:
factory.log(
"No CONTROL files seen in the repository, seems like it's time to refactor vcpkg parser and remove legacy bits"
)
def iter_parse(self, path: str, factory: PackageFactory,
transformer: PackageTransformer) -> Iterable[PackageMaker]:
total_entries = 0
for packagedata in _iter_packages(path):
entity = packagedata['project'].rsplit(
'/', 1)[-1] # this is URL, take only the ID from it
label = packagedata['projectLabel']
with factory.begin('{} ({})'.format(entity, label)) as pkg:
pkg.add_name(entity, NameType.WIKIDATA_ENTITY)
pkg.add_name(label, NameType.WIKIDATA_LABEL)
pkg.set_summary(packagedata.get('projectDescription'))
pkg.add_licenses(
sorted(packagedata.get('licenses', '').split('␞')))
pkg.add_links(
LinkType.UPSTREAM_HOMEPAGE,
sorted(packagedata.get('websites', '').split('␞')))
pkg.add_links(
LinkType.UPSTREAM_REPOSITORY,
sorted(packagedata.get('repositories', '').split('␞')))
names = sorted(packagedata['repology_projects'].split('␞'))
if len(names) > 1:
pkg.log('multiple Repology project names: {}'.format(
','.join(names)),
severity=Logger.WARNING)
total_entries += 1
pkgs_preferred = []
pkgs_normal = []
# generate a package for each version
for version in sorted(packagedata['versions'].split('␞')):
version, *flags = version.split('|')
verpkg = pkg.clone()
is_version_stable = 'S' in flags
is_version_unstable = 'U' in flags
is_version_bad = 'X' in flags
is_foreign_os_release = 'o' in flags and 'O' not in flags
is_foreign_platform_release = 'p' in flags and 'P' not in flags
is_preferred = 'R' in flags
if is_foreign_os_release:
verpkg.log(
'version {} skipped due to bad OS'.format(version),
severity=Logger.NOTICE)
continue
if is_foreign_platform_release:
verpkg.log(
'version {} skipped due to bad Platform'.format(
version),
severity=Logger.NOTICE)
continue
if is_version_bad:
verpkg.log(
'version {} skipped due to bad Version Type'.
format(version),
severity=Logger.ERROR)
continue
verpkg.set_flags(
PackageFlags.DEVEL, is_version_unstable
and not is_version_stable)
verpkg.set_version(version)
pkg.add_links(
LinkType.UPSTREAM_DOWNLOAD,
sorted(flag for flag in flags if len(flag) > 1))
# generate package for each repology project
for name in names:
namepkg = verpkg.clone()
namepkg.add_name(
name, NameType.WIKIDATA_REPOLOGY_PROJECT_NAME)
if is_preferred:
pkgs_preferred.append(namepkg)
else:
pkgs_normal.append(namepkg)
if pkgs_preferred:
# temporary diagnostics until Wikidata usage in Wikipedia gains more momentum
# before that, we can/have to help keeping it up to date
max_preferred_version = max(
(pkg.version for pkg in pkgs_preferred),
key=cmp_to_key(version_compare))
max_normal_version = max(
(pkg.version for pkg in pkgs_normal
if not pkg.flags & PackageFlags.DEVEL),
key=cmp_to_key(version_compare),
default=None)
max_devel_version = max(
(pkg.version for pkg in pkgs_normal
if pkg.flags & PackageFlags.DEVEL),
key=cmp_to_key(version_compare),
default=None)
if max_normal_version and version_compare(
max_preferred_version, max_normal_version) < 0:
pkg.log(
'preferred version {} < normal {} (may need rank update)'
.format(max_preferred_version, max_normal_version),
severity=Logger.WARNING)
if max_devel_version and version_compare(
max_devel_version, max_preferred_version) > 0:
pkg.log(
'devel version {} < preferred {} (may need preferred rank for devel version)'
.format(max_devel_version, max_preferred_version),
severity=Logger.WARNING)
yield from pkgs_preferred
else:
num_devel = 0
num_stable = 0
for pkg in pkgs_normal:
if pkg.flags & PackageFlags.DEVEL:
num_devel += 1
else:
num_stable += 1
if num_devel > len(names):
pkg.log(
'no preferred devel version(s) set (out of {})'.
format(num_devel // len(names)),
severity=Logger.WARNING)
if num_stable > len(names):
pkg.log(
'no preferred stable version(s) set (out of {})'.
format(num_stable // len(names)),
severity=Logger.WARNING)
yield from pkgs_normal
factory.log('{} total entries'.format(total_entries))
def iter_parse(self, path: str, factory: PackageFactory,
transformer: PackageTransformer) -> Iterable[PackageMaker]:
normalize_version = VersionStripper().strip_right_greedy('+')
skipped_archs: Dict[str, int] = Counter()
if self._arch_from_filename:
factory.log('mitigation for incorrect <arch></arch> enabled',
severity=Logger.WARNING)
for entry in iter_xml_elements_at_level(
path, 1, ['{http://linux.duke.edu/metadata/common}package']):
if self._arch_from_filename:
# XXX: openmandriva 3 hack, to be removed when it EoLs
location_elt = entry.find(
'{http://linux.duke.edu/metadata/common}location')
if location_elt is None:
raise RuntimeError('Cannot find <location> element')
arch = nevra_parse(safe_getattr(location_elt, 'href'))[4]
else:
arch = safe_findtext(
entry, '{http://linux.duke.edu/metadata/common}arch')
is_src = arch == 'src'
if (is_src and not self._src) or (not is_src and not self._binary):
skipped_archs[arch] += 1
continue
with factory.begin() as pkg:
name = safe_findtext(
entry, '{http://linux.duke.edu/metadata/common}name')
if '%{' in name:
pkg.log('incorrect package name (unexpanded substitution)',
severity=Logger.ERROR)
continue
if is_src:
pkg.add_name(name, NameType.SRCRPM_NAME)
else:
pkg.add_name(name, NameType.BINRPM_NAME)
sourcerpm = safe_findtext(
entry, '{http://linux.duke.edu/metadata/common}format/'
'{http://linux.duke.edu/metadata/rpm}sourcerpm')
pkg.add_name(
nevra_parse(sourcerpm)[0], NameType.BINRPM_SRCNAME)
version_elt = entry.find(
'{http://linux.duke.edu/metadata/common}version')
if version_elt is None:
raise RuntimeError('Cannot find <version> element')
epoch = version_elt.attrib['epoch']
version = version_elt.attrib['ver']
release = version_elt.attrib['rel']
fixed_version = version
match = re.match('0\\.[0-9]+\\.((?:alpha|beta|rc)[0-9]+)\\.',
release)
if match:
# known pre-release schema: https://fedoraproject.org/wiki/Packaging:Versioning#Prerelease_versions
fixed_version += '-' + match.group(1)
elif release < '1':
# unknown pre-release schema: https://fedoraproject.org/wiki/Packaging:Versioning#Some_definitions
# most likely a snapshot
pkg.set_flags(PackageFlags.IGNORE)
pkg.set_version(fixed_version, normalize_version)
pkg.set_rawversion(
nevra_construct(None, epoch, version, release))
pkg.set_summary(
entry.findtext(
'{http://linux.duke.edu/metadata/common}summary'))
pkg.add_homepages(
entry.findtext(
'{http://linux.duke.edu/metadata/common}url'))
pkg.add_categories(
entry.findtext(
'{http://linux.duke.edu/metadata/common}format/'
'{http://linux.duke.edu/metadata/rpm}group'))
pkg.add_licenses(
entry.findtext(
'{http://linux.duke.edu/metadata/common}format/'
'{http://linux.duke.edu/metadata/rpm}license'))
pkg.set_arch(
entry.findtext(
'{http://linux.duke.edu/metadata/common}arch'))
packager = entry.findtext(
'{http://linux.duke.edu/metadata/common}packager')
if packager:
pkg.add_maintainers(extract_maintainers(packager))
yield pkg
for arch, numpackages in sorted(skipped_archs.items()):
factory.log(
'skipped {} packages(s) with disallowed architecture {}'.
format(numpackages, arch))
def iter_parse(
self, path: str, factory: PackageFactory,
transformer: PackageTransformer
) -> Generator[PackageMaker, None, None]:
with open(path, 'rb') as jsonfile:
for key, packagedata in JsonSlicer(jsonfile, ('packages', None),
encoding='utf-8',
path_mode='map_keys'):
pkg = factory.begin(key)
# see how Nix parses 'derivative' names in
# https://github.com/NixOS src/libexpr/names.cc, DrvName::DrvName
# it just splits on dash followed by non-letter
#
# this doesn't work well on 100% cases, it's an upstream problem
match = re.match('(.+?)-([^a-zA-Z].*)$', packagedata['name'])
if not match:
factory.log('cannot extract version: {}/{}'.format(
key, packagedata['name']),
severity=Logger.ERROR)
continue
pkg.set_name(match.group(1))
pkg.set_version(match.group(2))
# some exceptions
for prefix in ('75dpi', '100dpi'):
if pkg.version.startswith(prefix):
pkg.set_name(pkg.name + '-' + prefix)
pkg.set_version(pkg.version[len(prefix) + 1:])
merged = pkg.name + '-' + pkg.version
for pkgname in [
'liblqr-1', 'python2.7-3to2', 'python3.6-3to2',
'libretro-4do', 'polkit-qt-1-qt5', 'polkit-qt-1-qt4'
]:
if merged.startswith(pkgname):
pkg.set_name(pkgname)
pkg.set_version(merged[len(pkgname) + 1:])
keyparts = key.split('.')
if len(keyparts) > 1:
pkg.add_categories(keyparts[0])
if pkg.name.endswith('-git'):
pkg.set_name(pkg.name[:-4])
pkg.set_flags(PackageFlags.ignore)
if re.match('.*20[0-9]{2}-[0-9]{2}-[0-9]{2}', pkg.version):
pkg.set_flags(PackageFlags.ignore)
if re.match('[0-9a-f]*[a-f][0-9a-f]*$',
pkg.version) and len(pkg.version) >= 7:
pkg.log(
'ignoring version which looks like commit hash: {}'.
format(pkg.version),
severity=Logger.ERROR)
pkg.set_flags(PackageFlags.ignore)
meta = packagedata['meta']
pkg.add_homepages(meta.get('homepage'))
if 'description' in meta:
pkg.set_summary(meta['description'].replace('\n', ' '))
if 'maintainers' in meta:
if not isinstance(meta['maintainers'], list):
pkg.log('maintainers is not a list: {}'.format(
meta['maintainers']),
severity=Logger.ERROR)
else:
pkg.add_maintainers(
extract_nix_maintainers(meta['maintainers']))
if 'license' in meta:
pkg.add_licenses(extract_nix_licenses(meta['license']))
if 'position' in meta:
posfile, posline = meta['position'].rsplit(':', 1)
pkg.set_extra_field('posfile', posfile)
pkg.set_extra_field('posline', posline)
if posfile.startswith('pkgs/development/haskell-modules'):
pkg.set_flags(
PackageFlags.rolling
) # XXX: haskell modules are autogenerated in nix: https://github.com/NixOS/nixpkgs/commits/master/pkgs/development/haskell-modules/hackage-packages.nix
yield pkg
def iter_parse(self, path: str,
factory: PackageFactory) -> Iterable[PackageMaker]:
# All encounered version_status values:
# alpha, beta, developmental, historical, mature, planning, rolling, stable, testing, unknown, unstable
_unstable_versions = {
'alpha', 'beta', 'developmental', 'planning', 'testing', 'unstable'
}
num_total = 0
num_nover = 0
num_noneng = 0
num_debian = 0
num_obsolete = 0
num_accepted = 0
num_devel = 0
for entry in iter_xml_elements_at_level(
path, 1,
['{http://semantic-mediawiki.org/swivt/1.0#}Subject']):
pages = _get_attrs(
entry, '{http://semantic-mediawiki.org/swivt/1.0#}page',
'{http://www.w3.org/1999/02/22-rdf-syntax-ns#}resource')
if not pages:
continue
page = _unescape(pages[0].split('/')[-1])
with factory.begin(page) as pkg:
label = safe_findtext(
entry, '{http://www.w3.org/2000/01/rdf-schema#}label')
name = entry.findtext(
'{http://directory.fsf.org/wiki/Special:URIResolver/Property-3A}Name'
)
version = entry.findtext(
'{http://directory.fsf.org/wiki/Special:URIResolver/Property-3A}Version_identifier'
)
if name is None:
continue
num_total += 1
if version is None:
num_nover += 1
continue
if entry.findtext(
'{http://semantic-mediawiki.org/swivt/1.0#}wikiPageContentLanguage'
) != 'en':
num_noneng += 1
continue
if entry.findtext(
'{http://directory.fsf.org/wiki/Special:URIResolver/Property-3A}Import_source'
) == 'Debian': # 'Debian import' seems OK though
num_debian += 1
continue
if entry.findtext(
'{http://directory.fsf.org/wiki/Special:URIResolver/Property-3A}Decommissioned_or_Obsolete'
) == 'Yes':
num_obsolete += 1
continue
if self._high_priority and entry.findtext(
'{http://directory.fsf.org/wiki/Special:URIResolver/Property-3A}Is_High_Priority_Project'
) != 'true':
continue
version_status = entry.findtext(
'{http://directory.fsf.org/wiki/Special:URIResolver/Property-3A}Version_status'
)
if version_status in _unstable_versions:
num_devel += 1
pkg.set_flags(PackageFlags.DEVEL)
elif version_status == 'rolling':
pkg.set_flags(PackageFlags.ROLLING)
num_accepted += 1
pkg.add_name(page, NameType.GENERIC_GEN_NAME)
pkg.set_version(version)
pkg.set_summary(
entry.findtext(
'{http://directory.fsf.org/wiki/Special:URIResolver/Property-3A}Short_description'
))
pkg.add_homepages(
_get_attrs(
entry,
'{http://directory.fsf.org/wiki/Special:URIResolver/Property-3A}Homepage_URL',
'{http://www.w3.org/1999/02/22-rdf-syntax-ns#}resource'
))
pkg.add_downloads(
_get_attrs(
entry,
'{http://directory.fsf.org/wiki/Special:URIResolver/Property-3A}Version_download',
'{http://www.w3.org/1999/02/22-rdf-syntax-ns#}resource'
))
pkg.set_extra_field('page', page)
pkg.set_extra_field('name', name)
pkg.set_extra_field('label', label)
yield pkg
factory.log(
'Total software entries (with Name and Version): {}'.format(
num_total))
factory.log(
'Dropped entries with no version defined: {}'.format(num_nover))
factory.log('Dropped non-english pages: {}'.format(num_noneng))
factory.log(
'Dropped entries marked as Import_source=Debian: {}'.format(
num_debian))
factory.log(
'Dropped entries marked as Decommissioned_or_Obsolete: {}'.format(
num_obsolete))
factory.log('Accepted entries: {} ({} unstable)'.format(
num_accepted, num_devel))
