def ParsePackage(self, fmri, pkgdata):
variables = {}
for action in pkgdata['actions']:
tokens = shlex.split(action)
if not tokens or tokens.pop(0) != 'set':
print('WARNING: unrecognized action ' + action,
file=sys.stderr)
continue
key = None
value = []
for token in tokens:
if token.startswith('name='):
key = token[5:]
elif token.startswith('value='):
value.append(token[6:])
elif token.startswith('last-fmri='):
pass
else:
print('WARNING: unrecognized token ' + token,
file=sys.stderr)
continue
if key and value:
variables[key] = value
pkg = Package()
pkg.extrafields['fmri'] = fmri
if 'com.oracle.info.name' in variables:
pkg.name = variables['com.oracle.info.name'][0]
if 'com.oracle.info.version' in variables:
pkg.version = variables['com.oracle.info.version'][0]
if 'pkg.summary' in variables:
pkg.comment = variables['pkg.summary'][0]
if 'info.classification' in variables:
pkg.category = variables['info.classification'][0]
if pkg.category.startswith('org.opensolaris.category.2008:'):
pkg.category = pkg.category.split(':', 1)[1]
if 'info.upstream-url' in variables:
pkg.homepage = variables['info.upstream-url'][0]
if 'info.source-url' in variables:
pkg.downloads = variables['info.source-url']
# Regarding comment requirement: there are some packages which lack it,
# however for ALL of them is a counterpart with comment and some
# additional fields (category, homepage, downloads). Packages without
# comment look like legacy, and it's OK and desirable to drop them here
if pkg.name and pkg.version and pkg.comment:
return pkg
return None
def ParsePackage(self, pkgpath, pkgdata):
variables = {}
for action in pkgdata['actions']:
tokens = shlex.split(action)
if not tokens or tokens.pop(0) != 'set':
print('WARNING: unrecognized action ' + action,
file=sys.stderr)
continue
key = None
value = []
for token in tokens:
if token.startswith('name='):
key = token[5:]
elif token.startswith('value='):
value.append(token[6:])
elif token.startswith('last-fmri='):
pass
else:
print('WARNING: unrecognized token ' + token,
file=sys.stderr)
continue
if key and value:
variables[key] = value
pkg = Package()
pkg.extrafields['path'] = pkgpath
if 'com.oracle.info.name' in variables:
pkg.name = variables['com.oracle.info.name'][0]
if 'com.oracle.info.version' in variables:
pkg.version = variables['com.oracle.info.version'][0]
if 'pkg.summary' in variables:
pkg.comment = variables['pkg.summary'][0]
if 'info.classification' in variables:
pkg.category = variables['info.classification'][0]
if pkg.category.startswith('org.opensolaris.category.2008:'):
pkg.category = pkg.category.split(':', 1)[1]
if 'info.upstream-url' in variables:
pkg.homepage = variables['info.upstream-url'][0]
if 'info.source-url' in variables:
pkg.downloads = variables['info.source-url']
if pkg.name and pkg.version:
return pkg
return None
def Parse(self, path):
result = []
with open(path, encoding='utf-8') as indexfile:
for line in indexfile:
fields = line.strip().split('|')
if len(fields) != 12:
print('WARNING: package {} skipped, incorrect number of fields in INDEX'.format(fields[0]), file=sys.stderr)
continue
pkg = Package()
pkg.name, version = SplitPackageNameVersion(fields[0])
pkg.version, pkg.origversion = SanitizeVersion(version)
pkg.comment = fields[3]
if fields[11]:
pkg.homepage = fields[11]
# sometimes OWNER variable is used in which case
# there's no MAINTAINER OWNER doesn't get to INDEX
pkg.maintainers = GetMaintainers(fields[5])
pkg.category = fields[6].split(' ')[0]
result.append(pkg)
return result
def Parse(self, path):
result = []
root = xml.etree.ElementTree.parse(path)
for entry in root.findall(
'{http://linux.duke.edu/metadata/common}package'):
pkg = Package()
pkg.name = entry.find(
'{http://linux.duke.edu/metadata/common}name').text
version = entry.find(
'{http://linux.duke.edu/metadata/common}version').attrib['ver']
pkg.version, pkg.origversion = SanitizeVersion(version)
pkg.comment = entry.find(
'{http://linux.duke.edu/metadata/common}summary').text
pkg.homepage = entry.find(
'{http://linux.duke.edu/metadata/common}url').text
pkg.category = entry.find(
'{http://linux.duke.edu/metadata/common}format/'
'{http://linux.duke.edu/metadata/rpm}group').text
pkg.licenses.append(
entry.find('{http://linux.duke.edu/metadata/common}format/'
'{http://linux.duke.edu/metadata/rpm}license').text)
packager = entry.find(
'{http://linux.duke.edu/metadata/common}packager').text
if packager:
pkg.maintainers = GetMaintainers(packager)
result.append(pkg)
return result
def Parse(self, path):
result = []
with open(path, encoding='utf-8') as indexfile:
for line in indexfile:
fields = line.strip().split('|')
if len(fields) != 13:
print(
'WARNING: package {} skipped, incorrect number of fields in INDEX'
.format(fields[0]),
file=sys.stderr)
continue
pkg = Package()
pkg.name, version = SplitPackageNameVersion(fields[0])
pkg.version, pkg.origversion = SanitizeVersion(version)
pkg.comment = fields[3]
pkg.maintainers = GetMaintainers(fields[5])
pkg.category = fields[6].split(' ')[0]
if fields[9]:
pkg.homepage = fields[9]
result.append(pkg)
return result
def iter_parse(self, path):
with open(path, encoding='utf-8') as indexfile:
for line in indexfile:
fields = line.strip().split('|')
if len(fields) != 12:
print(
'WARNING: package {} skipped, incorrect number of fields in INDEX'
.format(fields[0]),
file=sys.stderr)
continue
if not fields[0]:
print('WARNING: line {} bogus, critical fields are empty'.
format(line.strip()),
file=sys.stderr)
continue
pkg = Package()
pkg.name, version = fields[0].rsplit('-', 1)
pkg.version, pkg.origversion = SanitizeVersion(version)
pkg.comment = fields[3]
if fields[11]:
pkg.homepage = fields[11]
# sometimes OWNER variable is used in which case
# there's no MAINTAINER OWNER doesn't get to INDEX
pkg.maintainers = extract_maintainers(fields[5])
pkg.category = fields[6].split(' ')[0]
pkg.extrafields['portname'] = fields[1].split('/')[-1]
pkg.extrafields['origin'] = fields[1]
yield pkg
def Parse(self, path):
result = []
with open(path, 'r', encoding='utf-8') as jsonfile:
for package in json.load(jsonfile)['packages']:
pkg = Package()
pkg.name = package['name']
if package['version'] is None:
print('no version: {}'.format(pkg.name), file=sys.stderr)
continue
pkg.version, _ = SanitizeVersion(package['version'])
pkg.origversion = package['full_version']
pkg.category = package['pkg_section'] or package['section']
pkg.comment = package['description']
pkg.maintainers = GetMaintainers(package['committer'])
if pkg.version == '999':
pkg.SetFlag(PackageFlags.ignore) # XXX: rolling? revisit
result.append(pkg)
return result
def Parse(self, path):
result = []
with open(path, encoding='utf-8') as indexfile:
for line in indexfile:
fields = line.strip().split('|')
if len(fields) != 13:
print(
'WARNING: package {} skipped, incorrect number of fields in INDEX'
.format(fields[0]),
file=sys.stderr)
continue
pkg = Package()
pkg.name, version = fields[0].rsplit('-', 1)
pkg.version, pkg.origversion = SanitizeVersion(version)
pkg.comment = fields[3]
pkg.maintainers = extract_maintainers(fields[5])
pkg.category = fields[6].split(' ')[0]
if fields[12]:
pkg.homepage = fields[12]
path = fields[1].split('/')
pkg.extrafields['portname'] = path[-1]
pkg.extrafields['origin'] = '/'.join(path[-2:])
result.append(pkg)
return result
def Parse(self, path):
result = []
jsondata = None
with open(path, 'r', encoding='utf-8') as jsonfile:
jsondata = json.load(jsonfile)
for packagedata in jsondata['ravenports']:
pkg = Package()
pkg.name = packagedata['namebase']
pkg.version = packagedata['version']
pkg.category = packagedata['keywords'][0]
if 'homepage' in packagedata:
pkg.homepage = packagedata['homepage']
pkg.downloads = packagedata['distfile']
pkg.comment = packagedata['variants'][0]['sdesc']
pkg.extrafields['bucket'] = packagedata['bucket']
pkg.extrafields['variant'] = packagedata['variants'][0]['label']
result.append(pkg)
return result
def Parse(self, path):
result = []
for header in rpm.readHeaderListFromFile(path):
fields = {
key: str(header[key], self.encoding)
if header[key] is not None else None
for key in
['name', 'version', 'release', 'packager', 'group', 'summary']
}
pkg = Package()
pkg.name = fields['name']
pkg.version = fields['version'] # XXX: handle release
if fields['packager']:
pkg.maintainers = extract_maintainers(
fields['packager']) # XXX: may have multiple maintainers
pkg.category = fields['group']
pkg.comment = fields['summary']
result.append(pkg)
return result
def Parse(self, path):
result = []
with open(path, encoding='utf-8') as file:
reader = csv.reader(file, delimiter='|')
for row in reader:
pkg = Package()
pkgname = row[0]
# cut away string suffixws which come after version
match = re.match('(.*?)(-[a-z_]+[0-9]*)+$', pkgname)
if match is not None:
pkgname = match.group(1)
pkg.name, version = SplitPackageNameVersion(pkgname)
pkg.version, pkg.origversion = SanitizeVersion(version)
pkg.comment = row[3]
pkg.maintainers = GetMaintainers(row[5])
pkg.category = row[6].split(' ')[0].strip()
origin = row[1].rsplit(',', 1)[0]
pkg.extrafields['portname'] = origin.split('/')[1]
pkg.extrafields['origin'] = origin
result.append(pkg)
return result
def Parse(self, path):
result = []
with subprocess.Popen([repology.config.TCLSH, self.helperpath, path],
errors='ignore',
stdout=subprocess.PIPE,
universal_newlines=True) as macportsjson:
for pkgdata in json.load(macportsjson.stdout):
pkg = Package()
pkg.name = pkgdata['name']
pkg.version = pkgdata['version']
# drop obsolete ports (see #235)
if 'replaced_by' in pkgdata:
continue
if 'description' in pkgdata:
pkg.comment = pkgdata['description']
if 'homepage' in pkgdata:
pkg.homepage = pkgdata['homepage']
if 'categories' in pkgdata:
pkg.category = pkgdata['categories'].split()[0]
if 'license' in pkgdata:
pkg.licenses = [pkgdata['license']
] # XXX: properly handle braces
if 'maintainers' in pkgdata:
for maintainer in pkgdata['maintainers'].replace(
'{', '').replace('}', '').lower().split():
if maintainer.startswith('@'):
# @foo means github user foo
pkg.maintainers.append(maintainer[1:] + '@github')
elif '@' in maintainer:
# plain email
pkg.maintainers.append(maintainer)
elif ':' in maintainer:
# foo.com:bar means [email protected]
host, user = maintainer.split(':', 1)
pkg.maintainers.append(user + '@' + host)
elif maintainer == 'openmaintainer':
# ignore, this is a flag that minor changes to a port
# are allowed without involving the maintainer
pass
else:
# otherwise it's [email protected]
pkg.maintainers.append(maintainer +
'@macports.org')
pkg.extrafields['portdir'] = pkgdata['portdir']
pkg.extrafields['portname'] = pkgdata['portdir'].split('/')[1]
result.append(pkg)
return result
def Parse(self, path):
packages = []
for moduledir in os.listdir(path):
modulepath = os.path.join(path, moduledir)
cabalpath = None
maxversion = None
for versiondir in os.listdir(modulepath):
if versiondir == 'preferred-versions':
continue
if maxversion is None or version_compare(
versiondir, maxversion) > 0:
maxversion = versiondir
cabalpath = os.path.join(path, moduledir, maxversion,
moduledir + '.cabal')
if maxversion is None:
print('WARNING: cannot determine max version for {}'.format(
moduledir),
file=sys.stderr)
continue
pkg = Package()
pkg.name = moduledir
pkg.version = maxversion
pkg.homepage = 'http://hackage.haskell.org/package/' + moduledir
cabaldata = self.ParseCabal(cabalpath)
if cabaldata['name'] == pkg.name and version_compare(
cabaldata['version'], pkg.version) == 0:
if 'synopsis' in cabaldata and cabaldata['synopsis']:
pkg.comment = cabaldata['synopsis'].strip()
if 'maintainer' in cabaldata:
pkg.maintainers = extract_maintainers(
cabaldata['maintainer'])
if 'license' in cabaldata:
pkg.licenses = [cabaldata['license']]
if 'homepage' in cabaldata and (
cabaldata['homepage'].startswith('http://')
or cabaldata['homepage'].startswith('https://')):
pkg.homepage = cabaldata['homepage']
if 'category' in cabaldata:
pkg.category = cabaldata['category']
else:
print(
'WARNING: cabal data sanity check failed for {}, ignoring cabal data'
.format(cabalpath),
file=sys.stderr)
packages.append(pkg)
return packages
def Parse(self, path):
packages = []
for packagedir in os.listdir(path):
with open(os.path.join(path, packagedir, 'desc'),
'r',
encoding='utf-8') as descfile:
key = None
value = []
data = {}
for line in descfile:
line = line.strip()
if line.startswith('%') and line.endswith('%'):
key = line[1:-1]
value = []
elif line == '':
data[key] = value
else:
value.append(line)
if 'BASE' in data and data['NAME'][0] != data['BASE'][0]:
print('{} skipped, subpackage'.format(data['NAME'][0]),
file=sys.stderr)
continue
pkg = Package()
pkg.name = data['NAME'][0]
pkg.version, pkg.origversion = SanitizeVersion(
data['VERSION'][0])
if 'DESC' in data:
pkg.comment = data['DESC'][0]
if 'URL' in data:
pkg.homepage = data['URL'][0]
if 'LICENSE' in data:
pkg.licenses = data['LICENSE']
pkg.maintainers = sum(
map(extract_maintainers, data['PACKAGER']), [])
if 'GROUPS' in data:
pkg.category = data['GROUPS'][0]
packages.append(pkg)
return packages
def Parse(self, path):
result = []
for category in os.listdir(path):
category_path = os.path.join(path, category)
if not os.path.isdir(category_path):
continue
for package in os.listdir(category_path):
package_path = os.path.join(category_path, package)
if not os.path.isdir(package_path):
continue
for recipe in os.listdir(package_path):
if not recipe.endswith('.recipe'):
continue
pkg = Package()
pkg.name = package
pkg.category = category
# may want to shadow haiku-only ports
#if pkg.category.startswith('haiku-'):
# pkg.shadow = True
# it seems to be guaranteed there's only one hyphen in recipe filename
name, version = recipe[:-7].split('-', 1)
if package.replace('-', '_') != name:
print(
'WARNING: mismatch for package directory and recipe name: {} != {}'
.format(package, name),
file=sys.stderr)
pkg.version = version
# XXX: we rely on the fact that no substitutions happen in these
# variables. That's true as of 2018-05-14.
with open(os.path.join(category_path, package, recipe),
'r',
encoding='utf-8') as recipefile:
match = re.search('^HOMEPAGE="([^"]+)"',
recipefile.read(), re.MULTILINE)
if match:
pkg.homepage = match.group(1).split()[
0] # XXX: use all homepages
result.append(pkg)
return result
def Parse(self, path):
result = []
maintainers = []
with open(os.path.join(path, 'metadata/about.conf'),
'r',
encoding='utf-8') as metadata:
for line in metadata:
if '=' in line:
key, value = map(lambda s: s.strip(), line.split('=', 1))
if key == 'owner':
maintainers = extract_maintainers(value)
packages_path = os.path.join(path, 'packages')
for category in os.listdir(packages_path):
category_path = os.path.join(packages_path, category)
if not os.path.isdir(category_path):
continue
if category == 'virtual' or category == 'metadata':
continue
for package in os.listdir(category_path):
package_path = os.path.join(category_path, package)
if not os.path.isdir(package_path):
continue
for exheres in os.listdir(package_path):
if not exheres.startswith(package +
'-') and not exheres.endswith(
'.exheres-0'):
continue
pkg = Package()
pkg.category = category
pkg.name = package
pkg.version, pkg.origversion = SanitizeVersion(
exheres[len(package) + 1:-10])
pkg.maintainers = maintainers
if pkg.version == 'scm' or pkg.version.endswith('-scm'):
pkg.SetFlag(PackageFlags.rolling)
result.append(pkg)
return result
def Parse(self, path):
result = []
skipped_archs = {}
for entry in self.ParsePackagesEntriesFromXml(path):
pkg = Package()
arch = entry.find('{http://linux.duke.edu/metadata/common}arch').text
if self.allowed_archs and arch not in self.allowed_archs:
skipped_archs[arch] = skipped_archs.get(arch, 0) + 1
continue
pkg.name = entry.find('{http://linux.duke.edu/metadata/common}name').text
version = entry.find('{http://linux.duke.edu/metadata/common}version').attrib['ver']
release = entry.find('{http://linux.duke.edu/metadata/common}version').attrib['rel']
match = re.match('0\\.[0-9]+\\.((?:alpha|beta|rc)[0-9]+)\\.', release)
if match:
# known pre-release schema: https://fedoraproject.org/wiki/Packaging:Versioning#Prerelease_versions
version += '-' + match.group(1)
elif release < '1':
# unknown pre-release schema: https://fedoraproject.org/wiki/Packaging:Versioning#Some_definitions
# most likely a snapshot
pkg.SetFlag(PackageFlags.ignore)
pkg.version, pkg.origversion = SanitizeVersion(version)
# XXX: append origversion with release
pkg.comment = entry.find('{http://linux.duke.edu/metadata/common}summary').text
pkg.homepage = entry.find('{http://linux.duke.edu/metadata/common}url').text
pkg.category = entry.find('{http://linux.duke.edu/metadata/common}format/'
'{http://linux.duke.edu/metadata/rpm}group').text
pkg.licenses.append(entry.find('{http://linux.duke.edu/metadata/common}format/'
'{http://linux.duke.edu/metadata/rpm}license').text)
packager = entry.find('{http://linux.duke.edu/metadata/common}packager').text
if packager:
pkg.maintainers = extract_maintainers(packager)
result.append(pkg)
for arch, numpackages in sorted(skipped_archs.items()):
print('WARNING: skipping {} packages(s) with disallowed architecture {}'.format(numpackages, arch), file=sys.stderr)
return result
def Parse(self, path):
result = []
root = xml.etree.ElementTree.parse(path)
repository = root.find(
'{http://www.openpkg.org/xml-rdf-index/0.9}Repository')
for item in repository.findall(
'{http://www.w3.org/1999/02/22-rdf-syntax-ns#}Description'):
pkg = Package()
pkg.name = item.find(
'{http://www.openpkg.org/xml-rdf-index/0.9}Name').text
pkg.version = item.find(
'{http://www.openpkg.org/xml-rdf-index/0.9}Version').text
pkg.licenses = [
item.find(
'{http://www.openpkg.org/xml-rdf-index/0.9}License').text
]
pkg.comment = item.find(
'{http://www.openpkg.org/xml-rdf-index/0.9}Summary').text
pkg.category = item.find(
'{http://www.openpkg.org/xml-rdf-index/0.9}Group').text
pkg.homepage = item.find(
'{http://www.openpkg.org/xml-rdf-index/0.9}URL').text
for source in item.findall(
'./{http://www.openpkg.org/xml-rdf-index/0.9}Source/{http://www.w3.org/1999/02/22-rdf-syntax-ns#}bag/{http://www.w3.org/1999/02/22-rdf-syntax-ns#}li'
):
text = source.text
if (text.startswith('https://') or text.startswith('http://')
or text.startswith('ftp://')
) and 'openpkg.org' not in text:
pkg.downloads.append(text)
release = item.find(
'{http://www.openpkg.org/xml-rdf-index/0.9}Release').text
if pkg.version.endswith(release):
pkg.SetFlag(PackageFlags.untrusted)
result.append(pkg)
return result
def Parse(self, path):
result = []
root = xml.etree.ElementTree.parse(path)
content = root.find('.//div[@id="mw-content-text"]')
for item in content.findall(
'./div[@style="float:left; width:25.3em; height:8.5em; border:1px solid #ccc; padding:0.1em; margin-bottom: 2em; margin-right: 1em; overflow:hidden"]'
):
pkg = Package()
# name
cell = item.find('./p[1]/b[1]/a[1]')
if cell is None or not cell.text:
continue
pkg.name = cell.text
# version
cell = item.find('./p[2]')
if cell is None or not cell.text:
continue
pkg.version = cell.text
match = re.match('(.*) \(.*\)$', pkg.version)
if match:
pkg.origversion = pkg.version
pkg.version = match.group(1)
# www
for a in item.findall('./p[2]/a'):
if a.text == 'Website':
pkg.homepage = a.attrib['href']
# category
pkg.category = 'games'
result.append(pkg)
return result
def Parse(self, path):
result = []
skipped_archs = {}
for entry in self.ParsePackagesEntriesFromXml(path):
pkg = Package()
arch = entry.find(
'{http://linux.duke.edu/metadata/common}arch').text
if self.allowed_archs and arch not in self.allowed_archs:
skipped_archs[arch] = skipped_archs.get(arch, 0) + 1
continue
pkg.name = entry.find(
'{http://linux.duke.edu/metadata/common}name').text
version = entry.find(
'{http://linux.duke.edu/metadata/common}version').attrib['ver']
pkg.version, pkg.origversion = SanitizeVersion(version)
pkg.comment = entry.find(
'{http://linux.duke.edu/metadata/common}summary').text
pkg.homepage = entry.find(
'{http://linux.duke.edu/metadata/common}url').text
pkg.category = entry.find(
'{http://linux.duke.edu/metadata/common}format/'
'{http://linux.duke.edu/metadata/rpm}group').text
pkg.licenses.append(
entry.find('{http://linux.duke.edu/metadata/common}format/'
'{http://linux.duke.edu/metadata/rpm}license').text)
packager = entry.find(
'{http://linux.duke.edu/metadata/common}packager').text
if packager:
pkg.maintainers = GetMaintainers(packager)
result.append(pkg)
for arch, numpackages in sorted(skipped_archs.items()):
print(
'WARNING: skipping {} packages(s) with disallowed architecture {}'
.format(numpackages, arch),
file=sys.stderr)
return result
def Parse(self, path):
result = []
for root, _, files in os.walk(path):
for filename in files:
if not filename.endswith('.spec'):
continue
with open(os.path.join(root, filename),
encoding='utf-8',
errors='ignore') as specfile:
pkg = Package()
for line in specfile:
line = line.strip()
if '%' in line: # substitudes: ignore
continue
if line.startswith('Name:') and not pkg.name:
pkg.name = line[5:].strip()
elif line.startswith('Version:') and not pkg.version:
pkg.version = line[8:].strip()
elif line.startswith('Url:') and not pkg.homepage:
pkg.homepage = line[4:].strip()
elif line.startswith('License:') and not pkg.license:
pkg.license = line[8:].strip()
elif line.startswith('Group:') and not pkg.category:
pkg.category = line[6:].strip().lower()
elif line.startswith('Summary:') and not pkg.comment:
pkg.comment = line[8:].strip()
if pkg.name is not None and pkg.version is not None:
result.append(pkg)
else:
print(
'WARNING: %s skipped, likely due to parsing problems'
% filename,
file=sys.stderr)
return result
def Parse(self, path):
result = []
root = xml.etree.ElementTree.parse(path)
for application in root.findall('application'):
pkg = Package()
pkg.name = application.find('name').text
pkg.version = application.find('marketversion').text
pkg.licenses.append(application.find('license').text)
pkg.category = application.find('category').text
www = application.find('web').text
if www:
pkg.homepage = www
if pkg.name and pkg.version:
result.append(pkg)
return result
def Parse(self, path):
result = []
for category in os.listdir(path):
category_path = os.path.join(path, category)
if not os.path.isdir(category_path):
continue
for package in os.listdir(category_path):
package_path = os.path.join(category_path, package)
if not os.path.isdir(package_path):
continue
for recipe in os.listdir(package_path):
if not recipe.endswith('.recipe'):
continue
pkg = Package()
pkg.name = package
pkg.category = category
# may want to shadow haiku-only ports
#if pkg.category.startswith('haiku-'):
# pkg.shadow = True
# it seems to be guaranteed there's only one hyphen in recipe filename
name, version = recipe[:-7].split('-', 1)
if package.replace('-', '_') != name:
print(
'WARNING: mismatch for package directory and recipe name: {} != {}'
.format(package, name),
file=sys.stderr)
pkg.version = version
result.append(pkg)
return result
def Parse(self, path):
result = []
with subprocess.Popen([self.helperpath, path],
errors='ignore',
stdout=subprocess.PIPE,
universal_newlines=True) as proc:
for line in proc.stdout:
fields = line.strip().split('|')
pkg = Package()
pkg.name = fields[0]
pkg.version = fields[1]
pkg.maintainers = GetMaintainers(
fields[2]) # XXX: may have multiple maintainers
pkg.category = fields[3]
pkg.comment = fields[4]
result.append(pkg)
return result
def iter_parse(self, path, logger):
result = []
with open(path, encoding='utf-8') as indexfile:
for line in indexfile:
fields = line.split('\t')
if len(fields) != self.numfields:
print(
'WARNING: package {} skipped, incorrect number of fields in INDEX'
.format(fields[0]),
file=sys.stderr)
continue
pkg = Package()
pkg.name = fields[0]
pkg.version = fields[1]
pkg.category = fields[2]
pkg.comment = fields[3]
pkg.homepage = fields[4]
result.append(pkg)
return result
def Parse(self, path):
result = []
with open(path, encoding='utf-8') as file:
pkg = Package()
for line in file:
line = line.strip()
if line == '':
result.append(pkg)
pkg = Package()
elif line.startswith('Package: '):
pkg.name = line[9:]
elif line.startswith('Version: '):
pkg.version, pkg.origversion = SanitizeVersion(line[9:])
elif line.startswith('Maintainer: '):
pkg.maintainers += GetMaintainers(line[12:])
elif line.startswith('Uploaders: '):
pkg.maintainers += GetMaintainers(line[11:])
elif line.startswith('Section: '):
pkg.category = line[9:]
elif line.startswith('Homepage: '):
pkg.homepage = line[10:]
return result
def Parse(self, path):
result = []
for category in os.listdir(path):
if category.startswith('.'):
continue
category_path = os.path.join(path, category)
if not os.path.isdir(category_path):
continue
for package in os.listdir(category_path):
package_path = os.path.join(category_path, package)
if not os.path.isdir(package_path):
continue
info_path = os.path.join(category_path, package,
package + '.info')
if not os.path.isfile(info_path):
print('WARNING: {} does not exist, package skipped'.format(
info_path),
file=sys.stderr)
continue
with open(info_path, encoding='utf-8',
errors='ignore') as infofile:
variables = {}
key = None
total_value = []
for line in infofile:
line = line.strip()
if not line:
continue
value = None
if key: # continued
value = line
else: # new variable
key, value = line.split('=', 1)
value = value.lstrip('"').lstrip()
if value.endswith('\\'): # will continue
total_value.append(value.rstrip('\\').rstrip())
elif value.endswith('"'):
total_value.append(value.rstrip('"').rstrip())
variables[key] = ' '.join(total_value)
key = None
total_value = []
pkg = Package()
pkg.category = category
pkg.name = variables['PRGNAM']
pkg.version = variables['VERSION']
pkg.homepage = variables['HOMEPAGE']
pkg.maintainers = GetMaintainers(variables['EMAIL'])
pkg.downloads = variables['DOWNLOAD'].split()
if pkg.name is not None and pkg.version is not None:
result.append(pkg)
else:
print(
'WARNING: {} skipped, likely due to parsing problems'
.format(info_path),
file=sys.stderr)
return result
def Parse(self, path):
result = []
for category in os.listdir(path):
category_path = os.path.join(path, category)
if not os.path.isdir(category_path):
continue
if category == 'virtual' or category == 'metadata':
continue
for package in os.listdir(category_path):
package_path = os.path.join(category_path, package)
if not os.path.isdir(package_path):
continue
metadata_path = os.path.join(package_path, 'metadata.xml')
# parse maintainers from metadata.xml
# these are the same for all ebuilds for current package
maintainers = []
if os.path.isfile(metadata_path):
with open(metadata_path, 'r',
encoding='utf-8') as metafile:
meta = xml.etree.ElementTree.parse(metafile)
for entry in meta.findall('maintainer'):
email_node = entry.find('email')
if email_node is not None and email_node.text is not None:
maintainers += extract_maintainers(
email_node.text)
for ebuild in os.listdir(package_path):
if not ebuild.endswith('.ebuild'):
continue
pkg = Package()
pkg.name = package
pkg.category = category
pkg.maintainers = maintainers
pkg.version, pkg.origversion = SanitizeVersion(
ebuild[len(package) + 1:-7])
if pkg.version.endswith('9999'):
# ignore versions for snapshots
pkg.SetFlag(PackageFlags.rolling)
metadata_path = os.path.join(
path, 'metadata', 'md5-cache', category,
package + '-' +
(pkg.origversion if pkg.origversion else pkg.version))
if os.path.isfile(metadata_path):
with open(metadata_path, 'r',
encoding='utf-8') as metadata_file:
for line in metadata_file:
line = line.strip()
key, value = line.split('=', 1)
if key == 'DESCRIPTION':
pkg.comment = value
elif key == 'HOMEPAGE':
pkg.homepage = value.split(' ')[
0] # XXX: save all urls
elif key == 'LICENSE':
if '(' in value:
# XXX: conditionals and OR's: need more
# complex parsing and backend support
pkg.licenses.append(value)
else:
pkg.licenses += value.split(' ')
elif key == 'SRC_URI':
pkg.downloads += ParseConditionalExpr(
value)
result.append(pkg)
return result
def Parse(self, path):
result = []
with open(path, encoding='utf-8') as file:
current_data = {}
last_key = None
for line in file:
line = line.rstrip('\n')
# empty line, dump package
if line == '':
pkg = Package()
def GetField(key, type_=str, default=None):
if key in current_data:
if type_ is None or isinstance(
current_data[key], type_):
return current_data[key]
else:
print(
'WARNING: unable to parse field {}'.format(
key),
file=sys.stderr)
return default
else:
return default
pkg.name = GetField('Package')
pkg.version, pkg.origversion = SanitizeVersion(
GetField('Version'))
pkg.maintainers += GetMaintainers(
GetField('Maintainer', default=''))
pkg.maintainers += GetMaintainers(
GetField('Uploaders', default=''))
pkg.category = GetField('Section')
pkg.homepage = GetField('Homepage')
# This is long description
#pkg.comment = GetField('Description', type_=None)
#if isinstance(pkg.comment, list):
# pkg.comment = ' '.join(pkg.comment)
if pkg.name and pkg.version:
result.append(pkg)
else:
print('WARNING: unable to parse package {}'.format(
str(current_data)),
file=sys.stderr)
current_data = {}
last_key = None
continue
# key - value pair
match = re.match('([A-Za-z0-9-]+):(.*?)$', line)
if match:
key = match.group(1)
value = match.group(2).strip()
current_data[key] = value
last_key = key
continue
# continuation of previous key
match = re.match(' (.*)$', line)
if match:
value = match.group(1).strip()
if not isinstance(current_data[last_key], list):
current_data[last_key] = [current_data[last_key]]
current_data[last_key].append(value)
continue
print('WARNING: unable to parse line: {}'.format(line),
file=sys.stderr)
return result
def iter_parse(self, path):
with open(path, 'r', encoding='utf-8') as jsonfile:
for key, packagedata in json.load(jsonfile)['packages'].items():
# see how Nix parses 'derivative' names in
# https://github.com/NixOS src/libexpr/names.cc, DrvName::DrvName
# it just splits on dash followed by non-letter
#
# this doesn't work well on 100% cases, it's an upstream problem
match = re.match('(.+?)-([^a-zA-Z].*)$', packagedata['name'])
if not match:
print('cannot extract version: {}/{}'.format(
key, packagedata['name']),
file=sys.stderr)
continue
pkg = Package()
pkg.name = match.group(1)
pkg.version = match.group(2)
# some exceptions
for prefix in ('75dpi', '100dpi'):
if pkg.version.startswith(prefix):
pkg.name += '-' + prefix
pkg.version = pkg.version[len(prefix) + 1:]
merged = pkg.name + '-' + pkg.version
for pkgname in [
'liblqr-1', 'python2.7-3to2', 'python3.6-3to2'
]:
if merged.startswith(pkgname):
pkg.name = pkgname
pkg.version = merged[len(pkgname) + 1:]
keyparts = key.split('.')
if len(keyparts) > 1:
pkg.category = keyparts[0]
if pkg.name.endswith('-git'):
pkg.name = pkg.name[:-4]
pkg.SetFlag(PackageFlags.ignore)
if re.match('.*20[0-9]{2}-[0-9]{2}-[0-9]{2}', pkg.version):
pkg.SetFlag(PackageFlags.ignore)
if re.match('[0-9a-f]*[a-f][0-9a-f]*$',
pkg.version) and len(pkg.version) >= 7:
print(
'ignoring version which looks like commit hash: {}/{}'.
format(key, packagedata['name']),
file=sys.stderr)
pkg.SetFlag(PackageFlags.ignore)
meta = packagedata['meta']
if 'homepage' in meta:
pkg.homepage = meta['homepage']
if isinstance(
pkg.homepage, list
): # XXX: remove after adding support for homepages array
pkg.homepage = pkg.homepage[0]
if 'description' in meta and meta['description']:
pkg.comment = meta['description'].replace('\n',
' ').strip()
if 'maintainers' in meta:
if not isinstance(meta['maintainers'], list):
print('maintainers is not a list: {}/{}'.format(
key, packagedata['name']),
file=sys.stderr)
else:
pkg.maintainers += list(
extract_nix_maintainers(meta['maintainers']))
if 'license' in meta:
pkg.licenses = extract_nix_licenses(meta['license'])
if 'position' in meta:
posfile, posline = meta['position'].rsplit(':', 1)
pkg.extrafields['posfile'] = posfile
pkg.extrafields['posline'] = posline
yield pkg
