def ParsePackage(self, fmri, pkgdata): variables = {} for action in pkgdata['actions']: tokens = shlex.split(action) if not tokens or tokens.pop(0) != 'set': print('WARNING: unrecognized action ' + action, file=sys.stderr) continue key = None value = [] for token in tokens: if token.startswith('name='): key = token[5:] elif token.startswith('value='): value.append(token[6:]) elif token.startswith('last-fmri='): pass else: print('WARNING: unrecognized token ' + token, file=sys.stderr) continue if key and value: variables[key] = value pkg = Package() pkg.extrafields['fmri'] = fmri if 'com.oracle.info.name' in variables: pkg.name = variables['com.oracle.info.name'][0] if 'com.oracle.info.version' in variables: pkg.version = variables['com.oracle.info.version'][0] if 'pkg.summary' in variables: pkg.comment = variables['pkg.summary'][0] if 'info.classification' in variables: pkg.category = variables['info.classification'][0] if pkg.category.startswith('org.opensolaris.category.2008:'): pkg.category = pkg.category.split(':', 1)[1] if 'info.upstream-url' in variables: pkg.homepage = variables['info.upstream-url'][0] if 'info.source-url' in variables: pkg.downloads = variables['info.source-url'] # Regarding comment requirement: there are some packages which lack it, # however for ALL of them is a counterpart with comment and some # additional fields (category, homepage, downloads). Packages without # comment look like legacy, and it's OK and desirable to drop them here if pkg.name and pkg.version and pkg.comment: return pkg return None
def ParsePackage(self, pkgpath, pkgdata): variables = {} for action in pkgdata['actions']: tokens = shlex.split(action) if not tokens or tokens.pop(0) != 'set': print('WARNING: unrecognized action ' + action, file=sys.stderr) continue key = None value = [] for token in tokens: if token.startswith('name='): key = token[5:] elif token.startswith('value='): value.append(token[6:]) elif token.startswith('last-fmri='): pass else: print('WARNING: unrecognized token ' + token, file=sys.stderr) continue if key and value: variables[key] = value pkg = Package() pkg.extrafields['path'] = pkgpath if 'com.oracle.info.name' in variables: pkg.name = variables['com.oracle.info.name'][0] if 'com.oracle.info.version' in variables: pkg.version = variables['com.oracle.info.version'][0] if 'pkg.summary' in variables: pkg.comment = variables['pkg.summary'][0] if 'info.classification' in variables: pkg.category = variables['info.classification'][0] if pkg.category.startswith('org.opensolaris.category.2008:'): pkg.category = pkg.category.split(':', 1)[1] if 'info.upstream-url' in variables: pkg.homepage = variables['info.upstream-url'][0] if 'info.source-url' in variables: pkg.downloads = variables['info.source-url'] if pkg.name and pkg.version: return pkg return None
def Parse(self, path): result = [] with open(path, encoding='utf-8') as indexfile: for line in indexfile: fields = line.strip().split('|') if len(fields) != 12: print('WARNING: package {} skipped, incorrect number of fields in INDEX'.format(fields[0]), file=sys.stderr) continue pkg = Package() pkg.name, version = SplitPackageNameVersion(fields[0]) pkg.version, pkg.origversion = SanitizeVersion(version) pkg.comment = fields[3] if fields[11]: pkg.homepage = fields[11] # sometimes OWNER variable is used in which case # there's no MAINTAINER OWNER doesn't get to INDEX pkg.maintainers = GetMaintainers(fields[5]) pkg.category = fields[6].split(' ')[0] result.append(pkg) return result
def Parse(self, path): result = [] root = xml.etree.ElementTree.parse(path) for entry in root.findall( '{http://linux.duke.edu/metadata/common}package'): pkg = Package() pkg.name = entry.find( '{http://linux.duke.edu/metadata/common}name').text version = entry.find( '{http://linux.duke.edu/metadata/common}version').attrib['ver'] pkg.version, pkg.origversion = SanitizeVersion(version) pkg.comment = entry.find( '{http://linux.duke.edu/metadata/common}summary').text pkg.homepage = entry.find( '{http://linux.duke.edu/metadata/common}url').text pkg.category = entry.find( '{http://linux.duke.edu/metadata/common}format/' '{http://linux.duke.edu/metadata/rpm}group').text pkg.licenses.append( entry.find('{http://linux.duke.edu/metadata/common}format/' '{http://linux.duke.edu/metadata/rpm}license').text) packager = entry.find( '{http://linux.duke.edu/metadata/common}packager').text if packager: pkg.maintainers = GetMaintainers(packager) result.append(pkg) return result
def Parse(self, path): result = [] with open(path, encoding='utf-8') as indexfile: for line in indexfile: fields = line.strip().split('|') if len(fields) != 13: print( 'WARNING: package {} skipped, incorrect number of fields in INDEX' .format(fields[0]), file=sys.stderr) continue pkg = Package() pkg.name, version = SplitPackageNameVersion(fields[0]) pkg.version, pkg.origversion = SanitizeVersion(version) pkg.comment = fields[3] pkg.maintainers = GetMaintainers(fields[5]) pkg.category = fields[6].split(' ')[0] if fields[9]: pkg.homepage = fields[9] result.append(pkg) return result
def iter_parse(self, path): with open(path, encoding='utf-8') as indexfile: for line in indexfile: fields = line.strip().split('|') if len(fields) != 12: print( 'WARNING: package {} skipped, incorrect number of fields in INDEX' .format(fields[0]), file=sys.stderr) continue if not fields[0]: print('WARNING: line {} bogus, critical fields are empty'. format(line.strip()), file=sys.stderr) continue pkg = Package() pkg.name, version = fields[0].rsplit('-', 1) pkg.version, pkg.origversion = SanitizeVersion(version) pkg.comment = fields[3] if fields[11]: pkg.homepage = fields[11] # sometimes OWNER variable is used in which case # there's no MAINTAINER OWNER doesn't get to INDEX pkg.maintainers = extract_maintainers(fields[5]) pkg.category = fields[6].split(' ')[0] pkg.extrafields['portname'] = fields[1].split('/')[-1] pkg.extrafields['origin'] = fields[1] yield pkg
def Parse(self, path): result = [] with open(path, 'r', encoding='utf-8') as jsonfile: for package in json.load(jsonfile)['packages']: pkg = Package() pkg.name = package['name'] if package['version'] is None: print('no version: {}'.format(pkg.name), file=sys.stderr) continue pkg.version, _ = SanitizeVersion(package['version']) pkg.origversion = package['full_version'] pkg.category = package['pkg_section'] or package['section'] pkg.comment = package['description'] pkg.maintainers = GetMaintainers(package['committer']) if pkg.version == '999': pkg.SetFlag(PackageFlags.ignore) # XXX: rolling? revisit result.append(pkg) return result
def Parse(self, path): result = [] with open(path, encoding='utf-8') as indexfile: for line in indexfile: fields = line.strip().split('|') if len(fields) != 13: print( 'WARNING: package {} skipped, incorrect number of fields in INDEX' .format(fields[0]), file=sys.stderr) continue pkg = Package() pkg.name, version = fields[0].rsplit('-', 1) pkg.version, pkg.origversion = SanitizeVersion(version) pkg.comment = fields[3] pkg.maintainers = extract_maintainers(fields[5]) pkg.category = fields[6].split(' ')[0] if fields[12]: pkg.homepage = fields[12] path = fields[1].split('/') pkg.extrafields['portname'] = path[-1] pkg.extrafields['origin'] = '/'.join(path[-2:]) result.append(pkg) return result
def Parse(self, path): result = [] jsondata = None with open(path, 'r', encoding='utf-8') as jsonfile: jsondata = json.load(jsonfile) for packagedata in jsondata['ravenports']: pkg = Package() pkg.name = packagedata['namebase'] pkg.version = packagedata['version'] pkg.category = packagedata['keywords'][0] if 'homepage' in packagedata: pkg.homepage = packagedata['homepage'] pkg.downloads = packagedata['distfile'] pkg.comment = packagedata['variants'][0]['sdesc'] pkg.extrafields['bucket'] = packagedata['bucket'] pkg.extrafields['variant'] = packagedata['variants'][0]['label'] result.append(pkg) return result
def Parse(self, path): result = [] for header in rpm.readHeaderListFromFile(path): fields = { key: str(header[key], self.encoding) if header[key] is not None else None for key in ['name', 'version', 'release', 'packager', 'group', 'summary'] } pkg = Package() pkg.name = fields['name'] pkg.version = fields['version'] # XXX: handle release if fields['packager']: pkg.maintainers = extract_maintainers( fields['packager']) # XXX: may have multiple maintainers pkg.category = fields['group'] pkg.comment = fields['summary'] result.append(pkg) return result
def Parse(self, path): result = [] with open(path, encoding='utf-8') as file: reader = csv.reader(file, delimiter='|') for row in reader: pkg = Package() pkgname = row[0] # cut away string suffixws which come after version match = re.match('(.*?)(-[a-z_]+[0-9]*)+$', pkgname) if match is not None: pkgname = match.group(1) pkg.name, version = SplitPackageNameVersion(pkgname) pkg.version, pkg.origversion = SanitizeVersion(version) pkg.comment = row[3] pkg.maintainers = GetMaintainers(row[5]) pkg.category = row[6].split(' ')[0].strip() origin = row[1].rsplit(',', 1)[0] pkg.extrafields['portname'] = origin.split('/')[1] pkg.extrafields['origin'] = origin result.append(pkg) return result
def Parse(self, path): result = [] with subprocess.Popen([repology.config.TCLSH, self.helperpath, path], errors='ignore', stdout=subprocess.PIPE, universal_newlines=True) as macportsjson: for pkgdata in json.load(macportsjson.stdout): pkg = Package() pkg.name = pkgdata['name'] pkg.version = pkgdata['version'] # drop obsolete ports (see #235) if 'replaced_by' in pkgdata: continue if 'description' in pkgdata: pkg.comment = pkgdata['description'] if 'homepage' in pkgdata: pkg.homepage = pkgdata['homepage'] if 'categories' in pkgdata: pkg.category = pkgdata['categories'].split()[0] if 'license' in pkgdata: pkg.licenses = [pkgdata['license'] ] # XXX: properly handle braces if 'maintainers' in pkgdata: for maintainer in pkgdata['maintainers'].replace( '{', '').replace('}', '').lower().split(): if maintainer.startswith('@'): # @foo means github user foo pkg.maintainers.append(maintainer[1:] + '@github') elif '@' in maintainer: # plain email pkg.maintainers.append(maintainer) elif ':' in maintainer: # foo.com:bar means [email protected] host, user = maintainer.split(':', 1) pkg.maintainers.append(user + '@' + host) elif maintainer == 'openmaintainer': # ignore, this is a flag that minor changes to a port # are allowed without involving the maintainer pass else: # otherwise it's [email protected] pkg.maintainers.append(maintainer + '@macports.org') pkg.extrafields['portdir'] = pkgdata['portdir'] pkg.extrafields['portname'] = pkgdata['portdir'].split('/')[1] result.append(pkg) return result
def Parse(self, path): packages = [] for moduledir in os.listdir(path): modulepath = os.path.join(path, moduledir) cabalpath = None maxversion = None for versiondir in os.listdir(modulepath): if versiondir == 'preferred-versions': continue if maxversion is None or version_compare( versiondir, maxversion) > 0: maxversion = versiondir cabalpath = os.path.join(path, moduledir, maxversion, moduledir + '.cabal') if maxversion is None: print('WARNING: cannot determine max version for {}'.format( moduledir), file=sys.stderr) continue pkg = Package() pkg.name = moduledir pkg.version = maxversion pkg.homepage = 'http://hackage.haskell.org/package/' + moduledir cabaldata = self.ParseCabal(cabalpath) if cabaldata['name'] == pkg.name and version_compare( cabaldata['version'], pkg.version) == 0: if 'synopsis' in cabaldata and cabaldata['synopsis']: pkg.comment = cabaldata['synopsis'].strip() if 'maintainer' in cabaldata: pkg.maintainers = extract_maintainers( cabaldata['maintainer']) if 'license' in cabaldata: pkg.licenses = [cabaldata['license']] if 'homepage' in cabaldata and ( cabaldata['homepage'].startswith('http://') or cabaldata['homepage'].startswith('https://')): pkg.homepage = cabaldata['homepage'] if 'category' in cabaldata: pkg.category = cabaldata['category'] else: print( 'WARNING: cabal data sanity check failed for {}, ignoring cabal data' .format(cabalpath), file=sys.stderr) packages.append(pkg) return packages
def Parse(self, path): packages = [] for packagedir in os.listdir(path): with open(os.path.join(path, packagedir, 'desc'), 'r', encoding='utf-8') as descfile: key = None value = [] data = {} for line in descfile: line = line.strip() if line.startswith('%') and line.endswith('%'): key = line[1:-1] value = [] elif line == '': data[key] = value else: value.append(line) if 'BASE' in data and data['NAME'][0] != data['BASE'][0]: print('{} skipped, subpackage'.format(data['NAME'][0]), file=sys.stderr) continue pkg = Package() pkg.name = data['NAME'][0] pkg.version, pkg.origversion = SanitizeVersion( data['VERSION'][0]) if 'DESC' in data: pkg.comment = data['DESC'][0] if 'URL' in data: pkg.homepage = data['URL'][0] if 'LICENSE' in data: pkg.licenses = data['LICENSE'] pkg.maintainers = sum( map(extract_maintainers, data['PACKAGER']), []) if 'GROUPS' in data: pkg.category = data['GROUPS'][0] packages.append(pkg) return packages
def Parse(self, path): result = [] for category in os.listdir(path): category_path = os.path.join(path, category) if not os.path.isdir(category_path): continue for package in os.listdir(category_path): package_path = os.path.join(category_path, package) if not os.path.isdir(package_path): continue for recipe in os.listdir(package_path): if not recipe.endswith('.recipe'): continue pkg = Package() pkg.name = package pkg.category = category # may want to shadow haiku-only ports #if pkg.category.startswith('haiku-'): # pkg.shadow = True # it seems to be guaranteed there's only one hyphen in recipe filename name, version = recipe[:-7].split('-', 1) if package.replace('-', '_') != name: print( 'WARNING: mismatch for package directory and recipe name: {} != {}' .format(package, name), file=sys.stderr) pkg.version = version # XXX: we rely on the fact that no substitutions happen in these # variables. That's true as of 2018-05-14. with open(os.path.join(category_path, package, recipe), 'r', encoding='utf-8') as recipefile: match = re.search('^HOMEPAGE="([^"]+)"', recipefile.read(), re.MULTILINE) if match: pkg.homepage = match.group(1).split()[ 0] # XXX: use all homepages result.append(pkg) return result
def Parse(self, path): result = [] maintainers = [] with open(os.path.join(path, 'metadata/about.conf'), 'r', encoding='utf-8') as metadata: for line in metadata: if '=' in line: key, value = map(lambda s: s.strip(), line.split('=', 1)) if key == 'owner': maintainers = extract_maintainers(value) packages_path = os.path.join(path, 'packages') for category in os.listdir(packages_path): category_path = os.path.join(packages_path, category) if not os.path.isdir(category_path): continue if category == 'virtual' or category == 'metadata': continue for package in os.listdir(category_path): package_path = os.path.join(category_path, package) if not os.path.isdir(package_path): continue for exheres in os.listdir(package_path): if not exheres.startswith(package + '-') and not exheres.endswith( '.exheres-0'): continue pkg = Package() pkg.category = category pkg.name = package pkg.version, pkg.origversion = SanitizeVersion( exheres[len(package) + 1:-10]) pkg.maintainers = maintainers if pkg.version == 'scm' or pkg.version.endswith('-scm'): pkg.SetFlag(PackageFlags.rolling) result.append(pkg) return result
def Parse(self, path): result = [] skipped_archs = {} for entry in self.ParsePackagesEntriesFromXml(path): pkg = Package() arch = entry.find('{http://linux.duke.edu/metadata/common}arch').text if self.allowed_archs and arch not in self.allowed_archs: skipped_archs[arch] = skipped_archs.get(arch, 0) + 1 continue pkg.name = entry.find('{http://linux.duke.edu/metadata/common}name').text version = entry.find('{http://linux.duke.edu/metadata/common}version').attrib['ver'] release = entry.find('{http://linux.duke.edu/metadata/common}version').attrib['rel'] match = re.match('0\\.[0-9]+\\.((?:alpha|beta|rc)[0-9]+)\\.', release) if match: # known pre-release schema: https://fedoraproject.org/wiki/Packaging:Versioning#Prerelease_versions version += '-' + match.group(1) elif release < '1': # unknown pre-release schema: https://fedoraproject.org/wiki/Packaging:Versioning#Some_definitions # most likely a snapshot pkg.SetFlag(PackageFlags.ignore) pkg.version, pkg.origversion = SanitizeVersion(version) # XXX: append origversion with release pkg.comment = entry.find('{http://linux.duke.edu/metadata/common}summary').text pkg.homepage = entry.find('{http://linux.duke.edu/metadata/common}url').text pkg.category = entry.find('{http://linux.duke.edu/metadata/common}format/' '{http://linux.duke.edu/metadata/rpm}group').text pkg.licenses.append(entry.find('{http://linux.duke.edu/metadata/common}format/' '{http://linux.duke.edu/metadata/rpm}license').text) packager = entry.find('{http://linux.duke.edu/metadata/common}packager').text if packager: pkg.maintainers = extract_maintainers(packager) result.append(pkg) for arch, numpackages in sorted(skipped_archs.items()): print('WARNING: skipping {} packages(s) with disallowed architecture {}'.format(numpackages, arch), file=sys.stderr) return result
def Parse(self, path): result = [] root = xml.etree.ElementTree.parse(path) repository = root.find( '{http://www.openpkg.org/xml-rdf-index/0.9}Repository') for item in repository.findall( '{http://www.w3.org/1999/02/22-rdf-syntax-ns#}Description'): pkg = Package() pkg.name = item.find( '{http://www.openpkg.org/xml-rdf-index/0.9}Name').text pkg.version = item.find( '{http://www.openpkg.org/xml-rdf-index/0.9}Version').text pkg.licenses = [ item.find( '{http://www.openpkg.org/xml-rdf-index/0.9}License').text ] pkg.comment = item.find( '{http://www.openpkg.org/xml-rdf-index/0.9}Summary').text pkg.category = item.find( '{http://www.openpkg.org/xml-rdf-index/0.9}Group').text pkg.homepage = item.find( '{http://www.openpkg.org/xml-rdf-index/0.9}URL').text for source in item.findall( './{http://www.openpkg.org/xml-rdf-index/0.9}Source/{http://www.w3.org/1999/02/22-rdf-syntax-ns#}bag/{http://www.w3.org/1999/02/22-rdf-syntax-ns#}li' ): text = source.text if (text.startswith('https://') or text.startswith('http://') or text.startswith('ftp://') ) and 'openpkg.org' not in text: pkg.downloads.append(text) release = item.find( '{http://www.openpkg.org/xml-rdf-index/0.9}Release').text if pkg.version.endswith(release): pkg.SetFlag(PackageFlags.untrusted) result.append(pkg) return result
def Parse(self, path): result = [] root = xml.etree.ElementTree.parse(path) content = root.find('.//div[@id="mw-content-text"]') for item in content.findall( './div[@style="float:left; width:25.3em; height:8.5em; border:1px solid #ccc; padding:0.1em; margin-bottom: 2em; margin-right: 1em; overflow:hidden"]' ): pkg = Package() # name cell = item.find('./p[1]/b[1]/a[1]') if cell is None or not cell.text: continue pkg.name = cell.text # version cell = item.find('./p[2]') if cell is None or not cell.text: continue pkg.version = cell.text match = re.match('(.*) \(.*\)$', pkg.version) if match: pkg.origversion = pkg.version pkg.version = match.group(1) # www for a in item.findall('./p[2]/a'): if a.text == 'Website': pkg.homepage = a.attrib['href'] # category pkg.category = 'games' result.append(pkg) return result
def Parse(self, path): result = [] skipped_archs = {} for entry in self.ParsePackagesEntriesFromXml(path): pkg = Package() arch = entry.find( '{http://linux.duke.edu/metadata/common}arch').text if self.allowed_archs and arch not in self.allowed_archs: skipped_archs[arch] = skipped_archs.get(arch, 0) + 1 continue pkg.name = entry.find( '{http://linux.duke.edu/metadata/common}name').text version = entry.find( '{http://linux.duke.edu/metadata/common}version').attrib['ver'] pkg.version, pkg.origversion = SanitizeVersion(version) pkg.comment = entry.find( '{http://linux.duke.edu/metadata/common}summary').text pkg.homepage = entry.find( '{http://linux.duke.edu/metadata/common}url').text pkg.category = entry.find( '{http://linux.duke.edu/metadata/common}format/' '{http://linux.duke.edu/metadata/rpm}group').text pkg.licenses.append( entry.find('{http://linux.duke.edu/metadata/common}format/' '{http://linux.duke.edu/metadata/rpm}license').text) packager = entry.find( '{http://linux.duke.edu/metadata/common}packager').text if packager: pkg.maintainers = GetMaintainers(packager) result.append(pkg) for arch, numpackages in sorted(skipped_archs.items()): print( 'WARNING: skipping {} packages(s) with disallowed architecture {}' .format(numpackages, arch), file=sys.stderr) return result
def Parse(self, path): result = [] for root, _, files in os.walk(path): for filename in files: if not filename.endswith('.spec'): continue with open(os.path.join(root, filename), encoding='utf-8', errors='ignore') as specfile: pkg = Package() for line in specfile: line = line.strip() if '%' in line: # substitudes: ignore continue if line.startswith('Name:') and not pkg.name: pkg.name = line[5:].strip() elif line.startswith('Version:') and not pkg.version: pkg.version = line[8:].strip() elif line.startswith('Url:') and not pkg.homepage: pkg.homepage = line[4:].strip() elif line.startswith('License:') and not pkg.license: pkg.license = line[8:].strip() elif line.startswith('Group:') and not pkg.category: pkg.category = line[6:].strip().lower() elif line.startswith('Summary:') and not pkg.comment: pkg.comment = line[8:].strip() if pkg.name is not None and pkg.version is not None: result.append(pkg) else: print( 'WARNING: %s skipped, likely due to parsing problems' % filename, file=sys.stderr) return result
def Parse(self, path): result = [] root = xml.etree.ElementTree.parse(path) for application in root.findall('application'): pkg = Package() pkg.name = application.find('name').text pkg.version = application.find('marketversion').text pkg.licenses.append(application.find('license').text) pkg.category = application.find('category').text www = application.find('web').text if www: pkg.homepage = www if pkg.name and pkg.version: result.append(pkg) return result
def Parse(self, path): result = [] for category in os.listdir(path): category_path = os.path.join(path, category) if not os.path.isdir(category_path): continue for package in os.listdir(category_path): package_path = os.path.join(category_path, package) if not os.path.isdir(package_path): continue for recipe in os.listdir(package_path): if not recipe.endswith('.recipe'): continue pkg = Package() pkg.name = package pkg.category = category # may want to shadow haiku-only ports #if pkg.category.startswith('haiku-'): # pkg.shadow = True # it seems to be guaranteed there's only one hyphen in recipe filename name, version = recipe[:-7].split('-', 1) if package.replace('-', '_') != name: print( 'WARNING: mismatch for package directory and recipe name: {} != {}' .format(package, name), file=sys.stderr) pkg.version = version result.append(pkg) return result
def Parse(self, path): result = [] with subprocess.Popen([self.helperpath, path], errors='ignore', stdout=subprocess.PIPE, universal_newlines=True) as proc: for line in proc.stdout: fields = line.strip().split('|') pkg = Package() pkg.name = fields[0] pkg.version = fields[1] pkg.maintainers = GetMaintainers( fields[2]) # XXX: may have multiple maintainers pkg.category = fields[3] pkg.comment = fields[4] result.append(pkg) return result
def iter_parse(self, path, logger): result = [] with open(path, encoding='utf-8') as indexfile: for line in indexfile: fields = line.split('\t') if len(fields) != self.numfields: print( 'WARNING: package {} skipped, incorrect number of fields in INDEX' .format(fields[0]), file=sys.stderr) continue pkg = Package() pkg.name = fields[0] pkg.version = fields[1] pkg.category = fields[2] pkg.comment = fields[3] pkg.homepage = fields[4] result.append(pkg) return result
def Parse(self, path): result = [] with open(path, encoding='utf-8') as file: pkg = Package() for line in file: line = line.strip() if line == '': result.append(pkg) pkg = Package() elif line.startswith('Package: '): pkg.name = line[9:] elif line.startswith('Version: '): pkg.version, pkg.origversion = SanitizeVersion(line[9:]) elif line.startswith('Maintainer: '): pkg.maintainers += GetMaintainers(line[12:]) elif line.startswith('Uploaders: '): pkg.maintainers += GetMaintainers(line[11:]) elif line.startswith('Section: '): pkg.category = line[9:] elif line.startswith('Homepage: '): pkg.homepage = line[10:] return result
def Parse(self, path): result = [] for category in os.listdir(path): if category.startswith('.'): continue category_path = os.path.join(path, category) if not os.path.isdir(category_path): continue for package in os.listdir(category_path): package_path = os.path.join(category_path, package) if not os.path.isdir(package_path): continue info_path = os.path.join(category_path, package, package + '.info') if not os.path.isfile(info_path): print('WARNING: {} does not exist, package skipped'.format( info_path), file=sys.stderr) continue with open(info_path, encoding='utf-8', errors='ignore') as infofile: variables = {} key = None total_value = [] for line in infofile: line = line.strip() if not line: continue value = None if key: # continued value = line else: # new variable key, value = line.split('=', 1) value = value.lstrip('"').lstrip() if value.endswith('\\'): # will continue total_value.append(value.rstrip('\\').rstrip()) elif value.endswith('"'): total_value.append(value.rstrip('"').rstrip()) variables[key] = ' '.join(total_value) key = None total_value = [] pkg = Package() pkg.category = category pkg.name = variables['PRGNAM'] pkg.version = variables['VERSION'] pkg.homepage = variables['HOMEPAGE'] pkg.maintainers = GetMaintainers(variables['EMAIL']) pkg.downloads = variables['DOWNLOAD'].split() if pkg.name is not None and pkg.version is not None: result.append(pkg) else: print( 'WARNING: {} skipped, likely due to parsing problems' .format(info_path), file=sys.stderr) return result
def Parse(self, path): result = [] for category in os.listdir(path): category_path = os.path.join(path, category) if not os.path.isdir(category_path): continue if category == 'virtual' or category == 'metadata': continue for package in os.listdir(category_path): package_path = os.path.join(category_path, package) if not os.path.isdir(package_path): continue metadata_path = os.path.join(package_path, 'metadata.xml') # parse maintainers from metadata.xml # these are the same for all ebuilds for current package maintainers = [] if os.path.isfile(metadata_path): with open(metadata_path, 'r', encoding='utf-8') as metafile: meta = xml.etree.ElementTree.parse(metafile) for entry in meta.findall('maintainer'): email_node = entry.find('email') if email_node is not None and email_node.text is not None: maintainers += extract_maintainers( email_node.text) for ebuild in os.listdir(package_path): if not ebuild.endswith('.ebuild'): continue pkg = Package() pkg.name = package pkg.category = category pkg.maintainers = maintainers pkg.version, pkg.origversion = SanitizeVersion( ebuild[len(package) + 1:-7]) if pkg.version.endswith('9999'): # ignore versions for snapshots pkg.SetFlag(PackageFlags.rolling) metadata_path = os.path.join( path, 'metadata', 'md5-cache', category, package + '-' + (pkg.origversion if pkg.origversion else pkg.version)) if os.path.isfile(metadata_path): with open(metadata_path, 'r', encoding='utf-8') as metadata_file: for line in metadata_file: line = line.strip() key, value = line.split('=', 1) if key == 'DESCRIPTION': pkg.comment = value elif key == 'HOMEPAGE': pkg.homepage = value.split(' ')[ 0] # XXX: save all urls elif key == 'LICENSE': if '(' in value: # XXX: conditionals and OR's: need more # complex parsing and backend support pkg.licenses.append(value) else: pkg.licenses += value.split(' ') elif key == 'SRC_URI': pkg.downloads += ParseConditionalExpr( value) result.append(pkg) return result
def Parse(self, path): result = [] with open(path, encoding='utf-8') as file: current_data = {} last_key = None for line in file: line = line.rstrip('\n') # empty line, dump package if line == '': pkg = Package() def GetField(key, type_=str, default=None): if key in current_data: if type_ is None or isinstance( current_data[key], type_): return current_data[key] else: print( 'WARNING: unable to parse field {}'.format( key), file=sys.stderr) return default else: return default pkg.name = GetField('Package') pkg.version, pkg.origversion = SanitizeVersion( GetField('Version')) pkg.maintainers += GetMaintainers( GetField('Maintainer', default='')) pkg.maintainers += GetMaintainers( GetField('Uploaders', default='')) pkg.category = GetField('Section') pkg.homepage = GetField('Homepage') # This is long description #pkg.comment = GetField('Description', type_=None) #if isinstance(pkg.comment, list): # pkg.comment = ' '.join(pkg.comment) if pkg.name and pkg.version: result.append(pkg) else: print('WARNING: unable to parse package {}'.format( str(current_data)), file=sys.stderr) current_data = {} last_key = None continue # key - value pair match = re.match('([A-Za-z0-9-]+):(.*?)$', line) if match: key = match.group(1) value = match.group(2).strip() current_data[key] = value last_key = key continue # continuation of previous key match = re.match(' (.*)$', line) if match: value = match.group(1).strip() if not isinstance(current_data[last_key], list): current_data[last_key] = [current_data[last_key]] current_data[last_key].append(value) continue print('WARNING: unable to parse line: {}'.format(line), file=sys.stderr) return result
def iter_parse(self, path): with open(path, 'r', encoding='utf-8') as jsonfile: for key, packagedata in json.load(jsonfile)['packages'].items(): # see how Nix parses 'derivative' names in # https://github.com/NixOS src/libexpr/names.cc, DrvName::DrvName # it just splits on dash followed by non-letter # # this doesn't work well on 100% cases, it's an upstream problem match = re.match('(.+?)-([^a-zA-Z].*)$', packagedata['name']) if not match: print('cannot extract version: {}/{}'.format( key, packagedata['name']), file=sys.stderr) continue pkg = Package() pkg.name = match.group(1) pkg.version = match.group(2) # some exceptions for prefix in ('75dpi', '100dpi'): if pkg.version.startswith(prefix): pkg.name += '-' + prefix pkg.version = pkg.version[len(prefix) + 1:] merged = pkg.name + '-' + pkg.version for pkgname in [ 'liblqr-1', 'python2.7-3to2', 'python3.6-3to2' ]: if merged.startswith(pkgname): pkg.name = pkgname pkg.version = merged[len(pkgname) + 1:] keyparts = key.split('.') if len(keyparts) > 1: pkg.category = keyparts[0] if pkg.name.endswith('-git'): pkg.name = pkg.name[:-4] pkg.SetFlag(PackageFlags.ignore) if re.match('.*20[0-9]{2}-[0-9]{2}-[0-9]{2}', pkg.version): pkg.SetFlag(PackageFlags.ignore) if re.match('[0-9a-f]*[a-f][0-9a-f]*$', pkg.version) and len(pkg.version) >= 7: print( 'ignoring version which looks like commit hash: {}/{}'. format(key, packagedata['name']), file=sys.stderr) pkg.SetFlag(PackageFlags.ignore) meta = packagedata['meta'] if 'homepage' in meta: pkg.homepage = meta['homepage'] if isinstance( pkg.homepage, list ): # XXX: remove after adding support for homepages array pkg.homepage = pkg.homepage[0] if 'description' in meta and meta['description']: pkg.comment = meta['description'].replace('\n', ' ').strip() if 'maintainers' in meta: if not isinstance(meta['maintainers'], list): print('maintainers is not a list: {}/{}'.format( key, packagedata['name']), file=sys.stderr) else: pkg.maintainers += list( extract_nix_maintainers(meta['maintainers'])) if 'license' in meta: pkg.licenses = extract_nix_licenses(meta['license']) if 'position' in meta: posfile, posline = meta['position'].rsplit(':', 1) pkg.extrafields['posfile'] = posfile pkg.extrafields['posline'] = posline yield pkg