def _parse(self, repository, logger):
logger.log('parsing started')
packages = []
for source in repository['sources']:
logger.log('parsing source {} started'.format(source['name']))
packages.extend(
self._iter_parse_source(repository, source,
logger.GetIndented()))
logger.log('parsing source {} complete'.format(source['name']))
logger.log('parsing complete, {} packages, deduplicating'.format(
len(packages)))
packages = PackagesetDeduplicate(packages)
if self.safety_checks and len(packages) < repository['minpackages']:
raise TooLittlePackages(len(packages), repository['minpackages'])
logger.log('parsing complete, {} packages'.format(len(packages)))
return packages
def __Transform(self, packages, transformer, repository, logger):
logger.log('processing started')
sanitylogger = logger.GetIndented()
for package in packages:
package.repo = repository['name']
package.family = repository['family']
if repository.get('shadow', False):
package.shadow = True
if transformer:
transformer.Process(package)
# strip leading project name from flavor
def strip_flavor(flavor):
if flavor.startswith(package.effname + '-'):
return flavor[len(package.effname) + 1:]
return flavor
package.flavors = sorted(set(map(strip_flavor, package.flavors)))
try:
package.CheckSanity(transformed=transformer is not None)
except PackageSanityCheckFailure as err:
sanitylogger.log('sanity error: {}'.format(err),
severity=Logger.ERROR)
raise
except PackageSanityCheckProblem as err:
sanitylogger.log('sanity warning: {}'.format(err),
severity=Logger.WARNING)
package.Normalize()
# XXX: in future, ignored packages will not be dropped here, but
# ignored in summary and version calcualtions, but shown in
# package listing
packages = [
package for package in packages
if not package.HasFlag(PackageFlags.remove)
]
logger.log('processing complete, {} packages, deduplicating'.format(
len(packages)))
packages = PackagesetDeduplicate(packages)
if transformer:
logger.log('processing complete, {} packages, sorting'.format(
len(packages)))
packages = sorted(packages, key=lambda package: package.effname)
logger.log('processing complete, {} packages'.format(len(packages)))
return packages
def iter_parsed(self, reponames=None, logger=NoopLogger()):
def get_sources():
for repository in self.repomgr.GetRepositories(reponames):
sources = self._get_parsed_chunk_paths(repository)
if not sources:
logger.log('parsed packages for repository {} are missing, treating repository as empty'.format(repository['desc']), severity=Logger.ERROR)
yield from sources
with heap_deserializer(get_sources(), lambda package: package.effname) as heap:
for packageset in heap():
packageset = PackagesetDeduplicate(packageset)
yield packageset
def __Parse(self, repository, logger):
packages = []
logger.Log('parsing started')
for source in repository['sources']:
packages += self.__ParseSource(repository, source, logger.GetIndented())
logger.Log('parsing complete, {} packages, deduplicating'.format(len(packages)))
packages = PackagesetDeduplicate(packages)
logger.Log('parsing complete, {} packages'.format(len(packages)))
return packages
def __Parse(self, repository, logger):
packages = []
logger.Log('parsing started')
for source in repository['sources']:
packages += self.__ParseSource(repository, source, logger.GetIndented())
logger.Log('parsing complete, {} packages, deduplicating'.format(len(packages)))
packages = PackagesetDeduplicate(packages)
if self.safety_checks and len(packages) < repository['minpackages']:
raise TooLittlePackages(len(packages), repository['minpackages'])
logger.Log('parsing complete, {} packages'.format(len(packages)))
return packages
def __Transform(self, packages, transformer, repository, logger):
logger.Log('processing started')
sanitylogger = logger.GetIndented()
for package in packages:
package.repo = repository['name']
package.family = repository['family']
if 'shadow' in repository and repository['shadow']:
package.shadow = True
if transformer:
transformer.Process(package)
try:
package.CheckSanity(transformed=transformer is not None)
except PackageSanityCheckFailure as err:
sanitylogger.Log('sanity error: {}'.format(err))
raise
except PackageSanityCheckProblem as err:
sanitylogger.Log('sanity warning: {}'.format(err))
package.Normalize()
# XXX: in future, ignored packages will not be dropped here, but
# ignored in summary and version calcualtions, but shown in
# package listing
packages = [
package for package in packages
if not package.HasFlag(PackageFlags.remove)
]
logger.Log('processing complete, {} packages, deduplicating'.format(
len(packages)))
packages = PackagesetDeduplicate(packages)
if transformer:
logger.Log('processing complete, {} packages, sorting'.format(
len(packages)))
packages = sorted(packages, key=lambda package: package.effname)
logger.Log('processing complete, {} packages'.format(len(packages)))
return packages