class RepositoryProcessor:
def __init__(self,
repomgr: RepositoryManager,
statedir: str,
parseddir: str,
safety_checks: bool = True) -> None:
self.repomgr = repomgr
self.statedir = statedir
self.parseddir = parseddir
self.safety_checks = safety_checks
self.fetcher_factory = ClassFactory('repology.fetchers.fetchers',
superclass=Fetcher)
self.parser_factory = ClassFactory('repology.parsers.parsers',
superclass=Parser)
def _get_state_path(self, repository: RepositoryMetadata) -> str:
return os.path.join(self.statedir, repository['name'] + '.state')
def _get_state_source_path(self, repository: RepositoryMetadata,
source: RepositoryMetadata) -> str:
return os.path.join(self._get_state_path(repository),
source['name'].replace('/', '_'))
def _get_parsed_path(self, repository: RepositoryMetadata) -> str:
return os.path.join(self.parseddir, repository['name'] + '.parsed')
def _get_parsed_chunk_paths(self,
repository: RepositoryMetadata) -> List[str]:
dirpath = self._get_parsed_path(repository)
return [
os.path.join(dirpath, filename) for filename in os.listdir(dirpath)
] if os.path.isdir(dirpath) else []
# source level private methods
def _fetch_source(self, repository: RepositoryMetadata, update: bool,
source: RepositoryMetadata, logger: Logger) -> bool:
if 'fetcher' not in source:
logger.log('fetching source {} not supported'.format(
source['name']))
return False
logger.log('fetching source {} started'.format(source['name']))
fetcher: Fetcher = self.fetcher_factory.spawn_with_known_args(
source['fetcher'], source)
have_changes = fetcher.fetch(self._get_state_source_path(
repository, source),
update=update,
logger=logger.get_indented())
logger.log('fetching source {} complete'.format(source['name']) +
('' if have_changes else ' (no changes)'))
return have_changes
def _iter_parse_source(self, repository: RepositoryMetadata,
source: RepositoryMetadata,
transformer: Optional[PackageTransformer],
logger: Logger) -> Iterator[Package]:
def postprocess_parsed_packages(
packages_iter: Iterable[PackageMaker]) -> Iterator[Package]:
for packagemaker in packages_iter:
# unwrap packagemaker
if not packagemaker.check_sanity(verbose=True):
continue
package = packagemaker.unwrap()
# fill repository-specific fields
package.repo = repository['name']
package.family = repository['family']
if 'subrepo' in source:
package.subrepo = source['subrepo']
if repository.get('shadow', False):
package.shadow = True
if not package.maintainers:
default_maintainer = repository.get(
'default_maintainer', True)
if isinstance(default_maintainer, str):
package.maintainers = [
repository['default_maintainer']
]
elif default_maintainer:
package.maintainers = [
'fallback-mnt-{}@repology'.format(
repository['name'])
]
# transform
if transformer:
transformer.process(package)
# skip removed packages
if package.HasFlag(PackageFlags.remove):
continue
# postprocess
def strip_flavor(flavor: str) -> str:
if flavor.startswith(package.effname + '-'):
return flavor[len(package.effname) + 1:]
return flavor
package.flavors = sorted(
set(map(strip_flavor, package.flavors)))
yield package
return postprocess_parsed_packages(
self.parser_factory.spawn_with_known_args(
source['parser'], source).iter_parse(
self._get_state_source_path(repository, source),
PackageFactory(logger), transformer))
def _iter_parse_all_sources(self, repository: RepositoryMetadata,
transformer: Optional[PackageTransformer],
logger: Logger) -> Iterator[Package]:
for source in repository['sources']:
logger.log('parsing source {} started'.format(source['name']))
yield from self._iter_parse_source(repository, source, transformer,
logger.get_indented())
logger.log('parsing source {} complete'.format(source['name']))
# repository level private methods
def _fetch(self, repository: RepositoryMetadata, update: bool,
logger: Logger) -> bool:
logger.log('fetching started')
if not os.path.isdir(self.statedir):
os.mkdir(self.statedir)
have_changes = False
for source in repository['sources']:
if not os.path.isdir(self._get_state_path(repository)):
os.mkdir(self._get_state_path(repository))
have_changes |= self._fetch_source(repository, update, source,
logger.get_indented())
logger.log('fetching complete' +
('' if have_changes else ' (no changes)'))
return have_changes
def _parse(self, repository: RepositoryMetadata,
transformer: Optional[PackageTransformer],
logger: Logger) -> None:
logger.log('parsing started')
if not os.path.isdir(self.parseddir):
os.mkdir(self.parseddir)
with AtomicDir(self._get_parsed_path(repository)) as state_dir:
serializer = ChunkedSerializer(state_dir.get_path(),
MAX_PACKAGES_PER_CHUNK)
serializer.serialize(
self._iter_parse_all_sources(repository, transformer, logger))
if self.safety_checks and serializer.get_num_packages(
) < repository['minpackages']:
raise TooLittlePackages(serializer.get_num_packages(),
repository['minpackages'])
logger.log('parsing complete, {} packages'.format(
serializer.get_num_packages()))
# public methods
def fetch(self,
reponames: RepositoryNameList,
update: bool = True,
logger: Logger = NoopLogger()) -> bool:
have_changes = False
for repository in self.repomgr.get_repositories(reponames):
have_changes |= self._fetch(repository, update, logger)
return have_changes
def parse(
self,
reponames: RepositoryNameList,
transformer: Optional[PackageTransformer] = None,
logger: Logger = NoopLogger()
) -> None:
for repository in self.repomgr.get_repositories(reponames):
self._parse(repository, transformer, logger)
def iter_parse(
self,
reponames: RepositoryNameList,
transformer: Optional[PackageTransformer] = None,
logger: Logger = NoopLogger()
) -> Iterator[Package]:
for repository in self.repomgr.get_repositories(reponames):
yield from self._iter_parse_all_sources(repository, transformer,
logger)
def iter_parsed(
self,
reponames: Optional[RepositoryNameList] = None,
logger: Logger = NoopLogger()
) -> Iterator[List[Package]]:
def get_sources():
for repository in self.repomgr.get_repositories(reponames):
sources = self._get_parsed_chunk_paths(repository)
if not sources:
logger.log(
'parsed packages for repository {} are missing, treating repository as empty'
.format(repository['desc']),
severity=Logger.ERROR)
yield from sources
with heap_deserializer(get_sources(),
lambda package: package.effname) as heap:
for packageset in heap():
packageset = PackagesetDeduplicate(packageset)
yield packageset
class RepositoryProcessor:
def __init__(self,
repomgr: RepositoryManager,
statedir: str,
parseddir: str,
safety_checks: bool = True) -> None:
self.repomgr = repomgr
self.statedir = statedir
self.parseddir = parseddir
self.safety_checks = safety_checks
self.fetcher_factory = ClassFactory('repology.fetchers.fetchers',
superclass=Fetcher)
self.parser_factory = ClassFactory('repology.parsers.parsers',
superclass=Parser)
def _get_state_path(self, repository: RepositoryMetadata) -> str:
return os.path.join(self.statedir, repository['name'] + '.state')
def _get_state_source_path(self, repository: RepositoryMetadata,
source: RepositoryMetadata) -> str:
return os.path.join(self._get_state_path(repository),
source['name'].replace('/', '_'))
def _get_parsed_path(self, repository: RepositoryMetadata) -> str:
return os.path.join(self.parseddir, repository['name'] + '.parsed')
def _get_parsed_chunk_paths(self,
repository: RepositoryMetadata) -> List[str]:
dirpath = self._get_parsed_path(repository)
return [
os.path.join(dirpath, filename) for filename in os.listdir(dirpath)
] if os.path.isdir(dirpath) else []
# source level private methods
def _fetch_source(self, repository: RepositoryMetadata, update: bool,
source: RepositoryMetadata, logger: Logger) -> bool:
if 'fetcher' not in source:
logger.log('fetching source {} not supported'.format(
source['name']))
return False
logger.log('fetching source {} started'.format(source['name']))
fetcher: Fetcher = self.fetcher_factory.spawn_with_known_args(
source['fetcher'], source)
have_changes = fetcher.fetch(self._get_state_source_path(
repository, source),
update=update,
logger=logger.get_indented())
logger.log('fetching source {} complete'.format(source['name']) +
('' if have_changes else ' (no changes)'))
return have_changes
def _iter_parse_source(self, repository: RepositoryMetadata,
source: RepositoryMetadata,
transformer: Optional[PackageTransformer],
logger: Logger) -> Iterator[Package]:
def postprocess_parsed_packages(
packages_iter: Iterable[PackageMaker]) -> Iterator[Package]:
for packagemaker in packages_iter:
try:
package = packagemaker.spawn(
repo=repository['name'],
family=repository['family'],
subrepo=source.get('subrepo'),
shadow=repository.get('shadow', False),
default_maintainer=repository.get(
'default_maintainer'),
)
except RuntimeError as e:
packagemaker.log(str(e), Logger.ERROR)
raise
# transform
if transformer:
transformer.process(package)
# skip removed packages
if package.has_flag(PackageFlags.REMOVE):
continue
# postprocess flavors
def strip_flavor(flavor: str) -> str:
flavor.removeprefix(package.effname + '-')
return flavor
package.flavors = sorted(
set(map(strip_flavor, package.flavors)))
# XXX: arch is not used anywhere yet, and until #711 is implemented,
# it just introduces package duplicates; it's a crude solution, but
# just drop it here
package.arch = None
yield package
return postprocess_parsed_packages(
self.parser_factory.spawn_with_known_args(
source['parser'], source).iter_parse(
self._get_state_source_path(repository, source),
PackageFactory(logger), transformer))
def _iter_parse_all_sources(self, repository: RepositoryMetadata,
transformer: Optional[PackageTransformer],
logger: Logger) -> Iterator[Package]:
for source in repository['sources']:
logger.log('parsing source {} started'.format(source['name']))
yield from self._iter_parse_source(repository, source, transformer,
logger.get_indented())
logger.log('parsing source {} complete'.format(source['name']))
# repository level private methods
def _fetch(self, repository: RepositoryMetadata, update: bool,
logger: Logger) -> bool:
logger.log('fetching started')
if not os.path.isdir(self.statedir):
os.mkdir(self.statedir)
have_changes = False
for source in repository['sources']:
if not os.path.isdir(self._get_state_path(repository)):
os.mkdir(self._get_state_path(repository))
have_changes |= self._fetch_source(repository, update, source,
logger.get_indented())
logger.log('fetching complete' +
('' if have_changes else ' (no changes)'))
return have_changes
def _parse(self, repository: RepositoryMetadata,
transformer: Optional[PackageTransformer],
logger: Logger) -> None:
logger.log('parsing started')
if not os.path.isdir(self.parseddir):
os.mkdir(self.parseddir)
with AtomicDir(self._get_parsed_path(repository)) as state_dir:
serializer = ChunkedSerializer(state_dir.get_path(),
MAX_PACKAGES_PER_CHUNK)
serializer.serialize(
self._iter_parse_all_sources(repository, transformer, logger))
if self.safety_checks and serializer.get_num_packages(
) < repository['minpackages']:
raise TooLittlePackages(serializer.get_num_packages(),
repository['minpackages'])
logger.log('parsing complete, {} packages'.format(
serializer.get_num_packages()))
# public methods
def fetch(self,
reponames: RepositoryNameList,
update: bool = True,
logger: Logger = NoopLogger()) -> bool:
have_changes = False
for repository in self.repomgr.get_repositories(reponames):
have_changes |= self._fetch(repository, update, logger)
return have_changes
def parse(
self,
reponames: RepositoryNameList,
transformer: Optional[PackageTransformer] = None,
logger: Logger = NoopLogger()
) -> None:
for repository in self.repomgr.get_repositories(reponames):
self._parse(repository, transformer, logger)
def iter_parse(
self,
reponames: RepositoryNameList,
transformer: Optional[PackageTransformer] = None,
logger: Logger = NoopLogger()
) -> Iterator[Package]:
for repository in self.repomgr.get_repositories(reponames):
yield from self._iter_parse_all_sources(repository, transformer,
logger)
def iter_parsed(
self,
reponames: Optional[RepositoryNameList] = None,
logger: Logger = NoopLogger()
) -> Iterator[List[Package]]:
sources: List[str] = []
for repository in self.repomgr.get_repositories(reponames):
repo_sources = self._get_parsed_chunk_paths(repository)
if not repo_sources:
logger.log(
'parsed packages for repository {} are missing, treating repository as empty'
.format(repository['desc']),
severity=Logger.WARNING)
sources.extend(repo_sources)
if sources:
yield from map(packageset_deduplicate, heap_deserialize(sources))
else:
logger.log('no parsed packages found', severity=Logger.ERROR)
class RepositoryProcessor:
def __init__(self, repomgr: RepositoryManager, statedir: str, parseddir: str, safety_checks: bool = True) -> None:
self.repomgr = repomgr
self.statedir = statedir
self.parseddir = parseddir
self.safety_checks = safety_checks
self.fetcher_factory = ClassFactory('repology.fetchers.fetchers', superclass=Fetcher)
self.parser_factory = ClassFactory('repology.parsers.parsers', superclass=Parser)
def _get_state_path(self, repository: Repository) -> str:
return os.path.join(self.statedir, repository.name + '.state')
def _get_state_source_path(self, repository: Repository, source: Source) -> str:
return os.path.join(self._get_state_path(repository), source.name.replace('/', '_'))
def _get_parsed_path(self, repository: Repository) -> str:
return os.path.join(self.parseddir, repository.name + '.parsed')
def _get_parsed_chunk_paths(self, repository: Repository) -> list[str]:
dirpath = self._get_parsed_path(repository)
return [
os.path.join(dirpath, filename)
for filename in os.listdir(dirpath)
] if os.path.isdir(dirpath) else []
# source level private methods
def _fetch_source(self, repository: Repository, update: bool, source: Source, logger: Logger) -> bool:
logger.log(f'fetching source {source.name} started')
fetcher: Fetcher = self.fetcher_factory.spawn_with_known_args(
source.fetcher['class'],
source.fetcher
)
have_changes = fetcher.fetch(
self._get_state_source_path(repository, source),
update=update,
logger=logger.get_indented()
)
logger.log(f'fetching source {source.name} complete' + ('' if have_changes else ' (no changes)'))
return have_changes
def _iter_parse_source(
self,
repository: Repository,
source: Source,
transformer: PackageTransformer | None,
maintainermgr: MaintainerManager | None,
logger: Logger
) -> Iterator[Package]:
def postprocess_parsed_packages(packages_iter: Iterable[PackageMaker]) -> Iterator[Package]:
for packagemaker in packages_iter:
try:
package = packagemaker.spawn(
repo=repository.name,
family=repository.family,
subrepo=source.subrepo,
shadow=repository.shadow,
default_maintainer=repository.default_maintainer,
)
except RuntimeError as e:
packagemaker.log(str(e), Logger.ERROR)
raise
# transform
if transformer:
transformer.process(package)
# skip removed packages
if package.has_flag(PackageFlags.REMOVE):
continue
# postprocess flavors
def strip_flavor(flavor: str) -> str:
flavor.removeprefix(package.effname + '-')
return flavor
package.flavors = sorted(set(map(strip_flavor, package.flavors)))
# add packagelinks
packagelinks: list[tuple[int, str]] = []
for pkglink in source.packagelinks + repository.packagelinks:
link_type = pkglink.type
try:
packagelinks.extend(
(link_type, url)
for url in format_package_links(package, pkglink.url)
)
except Exception as e:
packagemaker.log(f'cannot spawn package link from template "{pkglink.url}": {str(e)}', Logger.ERROR)
raise
if package.links is None:
package.links = packagelinks
else:
seen = set(package.links)
package.links.extend(link for link in packagelinks if link not in seen)
# postprocess maintainers
if maintainermgr and package.maintainers:
package.maintainers = [maintainer for maintainer in package.maintainers if not maintainermgr.is_hidden(maintainer)]
yield package
return postprocess_parsed_packages(
self.parser_factory.spawn_with_known_args(
source.parser['class'],
source.parser
).iter_parse(
self._get_state_source_path(repository, source),
PackageFactory(logger)
)
)
def _iter_parse_all_sources(
self,
repository: Repository,
transformer: PackageTransformer | None,
maintainermgr: MaintainerManager | None,
logger: Logger
) -> Iterator[Package]:
for source in repository.sources:
logger.log(f'parsing source {source.name} started')
yield from self._iter_parse_source(repository, source, transformer, maintainermgr, logger.get_indented())
logger.log(f'parsing source {source.name} complete')
# repository level private methods
def _fetch(self, repository: Repository, update: bool, logger: Logger) -> bool:
logger.log('fetching started')
if not os.path.isdir(self.statedir):
os.mkdir(self.statedir)
have_changes = False
for source in repository.sources:
if not os.path.isdir(self._get_state_path(repository)):
os.mkdir(self._get_state_path(repository))
have_changes |= self._fetch_source(repository, update, source, logger.get_indented())
logger.log('fetching complete' + ('' if have_changes else ' (no changes)'))
return have_changes
def _parse(
self,
repository: Repository,
transformer: PackageTransformer | None,
maintainermgr: MaintainerManager | None,
logger: Logger
) -> None:
logger.log('parsing started')
if not os.path.isdir(self.parseddir):
os.mkdir(self.parseddir)
with AtomicDir(self._get_parsed_path(repository)) as state_dir:
serializer = ChunkedSerializer(state_dir.get_path(), MAX_PACKAGES_PER_CHUNK)
serializer.serialize(self._iter_parse_all_sources(repository, transformer, maintainermgr, logger))
if self.safety_checks and serializer.get_num_packages() < repository.minpackages:
raise TooLittlePackages(serializer.get_num_packages(), repository.minpackages)
logger.log('parsing complete, {} packages'.format(serializer.get_num_packages()))
# public methods
def fetch(self, reponames: RepositoryNameList, update: bool = True, logger: Logger = NoopLogger()) -> bool:
have_changes = False
for repository in self.repomgr.get_repositories(reponames):
have_changes |= self._fetch(repository, update, logger)
return have_changes
def parse(
self,
reponames: RepositoryNameList,
transformer: PackageTransformer | None = None,
maintainermgr: MaintainerManager | None = None,
logger: Logger = NoopLogger()
) -> None:
for repository in self.repomgr.get_repositories(reponames):
self._parse(repository, transformer, maintainermgr, logger)
def iter_parse(
self,
reponames: RepositoryNameList,
transformer: PackageTransformer | None = None,
maintainermgr: MaintainerManager | None = None,
logger: Logger = NoopLogger()
) -> Iterator[Package]:
for repository in self.repomgr.get_repositories(reponames):
yield from self._iter_parse_all_sources(repository, transformer, maintainermgr, logger)
def iter_parsed(self, reponames: RepositoryNameList | None = None, logger: Logger = NoopLogger()) -> Iterator[list[Package]]:
sources: list[str] = []
for repository in self.repomgr.get_repositories(reponames):
repo_sources = self._get_parsed_chunk_paths(repository)
if not repo_sources:
logger.log(f'parsed packages for repository {repository.desc} are missing, treating repository as empty', severity=Logger.WARNING)
sources.extend(repo_sources)
if sources:
yield from map(packageset_deduplicate, heap_deserialize(sources))
else:
logger.log('no parsed packages found', severity=Logger.ERROR)