def generate_dependency_changes(self, collection, repo_id, sack, packages, brs):
"""
Generates and persists dependency changes for given list of packages.
Emits package state change events.
"""
# pylint:disable=too-many-locals
results = []
build_group = self.get_build_group(collection, repo_id)
if build_group is None:
raise RuntimeError(
f"No build group found for {collection.name} at repo_id {repo_id}"
)
gen = ((package, self.resolve_dependencies(sack, br, build_group))
for package, br in zip(packages, brs))
queue_size = get_config('dependency.resolver_queue_size')
gen = util.parallel_generator(gen, queue_size=queue_size)
pkgs_done = 0
pkgs_reported = 0
progres_reported_at = time.time()
for package, (resolved, curr_problems, curr_deps) in gen:
changes = []
if curr_deps is not None:
prev_build = self.get_build_for_comparison(package)
if prev_build and prev_build.dependency_keys:
prev_deps = self.dependency_cache.get_by_ids(
prev_build.dependency_keys
)
changes = self.create_dependency_changes(
prev_deps, curr_deps, package_id=package.id,
)
results.append(ResolutionOutput(
package=package,
prev_resolved=package.resolved,
resolved=resolved,
problems=set(curr_problems),
changes=changes,
# last_build_id is used to detect concurrently registered builds
last_build_id=package.last_build_id,
))
if len(results) > get_config('dependency.persist_chunk_size'):
self.persist_resolution_output(results)
results = []
pkgs_done += 1
current_time = time.time()
time_diff = current_time - progres_reported_at
if time_diff > get_config('dependency.perf_report_interval'):
self.log.info(
"Resolution progress: resolved {} packages ({}%) ({} pkgs/min)"
.format(
pkgs_done,
int(pkgs_done / len(packages) * 100.0),
int((pkgs_done - pkgs_reported) / time_diff * 60.0)
)
)
pkgs_reported = pkgs_done
progres_reported_at = current_time
self.persist_resolution_output(results)
def generate_dependency_changes(self, collection, repo_id, sack, packages, brs):
"""
Generates and persists dependency changes for given list of packages.
Emits package state change events.
"""
# pylint:disable=too-many-locals
results = []
build_group = self.get_build_group(collection, repo_id)
if build_group is None:
raise RuntimeError(
f"No build group found for {collection.name} at repo_id {repo_id}"
)
gen = ((package, self.resolve_dependencies(sack, br, build_group))
for package, br in zip(packages, brs))
queue_size = get_config('dependency.resolver_queue_size')
gen = util.parallel_generator(gen, queue_size=queue_size)
pkgs_done = 0
pkgs_reported = 0
progres_reported_at = time.time()
for package, (resolved, curr_problems, curr_deps) in gen:
changes = []
if curr_deps is not None:
prev_build = self.get_build_for_comparison(package)
if prev_build and prev_build.dependency_keys:
prev_deps = self.dependency_cache.get_by_ids(
prev_build.dependency_keys
)
changes = self.create_dependency_changes(
prev_deps, curr_deps, package_id=package.id,
)
results.append(ResolutionOutput(
package=package,
prev_resolved=package.resolved,
resolved=resolved,
problems=set(curr_problems),
changes=changes,
# last_build_id is used to detect concurrently registered builds
last_build_id=package.last_build_id,
))
if len(results) > get_config('dependency.persist_chunk_size'):
self.persist_resolution_output(results)
results = []
pkgs_done += 1
current_time = time.time()
time_diff = current_time - progres_reported_at
if time_diff > get_config('dependency.perf_report_interval'):
self.log.info(
"Resolution progress: resolved {} packages ({}%) ({} pkgs/min)"
.format(
pkgs_done,
int(pkgs_done / len(packages) * 100.0),
int((pkgs_done - pkgs_reported) / time_diff * 60.0)
)
)
pkgs_reported = pkgs_done
progres_reported_at = current_time
self.persist_resolution_output(results)
def generate_repo(self, collection, repo_id):
"""
Generates new dependency changes for requested repo using given
collection. Finishes early when base buildroot is not resolvable.
Updates collection resolution metadata (repo_id, base_resolved) after
finished. Commits data in increments.
"""
total_time.reset()
generate_dependency_changes_time.reset()
total_time.start()
self.log.info("Generating new repo")
repo_descriptor = self.create_repo_descriptor(repo_id)
self.set_descriptor_tags([repo_descriptor])
if not repo_descriptor.build_tag:
self.log.error('Cannot generate repo: {}'.format(repo_id))
self.db.rollback()
return
packages = self.get_packages(collection)
brs = koji_util.get_rpm_requires(self.koji_sessions['secondary'],
[p.srpm_nvra for p in packages])
brs = util.parallel_generator(brs, queue_size=None)
try:
sack = self.repo_cache.get_sack(repo_descriptor)
if not sack:
self.log.error('Cannot generate repo: {}'.format(repo_id))
self.db.rollback()
return
build_group = self.get_build_group(collection)
resolved, base_problems, _ = self.resolve_dependencies(
sack, [], build_group)
resolution_time.stop()
if not resolved:
self.log.info("Build group not resolvable for {}".format(
collection.name))
collection.latest_repo_id = repo_id
collection.latest_repo_resolved = False
self.db.execute(BuildrootProblem.__table__.insert(),
[{
'collection_id': collection.id,
'problem': problem
} for problem in base_problems])
self.db.commit()
return
self.log.info("Resolving dependencies...")
resolution_time.start()
self.generate_dependency_changes(sack, collection, packages, brs,
repo_id)
resolution_time.stop()
finally:
brs.stop()
collection.latest_repo_id = repo_id
collection.latest_repo_resolved = True
self.db.commit()
total_time.stop()
total_time.display()
generate_dependency_changes_time.display()
def process_builds(self):
# pylint: disable=E1101
builds = self.db.query(Build.id, Build.repo_id, Build.real, Build.package_id,
Package.name, Build.version, Build.release,
Package.last_build_id, Package.collection_id)\
.join(Build.package)\
.filter(Build.deps_resolved == None)\
.filter(Build.repo_id != None)\
.order_by(Build.repo_id).all()
descriptors = [self.create_repo_descriptor(build.repo_id) for build in builds]
self.set_descriptor_tags(descriptors)
builds_to_process = []
repos_to_process = []
unavailable_build_ids = []
for descriptor, build in izip(descriptors, builds):
if descriptor.build_tag:
repos_to_process.append(descriptor)
builds_to_process.append(build)
else:
unavailable_build_ids.append(build.id)
if unavailable_build_ids:
self.db.query(Build)\
.filter(Build.id.in_(unavailable_build_ids))\
.update({'deps_resolved': False}, synchronize_session=False)
self.db.commit()
buildrequires = koji_util.get_rpm_requires(self.koji_sessions['secondary'],
[dict(name=b.name, version=b.version,
release=b.release, arch='src')
for b in builds_to_process])
if len(builds) > 100:
buildrequires = util.parallel_generator(buildrequires, queue_size=None)
for repo_descriptor, group in groupby(izip(repos_to_process,
builds_to_process,
buildrequires),
lambda item: item[0]):
sack = self.repo_cache.get_sack(repo_descriptor)
if sack:
for _, build, brs in group:
build_group = self.get_build_group(build.collection_id)
_, _, curr_deps = self.resolve_dependencies(sack, brs, build_group)
try:
self.process_build(sack, build, curr_deps)
self.db.commit()
except (StaleDataError, ObjectDeletedError):
# build deleted concurrently
self.db.rollback()
else:
self.log.info("Repo id=%d not available, skipping",
repo_descriptor.repo_id)
sack = None
self.db.query(Build)\
.filter_by(repo_id=None)\
.filter(Build.state.in_(Build.FINISHED_STATES))\
.update({'deps_resolved': False}, synchronize_session=False)
self.db.commit()
def generate_repo(self, collection, repo_id):
"""
Generates new dependency changes for requested repo using given
collection. Finishes early when base buildroot is not resolvable.
Updates collection resolution metadata (repo_id, base_resolved) after
finished. Commits data in increments.
"""
total_time.reset()
generate_dependency_changes_time.reset()
total_time.start()
self.log.info("Generating new repo")
repo_descriptor = self.create_repo_descriptor(repo_id)
self.set_descriptor_tags([repo_descriptor])
if not repo_descriptor.build_tag:
self.log.error('Cannot generate repo: {}'.format(repo_id))
self.db.rollback()
return
packages = self.get_packages(collection)
brs = koji_util.get_rpm_requires(self.koji_sessions['secondary'],
[p.srpm_nvra for p in packages])
brs = util.parallel_generator(brs, queue_size=None)
try:
sack = self.repo_cache.get_sack(repo_descriptor)
if not sack:
self.log.error('Cannot generate repo: {}'.format(repo_id))
self.db.rollback()
return
build_group = self.get_build_group(collection)
resolved, base_problems, _ = self.resolve_dependencies(sack, [], build_group)
resolution_time.stop()
if not resolved:
self.log.info("Build group not resolvable for {}"
.format(collection.name))
collection.latest_repo_id = repo_id
collection.latest_repo_resolved = False
self.db.execute(BuildrootProblem.__table__.insert(),
[{'collection_id': collection.id, 'problem': problem}
for problem in base_problems])
self.db.commit()
return
self.log.info("Resolving dependencies...")
resolution_time.start()
self.generate_dependency_changes(sack, collection, packages, brs, repo_id)
resolution_time.stop()
finally:
brs.stop()
collection.latest_repo_id = repo_id
collection.latest_repo_resolved = True
self.db.commit()
total_time.stop()
total_time.display()
generate_dependency_changes_time.display()
def run(self):
# pylint: disable=E1101
unprocessed = self.db.query(Build)\
.filter_by(deps_processed=False)\
.filter(Build.repo_id != None)\
.options(joinedload(Build.package))\
.order_by(Build.repo_id).all()
repo_ids = [repo_id for repo_id, _ in
itertools.groupby(unprocessed, lambda build: build.repo_id)]
dead_repos = self.prefetch_repos(repo_ids)
for repo_id, builds in itertools.groupby(unprocessed,
lambda build: build.repo_id):
builds = list(builds)
if repo_id not in dead_repos:
with self.repo_cache.get_sack(repo_id) as sack:
if sack:
brs = util.get_rpm_requires(self.koji_session,
[b.srpm_nvra for b in builds])
if len(builds) > 100:
brs = util.parallel_generator(brs, queue_size=None)
gen = ((build, self.resolve_dependencies(sack, br))
for build, br in itertools.izip(builds, brs))
if len(builds) > 2:
gen = util.parallel_generator(gen, queue_size=10)
for build, result in gen:
_, _, curr_deps = result
self.process_build(sack, build, curr_deps)
else:
self.log.info("Repo id=%d not available, skipping", repo_id)
self.db.query(Build).filter(Build.id.in_([b.id for b in builds]))\
.update({'deps_processed': True},
synchronize_session=False)
self.db.commit()
self.db.query(Build)\
.filter_by(repo_id=None)\
.filter(Build.state.in_(Build.FINISHED_STATES))\
.update({'deps_processed': True}, synchronize_session=False)
def generate_dependency_changes(self, sack, collection, packages, brs,
repo_id):
"""
Generates and persists dependency changes for given list of packages.
Emits package state change events.
"""
resolved_map = {}
problems = []
changes = []
def persist():
state_changes = self.check_package_state_changes(resolved_map)
self.persist_results(resolved_map, problems, changes)
self.db.commit()
self.emit_package_state_changes(state_changes)
build_group = self.get_build_group(collection)
gen = ((package, self.resolve_dependencies(sack, br, build_group))
for package, br in izip(packages, brs))
queue_size = get_config('dependency.resolver_queue_size')
gen = util.parallel_generator(gen, queue_size=queue_size)
for package, result in gen:
generate_dependency_changes_time.start()
resolved_map[package.id], curr_problems, curr_deps = result
problems += [
dict(package_id=package.id, problem=problem)
for problem in sorted(set(curr_problems))
]
if curr_deps is not None:
prev_build = self.get_build_for_comparison(package)
if prev_build and prev_build.dependency_keys:
prev_deps = self.dependency_cache.get_by_ids(
self.db, prev_build.dependency_keys)
create_dependency_changes_time.start()
changes += self.create_dependency_changes(
prev_deps,
curr_deps,
package_id=package.id,
prev_build_id=prev_build.id)
create_dependency_changes_time.stop()
if len(resolved_map) > get_config('dependency.persist_chunk_size'):
persist()
resolved_map = {}
problems = []
changes = []
generate_dependency_changes_time.stop()
persist()
def generate_dependency_changes(self, sack, collection, packages, brs, repo_id):
"""
Generates and persists dependency changes for given list of packages.
Emits package state change events.
"""
resolved_map = {}
problems = []
changes = []
def persist():
state_changes = self.check_package_state_changes(resolved_map)
self.persist_results(resolved_map, problems, changes)
self.db.commit()
self.emit_package_state_changes(state_changes)
build_group = self.get_build_group(collection)
gen = ((package, self.resolve_dependencies(sack, br, build_group))
for package, br in izip(packages, brs))
queue_size = get_config('dependency.resolver_queue_size')
gen = util.parallel_generator(gen, queue_size=queue_size)
for package, result in gen:
generate_dependency_changes_time.start()
resolved_map[package.id], curr_problems, curr_deps = result
problems += [dict(package_id=package.id, problem=problem)
for problem in sorted(set(curr_problems))]
if curr_deps is not None:
prev_build = self.get_build_for_comparison(package)
if prev_build and prev_build.dependency_keys:
prev_deps = self.dependency_cache.get_by_ids(
self.db, prev_build.dependency_keys)
create_dependency_changes_time.start()
changes += self.create_dependency_changes(
prev_deps, curr_deps, package_id=package.id,
prev_build_id=prev_build.id)
create_dependency_changes_time.stop()
if len(resolved_map) > get_config('dependency.persist_chunk_size'):
persist()
resolved_map = {}
problems = []
changes = []
generate_dependency_changes_time.stop()
persist()
def run(self, repo_id):
total_time.reset()
total_time.start()
self.log.info("Generating new repo")
if self.prefetch_repos([repo_id]):
self.db.rollback()
return
packages = self.get_packages(require_build=True)
repo = Repo(repo_id=repo_id)
brs = util.get_rpm_requires(self.koji_session,
[p.srpm_nvra for p in packages])
brs = util.parallel_generator(brs, queue_size=None)
try:
with self.repo_cache.get_sack(repo_id) as sack:
if not sack:
self.log.error('Cannot generate repo: {}'.format(repo_id))
self.db.rollback()
return
repo.base_resolved, base_problems, _ = self.resolve_dependencies(sack, [])
resolution_time.stop()
if not repo.base_resolved:
self.log.info("Build group not resolvable")
self.db.add(repo)
self.db.flush()
self.db.execute(BuildrootProblem.__table__.insert(),
[{'repo_id': repo.repo_id, 'problem': problem}
for problem in base_problems])
self.db.commit()
return
self.log.info("Resolving dependencies...")
resolution_time.start()
self.generate_dependency_changes(sack, packages, brs, repo_id)
resolution_time.stop()
finally:
brs.stop()
self.db.add(repo)
self.db.commit()
total_time.stop()
total_time.display()
def generate_dependency_changes(self, sack, packages, brs, repo_id):
"""
Generates and persists dependency changes for given list of packages.
Emits package state change events.
"""
resolved_map = {}
problems = []
changes = []
def persist():
self.check_package_state_changes(resolved_map)
self.persist_results(resolved_map, problems, changes)
self.db.commit()
gen = ((package, self.resolve_dependencies(sack, br))
for package, br in itertools.izip(packages, brs))
gen = util.parallel_generator(gen, queue_size=10)
for package, result in gen:
resolved_map[package.id], curr_problems, curr_deps = result
problems += [dict(package_id=package.id, problem=problem)
for problem in sorted(set(curr_problems))]
if curr_deps is not None:
last_build = self.get_build_for_comparison(package)
if last_build:
prev_deps = self.get_deps_from_db(last_build.package_id,
last_build.repo_id)
if prev_deps is not None:
create_dependency_changes_time.start()
changes += self.create_dependency_changes(
prev_deps, curr_deps, package_id=package.id,
prev_build_id=last_build.id)
create_dependency_changes_time.stop()
if len(resolved_map) > util.config['dependency']['persist_chunk_size']:
persist()
resolved_map = {}
problems = []
changes = []
persist()
def process_builds(self):
# pylint: disable=E1101
builds = self.db.query(Build.id, Build.repo_id, Build.real, Build.package_id,
Package.name, Build.version, Build.release,
Package.last_build_id, Package.collection_id)\
.join(Build.package)\
.filter(Build.deps_resolved == None)\
.filter(Build.repo_id != None)\
.order_by(Build.repo_id).all()
descriptors = [
self.create_repo_descriptor(build.repo_id) for build in builds
]
self.set_descriptor_tags(descriptors)
builds_to_process = []
repos_to_process = []
unavailable_build_ids = []
for descriptor, build in izip(descriptors, builds):
if descriptor.build_tag:
repos_to_process.append(descriptor)
builds_to_process.append(build)
else:
unavailable_build_ids.append(build.id)
if unavailable_build_ids:
self.db.query(Build)\
.filter(Build.id.in_(unavailable_build_ids))\
.update({'deps_resolved': False}, synchronize_session=False)
self.db.commit()
buildrequires = koji_util.get_rpm_requires(
self.koji_sessions['secondary'], [
dict(name=b.name,
version=b.version,
release=b.release,
arch='src') for b in builds_to_process
])
if len(builds) > 100:
buildrequires = util.parallel_generator(buildrequires,
queue_size=None)
for repo_descriptor, group in groupby(
izip(repos_to_process, builds_to_process, buildrequires),
lambda item: item[0]):
sack = self.repo_cache.get_sack(repo_descriptor)
if sack:
for _, build, brs in group:
build_group = self.get_build_group(build.collection_id)
_, _, curr_deps = self.resolve_dependencies(
sack, brs, build_group)
try:
self.process_build(sack, build, curr_deps)
self.db.commit()
except (StaleDataError, ObjectDeletedError):
# build deleted concurrently
self.db.rollback()
else:
self.log.info("Repo id=%d not available, skipping",
repo_descriptor.repo_id)
sack = None
self.db.query(Build)\
.filter_by(repo_id=None)\
.filter(Build.state.in_(Build.FINISHED_STATES))\
.update({'deps_resolved': False}, synchronize_session=False)
self.db.commit()