def refresh_packages(self):
"""
Refresh packages from Koji: add packages not yet known by Koschei
and update blocked flag.
"""
for collection in self.db.query(Collection):
koji_packages = self.koji_sessions['secondary']\
.listPackages(tagID=collection.target_tag, inherited=True)
whitelisted = {
p['package_name']
for p in koji_packages if not p['blocked']
}
packages = self.db.query(Package).filter_by(
collection_id=collection.id).all()
to_update = [
p.id for p in packages if p.blocked == (p.name in whitelisted)
]
if to_update:
self.db.query(Package).filter(Package.id.in_(to_update))\
.update({'blocked': ~Package.blocked}, synchronize_session=False)
self.db.flush()
existing_names = {p.name for p in packages}
to_add = [
p for p in koji_packages
if p['package_name'] not in existing_names
]
if to_add:
for p in to_add:
pkg = Package(name=p['package_name'],
collection_id=collection.id)
pkg.blocked = p['blocked']
pkg.tracked = False
self.db.add(pkg)
self.db.flush()
self.db.expire_all()
def refresh_packages(self):
"""
Refresh packages from Koji: add packages not yet known by Koschei
and update blocked flag.
"""
source_tag = util.koji_config['source_tag']
koji_packages = self.koji_session.listPackages(tagID=source_tag,
inherited=True)
whitelisted = {
p['package_name']
for p in koji_packages if not p['blocked']
}
packages = self.db.query(Package).all()
to_update = [
p.id for p in packages if p.blocked == (p.name in whitelisted)
]
if to_update:
self.db.query(Package).filter(Package.id.in_(to_update))\
.update({'blocked': ~Package.blocked}, synchronize_session=False)
self.db.expire_all()
self.db.flush()
existing_names = {p.name for p in packages}
to_add = [
p for p in koji_packages if p['package_name'] not in existing_names
]
if to_add:
for p in to_add:
pkg = Package(name=p['package_name'])
pkg.blocked = p['blocked']
pkg.tracked = False
self.db.add(pkg)
self.db.flush()
def refresh_packages(self):
"""
Refresh packages from Koji: add packages not yet known by Koschei
and update blocked flag.
"""
for collection in self.db.query(Collection):
koji_packages = self.koji_sessions['secondary']\
.listPackages(tagID=collection.target_tag, inherited=True)
whitelisted = {p['package_name'] for p in koji_packages if not p['blocked']}
packages = self.db.query(Package).filter_by(collection_id=collection.id).all()
to_update = [p.id for p in packages if p.blocked == (p.name in whitelisted)]
if to_update:
self.db.query(Package).filter(Package.id.in_(to_update))\
.update({'blocked': ~Package.blocked}, synchronize_session=False)
self.db.flush()
existing_names = {p.name for p in packages}
to_add = [p for p in koji_packages if p['package_name'] not in existing_names]
if to_add:
for p in to_add:
pkg = Package(name=p['package_name'], collection_id=collection.id)
pkg.blocked = p['blocked']
pkg.tracked = False
self.db.add(pkg)
self.db.flush()
self.db.expire_all()
def state_icons(package_row):
state_array = package_row.states
packages = {}
if len(state_array) > 4:
# format is like this {"(2,t,,3)","(1,t,f,3)"}
state_array = state_array[2:-2].split('","')
for state in state_array:
collection_id, tracked, resolved, build_state = state[1:-1].split(
',')
resolved = None if resolved == '' else (resolved == 't')
build_state = None if build_state == '' else int(build_state)
tracked = tracked == 't'
package = Package(name=package_row.name,
blocked=False,
tracked=tracked,
last_complete_build_state=build_state,
resolved=resolved)
packages[int(collection_id)] = package
out = ""
for collection in g.collections:
package = packages.get(collection.id)
out += '<td>'
if package:
out += '<img src="{icon}" title="{title}"/>'\
.format(icon=package.state_icon, title=package.state_string)
out += '</td>'
return Markup(out)
def package_state(row):
return Package(
tracked=True,
blocked=False,
resolved=row.package_resolved,
last_complete_build_state=row.package_lb_state,
).state_string
def refresh_user_packages(user):
if not user.packages_retrieved:
db = Session.object_session(user)
names = query_users_packages(user.name)
if names is not None:
user.packages_retrieved = True
existing = {
p
for [p] in db.query(Package.name).filter(
Package.name.in_(names))
}
for name in names:
if name not in existing:
pkg = Package(name=name, tracked=False)
db.add(pkg)
db.flush()
packages = db.query(Package.id)\
.filter(Package.name.in_(names)).all()
entries = [{
'user_id': user.id,
'package_id': pkg.id
} for pkg in packages]
db.execute(
delete(UserPackageRelation,
UserPackageRelation.user_id == user.id))
db.execute(insert(UserPackageRelation, entries))
db.commit()
def get_priority_join(self, pkg):
return self.db.query(
Package.current_priority_expression(
collection=Collection,
last_build=Build,
)
).join(Package.collection)\
.join(Package.last_build)\
.filter(Package.id == pkg.id).scalar()
def collection_has_schedulable_package(db, collection):
priority_threshold = get_config('priorities.build_threshold')
priority_expr = Package.current_priority_expression(Collection, Build)
return db.query(Package) \
.join(Package.collection) \
.join(Package.last_build) \
.filter(Package.collection_id == collection.id) \
.filter(priority_expr != None) \
.filter(priority_expr >= priority_threshold) \
.limit(1).count()
def get_priorities(self):
priority_expr = Package.current_priority_expression(
collection=Collection,
last_build=Build,
)
return self.db.query(Package.id, priority_expr)\
.join(Package.collection)\
.join(Package.last_build)\
.filter(priority_expr != None)\
.order_by(priority_expr.desc())\
.all()
def collection_package_view(template, query_fn=None, **template_args):
"""
Single-collection view of a list of packages.
:param template: name of jinja2 template to be used
:param query_fn: optional filter function of query -> filtered_query
:param template_args: additional arguments passed to the template
"""
# should be called only when len(g.current_collections) == 1
collection = g.current_collections[0]
# query current_priority separately as it's not a property of Package
current_prio_expr = Package.current_priority_expression(
collection=collection,
last_build=Build, # package is outerjoined with last_build
)
package_query = db.query(Package, current_prio_expr)\
.filter(Package.collection_id == collection.id)
if query_fn:
package_query = query_fn(package_query.join(BasePackage))
# whether to show untracked packages as well
untracked = request.args.get('untracked') == '1'
# determine correct ORDER BY
order_name = request.args.get('order_by', 'running,state,name')
order_map = {
'name': [Package.name],
'state': [Package.resolved, Reversed(Build.state)],
'running': [Package.last_complete_build_id == Package.last_build_id],
'task_id': [Build.task_id],
'started': [Build.started],
'current_priority': [NullsLastOrder(current_prio_expr)],
}
order_names, order = get_order(order_map, order_name)
if not untracked:
package_query = package_query.filter(Package.tracked == True)
pkgs = package_query.filter(Package.blocked == False)\
.outerjoin(Package.last_build)\
.options(contains_eager(Package.last_build))\
.order_by(*order)
page = pkgs.paginate(packages_per_page)
# monkeypatch the priority as an attribute for ease of use
for pkg, priority in page.items:
pkg.current_priority = priority
# extract only the package from the query results
page.items = [pkg for pkg, _ in page.items]
# monkeypatch visible package groups
populate_package_groups(page.items)
return render_template(template,
packages=page.items,
page=page,
order=order_names,
collection=collection,
**template_args)
def refresh_packages(session):
"""
Refresh package list from Koji. Add packages not yet known by Koschei
and update blocked flag of existing packages.
"""
bases = {
base.name: base
for base in session.db.query(BasePackage.id, BasePackage.name)
}
for collection in session.db.query(Collection.id, Collection.dest_tag,
Collection.secondary_mode):
koji_session = session.secondary_koji_for(collection)
koji_packages = koji_session.listPackages(tagID=collection.dest_tag,
inherited=True)
whitelisted = {
p['package_name']
for p in koji_packages if not p['blocked']
}
packages = session.db.query(Package.id, Package.name, Package.blocked)\
.filter_by(collection_id=collection.id)\
.all()
# Find packages which need to be blocked/unblocked
to_update = [
p.id for p in packages if p.blocked == (p.name in whitelisted)
]
if to_update:
session.db.query(Package).filter(Package.id.in_(to_update))\
.update({'blocked': ~Package.blocked}, synchronize_session=False)
existing_names = {p.name for p in packages}
# Find packages to be added
to_add = []
# Add PackageBases
for pkg_dict in koji_packages:
name = pkg_dict['package_name']
if name not in bases.keys():
base = BasePackage(name=name)
bases[name] = base
to_add.append(base)
session.db.bulk_insert(to_add)
to_add = []
# Add Packages
for pkg_dict in koji_packages:
name = pkg_dict['package_name']
if name not in existing_names:
pkg = Package(name=name,
base_id=bases.get(name).id,
collection_id=collection.id,
tracked=False,
blocked=pkg_dict['blocked'])
to_add.append(pkg)
session.db.bulk_insert(to_add)
session.db.expire_all()
def collection_package_view(template, query_fn=None, **template_args):
"""
Single-collection view of a list of packages.
:param template: name of jinja2 template to be used
:param query_fn: optional filter function of query -> filtered_query
:param template_args: additional arguments passed to the template
"""
# should be called only when len(g.current_collections) == 1
collection = g.current_collections[0]
# query current_priority separately as it's not a property of Package
current_prio_expr = Package.current_priority_expression(
collection=collection,
last_build=Build, # package is outerjoined with last_build
)
package_query = db.query(Package, current_prio_expr)\
.filter(Package.collection_id == collection.id)
if query_fn:
package_query = query_fn(package_query.join(BasePackage))
# whether to show untracked packages as well
untracked = request.args.get('untracked') == '1'
# determine correct ORDER BY
order_name = request.args.get('order_by', 'running,state,name')
order_map = {
'name': [Package.name],
'state': [Package.resolved, Reversed(Build.state)],
'running': [Package.last_complete_build_id == Package.last_build_id],
'task_id': [Build.task_id],
'started': [Build.started],
'current_priority': [NullsLastOrder(current_prio_expr)],
}
order_names, order = get_order(order_map, order_name)
if not untracked:
package_query = package_query.filter(Package.tracked == True)
pkgs = package_query.filter(Package.blocked == False)\
.outerjoin(Package.last_build)\
.options(contains_eager(Package.last_build))\
.order_by(*order)
page = pkgs.paginate(packages_per_page)
# monkeypatch the priority as an attribute for ease of use
for pkg, priority in page.items:
pkg.current_priority = priority
# extract only the package from the query results
page.items = [pkg for pkg, _ in page.items]
# monkeypatch visible package groups
populate_package_groups(page.items)
return render_template(template, packages=page.items, page=page,
order=order_names, collection=collection,
**template_args)
def prepare_package(self, name=None, collection=None, **kwargs):
if 'collection_id' not in kwargs:
if collection is None:
collection = self.collection
if not name:
name = 'p{}'.format(self.pkg_name_counter)
self.pkg_name_counter += 1
base = self.db.query(BasePackage).filter_by(name=name).first()
if not base:
base = BasePackage(name=name)
pkg = Package(name=name, base=base, collection=collection, **kwargs)
self.db.add(pkg)
self.db.commit()
return pkg
def __init__(self, row):
self.name = row.name
self.has_running_build = row.has_running_build
self.base_id = row.base_id
self.packages = []
for collection in g.current_collections:
str_id = str(collection.id)
package = Package(
name=row.name,
blocked=False,
collection=collection,
tracked=getattr(row, 'tracked' + str_id) or False,
last_complete_build_state=getattr(row, 'state' + str_id),
resolved=getattr(row, 'resolved' + str_id),
)
self.packages.append(package)
def package_detail(name, form=None, collection=None):
if not collection:
collection = g.current_collections[0]
g.current_collections = [collection]
base = db.query(BasePackage).filter_by(name=name).first_or_404()
# Get packages for all collections, so that we can display
# "State in other collections" table
packages = {p.collection_id: p for p in db.query(Package).filter_by(base_id=base.id)}
# assign packages to collections in the right order
package = None # the current package, may stay None
all_packages = []
for coll in g.collections:
p = packages.get(coll.id)
if p:
all_packages.append((coll, p))
if coll is collection:
package = p
# prepare group checkboxes
base.global_groups = db.query(PackageGroup)\
.join(PackageGroupRelation)\
.filter(PackageGroupRelation.base_id == base.id)\
.filter(PackageGroup.namespace == None)\
.all()
base.user_groups = []
base.available_groups = []
if g.user:
user_groups = \
db.query(PackageGroup,
func.bool_or(PackageGroupRelation.base_id == base.id))\
.outerjoin(PackageGroupRelation)\
.join(GroupACL)\
.filter(GroupACL.user_id == g.user.id)\
.order_by(PackageGroup.namespace.nullsfirst(), PackageGroup.name)\
.group_by(PackageGroup.id)\
.distinct().all()
base.user_groups = [group for group, checked in user_groups
if checked and group.namespace]
base.available_groups = [group for group, checked in user_groups
if not checked]
# History entry pagination pivot timestamp
# We only display entries older than this
last_seen_ts = request.args.get('last_seen_ts')
if last_seen_ts:
try:
last_seen_ts = int(last_seen_ts)
except ValueError:
abort(400)
def to_ts(col):
return cast(func.extract('EPOCH', col), Integer)
entries = None
if package:
# set current priority
package.current_priority = db.query(
Package.current_priority_expression(
collection=package.collection,
last_build=package.last_build,
)
).filter(Package.id == package.id).scalar()
# prepare history entries - builds and resolution changes
builds = db.query(Build)\
.filter_by(package_id=package.id)\
.filter(to_ts(Build.started) < last_seen_ts
if last_seen_ts else true())\
.options(subqueryload(Build.dependency_changes),
subqueryload(Build.build_arch_tasks))\
.order_by(Build.started.desc())\
.limit(builds_per_page)\
.all()
resolutions = db.query(ResolutionChange)\
.filter_by(package_id=package.id)\
.filter(to_ts(ResolutionChange.timestamp) < last_seen_ts
if last_seen_ts else true())\
.options(joinedload(ResolutionChange.problems))\
.order_by(ResolutionChange.timestamp.desc())\
.limit(builds_per_page)\
.all()
entries = sorted(
builds + resolutions,
key=lambda x: getattr(x, 'started', None) or getattr(x, 'timestamp'),
reverse=True,
)[:builds_per_page]
if not form:
form = forms.EditPackageForm(
tracked=package.tracked,
collection_id=package.collection_id,
manual_priority=package.manual_priority,
arch_override=(package.arch_override or '').split(' '),
skip_resolution=package.skip_resolution,
)
# Note: package might be None
return render_template(
"package-detail.html",
base=base,
package=package,
collection=collection,
form=form,
entries=entries,
all_packages=all_packages,
is_continuation=bool(last_seen_ts),
is_last=len(entries) < builds_per_page if package else True,
)
def package_detail(name, form=None, collection=None):
if not collection:
collection = g.current_collections[0]
g.current_collections = [collection]
base = db.query(BasePackage).filter_by(name=name).first_or_404()
# Get packages for all collections, so that we can display
# "State in other collections" table
packages = {
p.collection_id: p
for p in db.query(Package).filter_by(base_id=base.id)
}
# assign packages to collections in the right order
package = None # the current package, may stay None
all_packages = []
for coll in g.collections:
p = packages.get(coll.id)
if p:
all_packages.append((coll, p))
if coll is collection:
package = p
# prepare group checkboxes
base.global_groups = db.query(PackageGroup)\
.join(PackageGroupRelation)\
.filter(PackageGroupRelation.base_id == base.id)\
.filter(PackageGroup.namespace == None)\
.all()
base.user_groups = []
base.available_groups = []
if g.user:
user_groups = \
db.query(PackageGroup,
func.bool_or(PackageGroupRelation.base_id == base.id))\
.outerjoin(PackageGroupRelation)\
.join(GroupACL)\
.filter(GroupACL.user_id == g.user.id)\
.order_by(PackageGroup.namespace.nullsfirst(), PackageGroup.name)\
.group_by(PackageGroup.id)\
.distinct().all()
base.user_groups = [
group for group, checked in user_groups
if checked and group.namespace
]
base.available_groups = [
group for group, checked in user_groups if not checked
]
# History entry pagination pivot timestamp
# We only display entries older than this
last_seen_ts = request.args.get('last_seen_ts')
if last_seen_ts:
try:
last_seen_ts = int(last_seen_ts)
except ValueError:
abort(400)
def to_ts(col):
return cast(func.extract('EPOCH', col), Integer)
entries = None
if package:
# set current priority
package.current_priority = db.query(
Package.current_priority_expression(
collection=package.collection,
last_build=package.last_build,
)).filter(Package.id == package.id).scalar()
# prepare history entries - builds and resolution changes
builds = db.query(Build)\
.filter_by(package_id=package.id)\
.filter(to_ts(Build.started) < last_seen_ts
if last_seen_ts else true())\
.options(subqueryload(Build.dependency_changes),
subqueryload(Build.build_arch_tasks))\
.order_by(Build.started.desc())\
.limit(builds_per_page)\
.all()
resolutions = db.query(ResolutionChange)\
.filter_by(package_id=package.id)\
.filter(to_ts(ResolutionChange.timestamp) < last_seen_ts
if last_seen_ts else true())\
.options(joinedload(ResolutionChange.problems))\
.order_by(ResolutionChange.timestamp.desc())\
.limit(builds_per_page)\
.all()
entries = sorted(
builds + resolutions,
key=lambda x: getattr(x, 'started', None) or getattr(
x, 'timestamp'),
reverse=True,
)[:builds_per_page]
if not form:
form = forms.EditPackageForm(
tracked=package.tracked,
collection_id=package.collection_id,
manual_priority=package.manual_priority,
arch_override=(package.arch_override or '').split(' '),
skip_resolution=package.skip_resolution,
)
# Note: package might be None
return render_template(
"package-detail.html",
base=base,
package=package,
collection=collection,
form=form,
entries=entries,
all_packages=all_packages,
is_continuation=bool(last_seen_ts),
is_last=len(entries) < builds_per_page if package else True,
)
