def list_packages():
query = (
db.query(
Package.name.label('name'),
Collection.name.label('collection'),
# pylint:disable=no-member
Package.state_string.label('state'),
sql_if(
Build.id != None,
db.query(Build.task_id.label('task_id')).correlate(
Build).as_record()).label(
'last_complete_build')).join(Collection).outerjoin(
Build, (Package.last_complete_build_id == Build.id)
& Build.last_complete).order_by(Package.name))
if 'name' in request.args:
query = query.filter(Package.name.in_(request.args.getlist('name')))
if 'collection' in request.args:
query = query.filter(
Collection.name.in_(request.args.getlist('collection')))
result = (db.query(
literal_column(
"coalesce(array_to_json(array_agg(row_to_json(pkg_query)))::text, '[]')"
).label('q')).select_from(query.subquery('pkg_query')).scalar())
return Response(result, mimetype='application/json')
def list_packages():
"""
Return a list of all packages as JSON. Uses Postgres to generate all the JSON in a
single query.
Optional query parameters:
collection: filter by collection name (list, literal match)
name: filter by package name (list, literal match)
Response format:
[
{
"name": "foo",
"collection": "f29",
"state": "unresolved",
"last_complete_build": {
"task_id": 123
}
},
...
]
"""
query = (
db.query(
Package.name.label('name'),
Collection.name.label('collection'),
# pylint:disable=no-member
Package.state_string.label('state'),
sql_if(
Build.id != None,
db.query(Build.task_id.label('task_id'))
.correlate(Build)
.as_record()
).label('last_complete_build')
)
.join(Collection)
.outerjoin(
Build,
(Package.last_complete_build_id == Build.id) & Build.last_complete
)
.order_by(Package.name)
)
if 'name' in request.args:
query = query.filter(Package.name.in_(request.args.getlist('name')))
if 'collection' in request.args:
query = query.filter(Collection.name.in_(request.args.getlist('collection')))
result = (
db.query(literal_column(
"coalesce(array_to_json(array_agg(row_to_json(pkg_query)))::text, '[]')"
).label('q'))
.select_from(query.subquery('pkg_query'))
.scalar()
)
return Response(result, mimetype='application/json')
def edit_package(name):
"""
Edit package attributes or groups. Everyone can edit attributes, group membership
requires permissions.
"""
form = forms.EditPackageForm()
collection = g.collections_by_id.get(form.collection_id.data) or abort(400)
if not form.validate_or_flash():
return package_detail(name=name, form=form, collection=collection)
package = db.query(Package)\
.filter_by(name=name, collection_id=collection.id)\
.first_or_404()
# Interpret group checkboxes
for key, prev_val in request.form.items():
if key.startswith('group-prev-'):
group = db.query(PackageGroup).get_or_404(int(key[len('group-prev-'):]))
new_val = request.form.get('group-{}'.format(group.id))
if bool(new_val) != (prev_val == 'true'):
if not group.editable:
abort(403)
if new_val:
data.set_group_content(session, group, [package.name], append=True)
else:
data.set_group_content(session, group, [package.name], delete=True)
# Using set_package_attribute to generate audit log events
if form.tracked.data is not None:
data.set_package_attribute(
session, package, 'tracked',
form.tracked.data,
)
if form.manual_priority.data is not None:
data.set_package_attribute(
session, package, 'manual_priority',
form.manual_priority.data,
)
if form.arch_override.data is not None:
data.set_package_attribute(
session, package, 'arch_override',
' '.join(form.arch_override.data) or None,
)
if form.skip_resolution.data is not None:
data.set_package_attribute(
session, package, 'skip_resolution',
form.skip_resolution.data,
)
if package.skip_resolution:
package.resolved = None
db.query(UnappliedChange).filter_by(package_id=package.id).delete()
flash_ack("Package modified")
db.commit()
return redirect(url_for('package_detail', name=package.name) +
"?collection=" + collection.name)
def list_rebuild_requests(username):
user = db.query(User).filter_by(name=username).first_or_404()
requests = db.query(CoprRebuildRequest)\
.filter(CoprRebuildRequest.user_id == user.id)\
.order_by(CoprRebuildRequest.id.desc())\
.all()
form = RebuildRequestForm(
) if can_create_request() and user == g.user else None
return render_template('list-rebuild-requests.html',
user=user,
requests=requests,
form=form)
def statistics():
"""
Show global and per-package statistics about build times etc.
Uses materialized views that are refreshed by backend's polling.
"""
now = db.query(func.now()).scalar()
scalar_stats = db.query(ScalarStats).one()
resource_query = db.query(ResourceConsumptionStats)\
.order_by(ResourceConsumptionStats.time.desc().nullslast())\
.paginate(20)
return render_template("stats.html", now=now, stats=scalar_stats,
packages=resource_query.items,
page=resource_query)
def get_global_notices():
notices = [
n.content
for n in db.query(AdminNotice.content).filter_by(key="global_notice")
]
for collection in g.current_collections:
if collection.latest_repo_resolved is False:
problems = db.query(BuildrootProblem)\
.filter_by(collection_id=collection.id).all()
notices.append("Base buildroot for {} is not installable. "
"Dependency problems:<br/>".format(collection) +
'<br/>'.join((p.problem for p in problems)))
notices = list(map(Markup, notices))
return notices
def statistics():
"""
Show global and per-package statistics about build times etc.
Uses materialized views that are refreshed by backend's polling.
"""
now = db.query(func.now()).scalar()
scalar_stats = db.query(ScalarStats).one()
resource_query = db.query(ResourceConsumptionStats)\
.order_by(ResourceConsumptionStats.time.desc().nullslast())\
.paginate(20)
return render_template("stats.html",
now=now,
stats=scalar_stats,
packages=resource_query.items,
page=resource_query)
def bugreport(name):
"""
Redirect to a pre-filled bugzilla new bug page.
"""
# Package must have last build, so we can have rebuild instructions.
# It doesn't need to be failing, that's up to the user to check.
package = db.query(Package)\
.filter(Package.name == name)\
.filter(Package.blocked == False)\
.filter(Package.last_complete_build_id != None)\
.filter(Package.collection_id == g.current_collections[0].id)\
.options(joinedload(Package.last_complete_build))\
.first() or abort(404)
# Set up variables taht are interpolated into a template specified by configuration
variables = package.srpm_nvra or abort(404)
variables['package'] = package
variables['collection'] = package.collection
# Absolute URL of this instance, for the link back to Koschei
external_url = frontend_config.get('external_url',
request.host_url).rstrip('/')
package_url = url_for('package_detail', name=package.name)
variables['url'] = f'{external_url}{package_url}'
template = get_config('bugreport.template')
bug = {key: template[key].format(**variables) for key in template.keys()}
bug['comment'] = dedent(bug['comment']).strip()
query = urlencode(bug)
bugreport_url = get_config('bugreport.url').format(query=query)
return redirect(bugreport_url)
def bugreport(name):
"""
Redirect to a pre-filled bugzilla new bug page.
"""
# Package must have last build, so we can have rebuild instructions.
# It doesn't need to be failing, that's up to the user to check.
package = db.query(Package)\
.filter(Package.name == name)\
.filter(Package.blocked == False)\
.filter(Package.last_complete_build_id != None)\
.filter(Package.collection_id == g.current_collections[0].id)\
.options(joinedload(Package.last_complete_build))\
.first() or abort(404)
# Set up variables taht are interpolated into a template specified by configuration
variables = package.srpm_nvra or abort(404)
variables['package'] = package
variables['collection'] = package.collection
# Absolute URL of this instance, for the link back to Koschei
external_url = frontend_config.get('external_url', request.host_url).rstrip('/')
package_url = url_for('package_detail', name=package.name)
variables['url'] = f'{external_url}{package_url}'
template = get_config('bugreport.template')
bug = {key: template[key].format(**variables) for key in template.keys()}
bug['comment'] = dedent(bug['comment']).strip()
query = urlencode(bug)
bugreport_url = get_config('bugreport.url').format(query=query)
return redirect(bugreport_url)
def lookup_current_user():
if request.endpoint == 'static':
return
g.user = None
user_name = session.get('user', None)
if user_name:
g.user = db.query(m.User).filter_by(name=user_name).first()
def login():
"""
Acknowledge the logged in user by adding it's name to session.
The login itself must have had already happened in httpd.
Adds the user to the database on first login.
"""
if bypass_login:
identity = "none"
user_name = bypass_login
else:
identity = request.environ.get(user_env) or abort(501)
user_name = re.match(user_re, identity).group(1)
user = db.query(m.User).filter_by(name=user_name).first()
if not user:
user = m.User(name=user_name, admin=bool(bypass_login))
db.add(user)
db.commit()
flash_info('New user "{}" was registered.'.format(user_name))
session['user'] = user_name
flash_ack('Logged in as user "{}" with identity "{}".'
.format(user_name, identity))
if user.admin:
flash_info('You have admin privileges.')
next_url = request.values.get("next", url_for('frontpage'))
return redirect(next_url)
def populate_package_groups(packages):
"""
Adds `visible_groups` field to package objects. It contains a list of PackageGroup
objects that are visible to current user - user is on Group's ACL.
Global groups are visible to everyone.
Ideally, this would be expressed using a SQLA relationship instead, but realtionships
don't allow additional inputs (current user).
:param packages: object with base_id attribute that allows adding attributes
"""
base_map = {}
for package in packages:
package.visible_groups = []
base_map[package.base_id] = package
filter_expr = PackageGroup.namespace == None
if g.user:
filter_expr |= GroupACL.user_id == g.user.id
query = (
db.query(PackageGroupRelation)
.options(contains_eager(PackageGroupRelation.group))
.filter(
PackageGroupRelation.base_id.in_(base_map.keys())
if base_map else false()
)
.join(PackageGroup)
.filter(filter_expr)
.order_by(PackageGroup.namespace, PackageGroup.name)
)
if g.user:
query = query.outerjoin(GroupACL)
for r in query:
base_map[r.base_id].visible_groups.append(r.group)
def populate_package_groups(packages):
"""
Adds `visible_groups` field to package objects. It contains a list of PackageGroup
objects that are visible to current user - user is on Group's ACL.
Global groups are visible to everyone.
Ideally, this would be expressed using a SQLA relationship instead, but realtionships
don't allow additional inputs (current user).
:param packages: object with base_id attribute that allows adding attributes
"""
base_map = {}
for package in packages:
package.visible_groups = []
base_map[package.base_id] = package
filter_expr = PackageGroup.namespace == None
if g.user:
filter_expr |= GroupACL.user_id == g.user.id
query = (db.query(PackageGroupRelation).options(
contains_eager(PackageGroupRelation.group)).filter(
PackageGroupRelation.base_id.in_(base_map.keys()) if base_map else
false()).join(PackageGroup).filter(filter_expr).order_by(
PackageGroup.namespace, PackageGroup.name))
if g.user:
query = query.outerjoin(GroupACL)
for r in query:
base_map[r.base_id].visible_groups.append(r.group)
def confirm_delete_group(name, namespace=None):
group = db.query(PackageGroup)\
.options(joinedload(PackageGroup.packages))\
.filter_by(name=name, namespace=namespace).first_or_404()
if not group.editable:
abort(401)
return render_template('delete-group.html', group=group,
form=forms.EmptyForm())
def confirm_delete_group(name, namespace=None):
group = db.query(PackageGroup)\
.options(joinedload(PackageGroup.packages))\
.filter_by(name=name, namespace=namespace).first_or_404()
if not group.editable:
abort(401)
return render_template('delete-group.html',
group=group,
form=forms.EmptyForm())
def badge(name, collection):
"""
Redirects to a status badge image for use in external pages,
such as GitHub's README.md of a project.
"""
c = g.collections_by_name.get(collection) or abort(404, "Collection not found")
p = db.query(Package).filter_by(name=name, collection_id=c.id).first_or_404()
image = 'images/badges/{}.{}'.format(p.state_string, request.path[-3:])
return redirect(url_for('static', filename=image))
def build_detail(build_id):
# pylint: disable=E1101
build = db.query(Build)\
.options(joinedload(Build.package),
subqueryload(Build.dependency_changes),
subqueryload(Build.build_arch_tasks))\
.filter_by(id=build_id).first_or_404()
return render_template("build-detail.html", build=build,
cancel_form=forms.EmptyForm())
def can_edit_group(group):
"""
Whether the group is editable by the current user.
Available as `editable` property of PackageGroup.
"""
# TODO move to model_additions where it belongs
return g.user and (g.user.admin or db.query(
exists().where((GroupACL.user_id == g.user.id)
& (GroupACL.group_id == group.id))).scalar())
def build_detail(build_id):
# pylint: disable=E1101
build = db.query(Build)\
.options(joinedload(Build.package),
subqueryload(Build.dependency_changes),
subqueryload(Build.build_arch_tasks))\
.filter_by(id=build_id).first_or_404()
return render_template("build-detail.html",
build=build,
cancel_form=forms.EmptyForm())
def lookup_current_user():
"""
If logged in, bind the current user to g.user
"""
if request.endpoint == 'static':
return
g.user = None
user_name = session.get('user', None)
if user_name:
g.user = db.query(m.User).filter_by(name=user_name).first()
def rebuild_request_detail(request_id):
rebuild_request = db.query(CoprRebuildRequest).options(
subqueryload('resolution_changes'),
joinedload('resolution_changes.package'),
subqueryload('rebuilds'),
joinedload('rebuilds.package'),
).get_or_404(request_id)
return render_template('rebuild-request-detail.html',
request=rebuild_request,
form=EditRebuildForm())
def delete_group(name, namespace=None):
group = db.query(PackageGroup)\
.options(joinedload(PackageGroup.packages))\
.filter_by(name=name, namespace=namespace).first_or_404()
# Validate CSRF and permissions
if not forms.EmptyForm().validate_or_flash() or not group.editable:
abort(401)
data.delete_group(session, group)
db.commit()
flash_ack("Group was deleted")
return redirect(url_for('groups_overview'))
def collection_package_view(template, query_fn=None, **template_args):
"""
Single-collection view of a list of packages.
:param template: name of jinja2 template to be used
:param query_fn: optional filter function of query -> filtered_query
:param template_args: additional arguments passed to the template
"""
# should be called only when len(g.current_collections) == 1
collection = g.current_collections[0]
# query current_priority separately as it's not a property of Package
current_prio_expr = Package.current_priority_expression(
collection=collection,
last_build=Build, # package is outerjoined with last_build
)
package_query = db.query(Package, current_prio_expr)\
.filter(Package.collection_id == collection.id)
if query_fn:
package_query = query_fn(package_query.join(BasePackage))
# whether to show untracked packages as well
untracked = request.args.get('untracked') == '1'
# determine correct ORDER BY
order_name = request.args.get('order_by', 'running,state,name')
order_map = {
'name': [Package.name],
'state': [Package.resolved, Reversed(Build.state)],
'running': [Package.last_complete_build_id == Package.last_build_id],
'task_id': [Build.task_id],
'started': [Build.started],
'current_priority': [NullsLastOrder(current_prio_expr)],
}
order_names, order = get_order(order_map, order_name)
if not untracked:
package_query = package_query.filter(Package.tracked == True)
pkgs = package_query.filter(Package.blocked == False)\
.outerjoin(Package.last_build)\
.options(contains_eager(Package.last_build))\
.order_by(*order)
page = pkgs.paginate(packages_per_page)
# monkeypatch the priority as an attribute for ease of use
for pkg, priority in page.items:
pkg.current_priority = priority
# extract only the package from the query results
page.items = [pkg for pkg, _ in page.items]
# monkeypatch visible package groups
populate_package_groups(page.items)
return render_template(template,
packages=page.items,
page=page,
order=order_names,
collection=collection,
**template_args)
def delete_group(name, namespace=None):
group = db.query(PackageGroup)\
.options(joinedload(PackageGroup.packages))\
.filter_by(name=name, namespace=namespace).first_or_404()
# Validate CSRF and permissions
if not forms.EmptyForm().validate_or_flash() or not group.editable:
abort(401)
data.delete_group(session, group)
db.commit()
flash_ack("Group was deleted")
return redirect(url_for('groups_overview'))
def get_global_notices():
"""
Constructs a list of HTML elements representing current global notices taken from the
DB table AdminNotice and also adds a warning if the base buildroot is unresolved.
:return: List of directly renderable items. May be empty.
"""
notices = [
n.content
for n in db.query(AdminNotice.content).filter_by(key="global_notice")
]
for collection in g.current_collections:
if collection.latest_repo_resolved is False:
problems = db.query(BuildrootProblem)\
.filter_by(collection_id=collection.id).all()
notices.append("Base buildroot for {} is not installable. "
"Dependency problems:<br/>".format(collection) +
'<br/>'.join((p.problem for p in problems)))
notices = list(map(Markup, notices))
return notices
def badge(name, collection):
"""
Redirects to a status badge image for use in external pages,
such as GitHub's README.md of a project.
"""
c = g.collections_by_name.get(collection) or abort(404,
"Collection not found")
p = db.query(Package).filter_by(name=name,
collection_id=c.id).first_or_404()
image = 'images/badges/{}.{}'.format(p.state_string, request.path[-3:])
return redirect(url_for('static', filename=image))
def group_detail(name=None, namespace=None):
group = db.query(PackageGroup)\
.filter_by(name=name, namespace=namespace).first_or_404()
owners = ", ".join(owner.name for owner in group.owners)
def query_fn(query):
return query.outerjoin(PackageGroupRelation,
PackageGroupRelation.base_id == BasePackage.id)\
.filter(PackageGroupRelation.group_id == group.id)
return package_view("group-detail.html", query_fn=query_fn,
group=group, owners=owners)
def can_edit_group(group):
"""
Whether the group is editable by the current user.
Available as `editable` property of PackageGroup.
"""
# TODO move to model_additions where it belongs
return g.user and (g.user.admin or
db.query(exists()
.where((GroupACL.user_id == g.user.id) &
(GroupACL.group_id == group.id)))
.scalar())
def edit_rebuild():
form = EditRebuildForm()
if not form.validate_on_submit():
abort(400)
rebuild = db.query(CoprRebuild)\
.filter_by(request_id=form.request_id.data,
package_id=form.package_id.data)\
.first_or_404()
if not rebuild.request.editable:
abort(403)
if form.action.data == 'move-top':
db.query(CoprRebuild)\
.filter(CoprRebuild.request_id == rebuild.request_id)\
.filter(CoprRebuild.state == None)\
.filter(CoprRebuild.order < rebuild.order)\
.update({'order': CoprRebuild.order + 1})
rebuild.order = db.query(func.min(CoprRebuild.order) - 1)\
.filter(CoprRebuild.request_id == rebuild.request_id)\
.filter(CoprRebuild.state == None)\
.scalar()
# Moving to top should ensure the package will be scheduled
rebuild.request.schedule_count += 1
rebuild.request.state = 'in progress'
elif form.action.data == 'remove':
db.query(CoprRebuild)\
.filter(CoprRebuild.request_id == rebuild.request_id)\
.filter(CoprRebuild.order > rebuild.order)\
.update({'order': CoprRebuild.order - 1})
db.delete(rebuild)
db.commit()
return redirect(
url_for('rebuild_request_detail', request_id=rebuild.request_id))
def diff_collections(name1, name2):
"""
Compare two collections and return a list of packages with differing states
packages as JSON. Uses Postgres to generate all the JSON in a single query.
Response format:
[
{
"name": "foo",
"state: {
"f25": "ok",
"f26": "failing",
}
},
...
]
"""
Package1 = aliased(Package)
Package2 = aliased(Package)
collection1 = db.query(Collection).filter_by(name=name1).first_or_404()
collection2 = db.query(Collection).filter_by(name=name2).first_or_404()
query = (
db.query(
BasePackage.name.label('name'),
db.query(
Package1.state_string.label(collection1.name),
Package2.state_string.label(collection2.name),
)
.correlate(Package1, Package2)
.as_record().label('state'),
)
.join(Package1, Package1.base_id == BasePackage.id)
.join(Package2, Package2.base_id == BasePackage.id)
.filter(Package1.state_string != Package2.state_string)
.filter(Package1.collection_id == collection1.id)
.filter(Package2.collection_id == collection2.id)
.order_by(BasePackage.name)
)
return Response(query.json(), mimetype='application/json')
def collection_package_view(template, query_fn=None, **template_args):
"""
Single-collection view of a list of packages.
:param template: name of jinja2 template to be used
:param query_fn: optional filter function of query -> filtered_query
:param template_args: additional arguments passed to the template
"""
# should be called only when len(g.current_collections) == 1
collection = g.current_collections[0]
# query current_priority separately as it's not a property of Package
current_prio_expr = Package.current_priority_expression(
collection=collection,
last_build=Build, # package is outerjoined with last_build
)
package_query = db.query(Package, current_prio_expr)\
.filter(Package.collection_id == collection.id)
if query_fn:
package_query = query_fn(package_query.join(BasePackage))
# whether to show untracked packages as well
untracked = request.args.get('untracked') == '1'
# determine correct ORDER BY
order_name = request.args.get('order_by', 'running,state,name')
order_map = {
'name': [Package.name],
'state': [Package.resolved, Reversed(Build.state)],
'running': [Package.last_complete_build_id == Package.last_build_id],
'task_id': [Build.task_id],
'started': [Build.started],
'current_priority': [NullsLastOrder(current_prio_expr)],
}
order_names, order = get_order(order_map, order_name)
if not untracked:
package_query = package_query.filter(Package.tracked == True)
pkgs = package_query.filter(Package.blocked == False)\
.outerjoin(Package.last_build)\
.options(contains_eager(Package.last_build))\
.order_by(*order)
page = pkgs.paginate(packages_per_page)
# monkeypatch the priority as an attribute for ease of use
for pkg, priority in page.items:
pkg.current_priority = priority
# extract only the package from the query results
page.items = [pkg for pkg, _ in page.items]
# monkeypatch visible package groups
populate_package_groups(page.items)
return render_template(template, packages=page.items, page=page,
order=order_names, collection=collection,
**template_args)
def group_detail(name=None, namespace=None):
group = db.query(PackageGroup)\
.filter_by(name=name, namespace=namespace).first_or_404()
owners = ", ".join(owner.name for owner in group.owners)
def query_fn(query):
return query.outerjoin(PackageGroupRelation,
PackageGroupRelation.base_id == BasePackage.id)\
.filter(PackageGroupRelation.group_id == group.id)
return package_view("group-detail.html",
query_fn=query_fn,
group=group,
owners=owners)
def collection_list():
groups = db.query(CollectionGroup)\
.options(joinedload(CollectionGroup.collections))\
.order_by(CollectionGroup.name)\
.all()
# collections belonging to a category
categorized_ids = {
collection.id for group in groups for collection in group.collections
}
# collections that don't belong to any category and should be displayed in
# "Uncategorized collections" pseudo-category.
uncategorized = [
collection for collection in g.collections if collection.id not in categorized_ids
]
return render_template("list-collections.html", groups=groups,
uncategorized=uncategorized)
def add_packages():
"""
Mark multiple packages as tracked. Optionally add them to a group.
"""
form = forms.AddPackagesForm()
if request.method == 'POST':
if not form.validate_or_flash():
return render_template("add-packages.html", form=form)
names = set(form.packages.data)
try:
collection = [
c for c in g.collections if c.name == form.collection.data
][0]
except IndexError:
abort(404)
try:
added = data.track_packages(session, collection, names)
except data.PackagesDontExist as e:
db.rollback()
flash_nak(str(e))
# frontend doesn't have Koji access, so it needs to rely on backend's polling
flash_nak(
dedent("""
If a package has been just created, it is possible that it
hasn't been propagated to our database yet. In that case,
please, try again later.
"""))
return render_template("add-packages.html", form=form)
if form.group.data:
namespace, name = PackageGroup.parse_name(form.group.data)
group = db.query(PackageGroup)\
.filter_by(namespace=namespace, name=name)\
.first_or_404()
if not group.editable:
abort(400)
data.set_group_content(session, group, names, append=True)
flash_ack("Packages added: {}".format(','.join(p.name for p in added)))
db.commit()
return redirect(request.form.get('next') or url_for('frontpage'))
return render_template("add-packages.html", form=form)
def collection_list():
groups = db.query(CollectionGroup)\
.options(joinedload(CollectionGroup.collections))\
.order_by(CollectionGroup.name)\
.all()
# collections belonging to a category
categorized_ids = {
collection.id
for group in groups for collection in group.collections
}
# collections that don't belong to any category and should be displayed in
# "Uncategorized collections" pseudo-category.
uncategorized = [
collection for collection in g.collections
if collection.id not in categorized_ids
]
return render_template("list-collections.html",
groups=groups,
uncategorized=uncategorized)
def login():
if bypass_login:
identity = "none"
user_name = bypass_login
else:
identity = request.environ.get(user_env) or abort(501)
user_name = re.match(user_re, identity).group(1)
user = db.query(m.User).filter_by(name=user_name).first()
if not user:
user = m.User(name=user_name, admin=bool(bypass_login))
db.add(user)
db.commit()
flash_info('New user "{}" was registered.'.format(user_name))
session['user'] = user_name
flash_ack('Logged in as user "{}" with identity "{}".'
.format(user_name, identity))
if user.admin:
flash_info('You have admin privileges.')
next_url = request.values.get("next", url_for('frontpage'))
return redirect(next_url)
def add_packages():
"""
Mark multiple packages as tracked. Optionally add them to a group.
"""
form = forms.AddPackagesForm()
if request.method == 'POST':
if not form.validate_or_flash():
return render_template("add-packages.html", form=form)
names = set(form.packages.data)
try:
collection = [c for c in g.collections
if c.name == form.collection.data][0]
except IndexError:
abort(404)
try:
added = data.track_packages(session, collection, names)
except data.PackagesDontExist as e:
db.rollback()
flash_nak(str(e))
# frontend doesn't have Koji access, so it needs to rely on backend's polling
flash_nak(dedent("""
If a package has been just created, it is possible that it
hasn't been propagated to our database yet. In that case,
please, try again later.
"""))
return render_template("add-packages.html", form=form)
if form.group.data:
namespace, name = PackageGroup.parse_name(form.group.data)
group = db.query(PackageGroup)\
.filter_by(namespace=namespace, name=name)\
.first_or_404()
if not group.editable:
abort(400)
data.set_group_content(session, group, names, append=True)
flash_ack("Packages added: {}".format(','.join(p.name for p in added)))
db.commit()
return redirect(request.form.get('next') or url_for('frontpage'))
return render_template("add-packages.html", form=form)
def cancel_build(build_id):
"""
Requests cancellation of a build by marking the build in the DB.
Doesn't do the cancellation itself, as frontend doens't have access to Koji.
Backend polls for the atttribute.
"""
if not g.user.admin:
abort(403)
build = db.query(Build).filter_by(id=build_id).first_or_404()
if forms.EmptyForm().validate_or_flash():
if build.state != Build.RUNNING:
flash_nak("Only running builds can be canceled.")
elif build.cancel_requested:
flash_nak("Build already has pending cancelation request.")
else:
flash_ack("Cancelation request sent.")
session.log_user_action(
"Build (id={build.id}, task_id={build.task_id}) cancelation requested"
.format(build=build))
build.cancel_requested = True
db.commit()
return redirect(url_for('package_detail', name=build.package.name))
def cancel_build(build_id):
"""
Requests cancellation of a build by marking the build in the DB.
Doesn't do the cancellation itself, as frontend doens't have access to Koji.
Backend polls for the atttribute.
"""
if not g.user.admin:
abort(403)
build = db.query(Build).filter_by(id=build_id).first_or_404()
if forms.EmptyForm().validate_or_flash():
if build.state != Build.RUNNING:
flash_nak("Only running builds can be canceled.")
elif build.cancel_requested:
flash_nak("Build already has pending cancelation request.")
else:
flash_ack("Cancelation request sent.")
session.log_user_action(
"Build (id={build.id}, task_id={build.task_id}) cancelation requested"
.format(build=build)
)
build.cancel_requested = True
db.commit()
return redirect(url_for('package_detail', name=build.package.name))
def groups_overview():
groups = db.query(PackageGroup)\
.options(undefer(PackageGroup.package_count))\
.filter_by(namespace=None)\
.order_by(PackageGroup.name).all()
return render_template("list-groups.html", groups=groups)
def unified_package_view(template, query_fn=None, **template_args):
"""
View of package in multiple collections at the same time.
:param template: name of jinja2 template to be used
:param query_fn: optional filter function of query -> filtered_query
:param template_args: additional arguments passed to the template
"""
# whether to include untracked packages as well
untracked = request.args.get('untracked') == '1'
order_name = request.args.get('order_by', 'running,failing,name')
# all of the following variables are iteratively built in the follwoing loops (fold)
# columns queried for each collection
exprs = []
# aliased package tables for each collection
tables = []
# whether the package has a running build in any of the collections
running_build_expr = false()
# whether the package has a failed build or is unresolved in any of the collections
failing_expr = false()
# whether the package is tracked in any of the collections
tracked_expr = false()
order_map = {'name': [BasePackage.name]}
# All collections are queried in single query that has variable number of tables and
# columns. For each collection there's an additional joined aliased table.
# Now, build the aliased tables (no joins yet), their columns and boolean properties
for collection in g.current_collections:
table = aliased(Package)
tables.append(table)
exprs.append(table.tracked.label('tracked{}'.format(collection.id)))
exprs.append(table.resolved.label('resolved{}'.format(collection.id)))
exprs.append(table.last_complete_build_state
.label('state{}'.format(collection.id)))
running_build_expr |= table.last_build_id != table.last_complete_build_id
failing_expr |= table.last_complete_build_state == Build.FAILED
failing_expr |= table.resolved == False
tracked_expr |= table.tracked == True
running_build_expr = func.coalesce(running_build_expr, false())
failing_expr = func.coalesce(failing_expr, false())
# Declare query columns
query = db.query(BasePackage.name, BasePackage.id.label('base_id'),
running_build_expr.label('has_running_build'),
*exprs).filter(~BasePackage.all_blocked)
if not untracked:
# TODO I'm not sure if this is necessary.
# We filter by "tracked" in JOIN's ON exprs
query = query.filter(tracked_expr)
# Build joins and collection-specific order expressions
for collection, table in zip(g.current_collections, tables):
on_expr = BasePackage.id == table.base_id
on_expr &= table.collection_id == collection.id
on_expr &= ~table.blocked
if not untracked:
on_expr &= table.tracked
query = query.outerjoin(table, on_expr)
order_map['state-' + collection.name] = \
[table.resolved, Reversed(table.last_complete_build_state)]
if query_fn:
query = query_fn(query)
order_map['running'] = [Reversed(running_build_expr)]
order_map['failing'] = [Reversed(failing_expr)]
order_names, order = get_order(order_map, order_name)
page = query.order_by(*order).paginate(packages_per_page)
page.items = list(map(UnifiedPackage, page.items))
# monkey-patch visible groups on the row
populate_package_groups(page.items)
return render_template(template, packages=page.items, page=page,
order=order_names, collection=None, **template_args)
def affected_by(dep_name):
"""
Display which packages are possibly affected by given dependency change.
"""
if len(g.current_collections) != 1:
abort(400)
collection = g.current_collections[0]
try:
evr1 = RpmEVR(
int(request.args['epoch1']),
request.args['version1'],
request.args['release1']
)
evr2 = RpmEVR(
int(request.args['epoch2']),
request.args['version2'],
request.args['release2']
)
except (KeyError, ValueError):
abort(400)
# Dependencies in the evr1 to evr2 interval
# Note that evr comparisons are overloaded custom comparators that invoke RPM-correct
# comparisons implemented in rpmvercmp.sql
deps_in = (
db.query(Dependency.id)
.filter(Dependency.name == dep_name)
.filter(Dependency.evr > evr1)
.filter(Dependency.evr < evr2)
.cte('deps_in')
)
# Dependencies with greater evr than evr2
deps_higher = (
db.query(Dependency.id)
.filter(Dependency.name == dep_name)
.filter(Dependency.evr >= evr2)
.cte('deps_higher')
)
# Dependencies with lesser evr than evr1
deps_lower = (
db.query(Dependency.id)
.filter(Dependency.name == dep_name)
.filter(Dependency.evr <= evr1)
.cte('deps_lower')
)
# Get only changes where the prev_evr to curr_evr interval overlaps with evr1 to evr2
filtered_changes = union(
# Changes with previous evr in the evr1 to evr2 interval
db.query(AppliedChange)
.filter(AppliedChange.prev_dep_id.in_(db.query(deps_in))),
# Changes with current evr in the evr1 to evr2 interval
db.query(AppliedChange)
.filter(AppliedChange.curr_dep_id.in_(db.query(deps_in))),
# Changes with both evrs "around" the evr1 to evr2 interval
db.query(AppliedChange)
.filter(
(AppliedChange.prev_dep_id.in_(db.query(deps_lower))) &
(AppliedChange.curr_dep_id.in_(db.query(deps_higher)))
),
).alias('filtered_changes')
prev_build = aliased(Build)
# Get a subquery for previous build state
subq = db.query(prev_build.state.label('prev_state'))\
.order_by(prev_build.started.desc())\
.filter(prev_build.started < Build.started)\
.filter(prev_build.package_id == Build.package_id)\
.limit(1)\
.correlate().as_scalar()
prev_dep = aliased(Dependency)
curr_dep = aliased(Dependency)
failed = (
db.query(
prev_dep.name.label('dep_name'),
prev_dep.evr.label('prev_evr'),
curr_dep.evr.label('curr_evr'),
AppliedChange.distance,
Build.id.label('build_id'),
Build.state.label('build_state'),
Build.started.label('build_started'),
Package.name.label('package_name'),
Package.resolved.label('package_resolved'),
Package.last_complete_build_state.label('package_lb_state'),
subq.label('prev_build_state'),
)
.select_entity_from(filtered_changes)
.join(prev_dep, AppliedChange.prev_dep)
.join(curr_dep, AppliedChange.curr_dep)
.join(AppliedChange.build).join(Build.package)
.filter_by(blocked=False, tracked=True, collection_id=collection.id)
# Show only packages where the build after failed, but the previous one was ok
.filter(Build.state == Build.FAILED)
.filter(subq != Build.FAILED)
.order_by(AppliedChange.distance, Build.started.desc())
.all()
)
# Auxiliary function to compute state string for the query row
def package_state(row):
return Package(
tracked=True,
blocked=False,
resolved=row.package_resolved,
last_complete_build_state=row.package_lb_state,
).state_string
return render_template("affected-by.html", package_state=package_state,
dep_name=dep_name, evr1=evr1, evr2=evr2,
collection=collection, failed=failed)
def package_detail(name, form=None, collection=None):
if not collection:
collection = g.current_collections[0]
g.current_collections = [collection]
base = db.query(BasePackage).filter_by(name=name).first_or_404()
# Get packages for all collections, so that we can display
# "State in other collections" table
packages = {p.collection_id: p for p in db.query(Package).filter_by(base_id=base.id)}
# assign packages to collections in the right order
package = None # the current package, may stay None
all_packages = []
for coll in g.collections:
p = packages.get(coll.id)
if p:
all_packages.append((coll, p))
if coll is collection:
package = p
# prepare group checkboxes
base.global_groups = db.query(PackageGroup)\
.join(PackageGroupRelation)\
.filter(PackageGroupRelation.base_id == base.id)\
.filter(PackageGroup.namespace == None)\
.all()
base.user_groups = []
base.available_groups = []
if g.user:
user_groups = \
db.query(PackageGroup,
func.bool_or(PackageGroupRelation.base_id == base.id))\
.outerjoin(PackageGroupRelation)\
.join(GroupACL)\
.filter(GroupACL.user_id == g.user.id)\
.order_by(PackageGroup.namespace.nullsfirst(), PackageGroup.name)\
.group_by(PackageGroup.id)\
.distinct().all()
base.user_groups = [group for group, checked in user_groups
if checked and group.namespace]
base.available_groups = [group for group, checked in user_groups
if not checked]
# History entry pagination pivot timestamp
# We only display entries older than this
last_seen_ts = request.args.get('last_seen_ts')
if last_seen_ts:
try:
last_seen_ts = int(last_seen_ts)
except ValueError:
abort(400)
def to_ts(col):
return cast(func.extract('EPOCH', col), Integer)
entries = None
if package:
# set current priority
package.current_priority = db.query(
Package.current_priority_expression(
collection=package.collection,
last_build=package.last_build,
)
).filter(Package.id == package.id).scalar()
# prepare history entries - builds and resolution changes
builds = db.query(Build)\
.filter_by(package_id=package.id)\
.filter(to_ts(Build.started) < last_seen_ts
if last_seen_ts else true())\
.options(subqueryload(Build.dependency_changes),
subqueryload(Build.build_arch_tasks))\
.order_by(Build.started.desc())\
.limit(builds_per_page)\
.all()
resolutions = db.query(ResolutionChange)\
.filter_by(package_id=package.id)\
.filter(to_ts(ResolutionChange.timestamp) < last_seen_ts
if last_seen_ts else true())\
.options(joinedload(ResolutionChange.problems))\
.order_by(ResolutionChange.timestamp.desc())\
.limit(builds_per_page)\
.all()
entries = sorted(
builds + resolutions,
key=lambda x: getattr(x, 'started', None) or getattr(x, 'timestamp'),
reverse=True,
)[:builds_per_page]
if not form:
form = forms.EditPackageForm(
tracked=package.tracked,
collection_id=package.collection_id,
manual_priority=package.manual_priority,
arch_override=(package.arch_override or '').split(' '),
skip_resolution=package.skip_resolution,
)
# Note: package might be None
return render_template(
"package-detail.html",
base=base,
package=package,
collection=collection,
form=form,
entries=entries,
all_packages=all_packages,
is_continuation=bool(last_seen_ts),
is_last=len(entries) < builds_per_page if package else True,
)
def package_detail(name, form=None, collection=None):
if not collection:
collection = g.current_collections[0]
g.current_collections = [collection]
base = db.query(BasePackage).filter_by(name=name).first_or_404()
# Get packages for all collections, so that we can display
# "State in other collections" table
packages = {
p.collection_id: p
for p in db.query(Package).filter_by(base_id=base.id)
}
# assign packages to collections in the right order
package = None # the current package, may stay None
all_packages = []
for coll in g.collections:
p = packages.get(coll.id)
if p:
all_packages.append((coll, p))
if coll is collection:
package = p
# prepare group checkboxes
base.global_groups = db.query(PackageGroup)\
.join(PackageGroupRelation)\
.filter(PackageGroupRelation.base_id == base.id)\
.filter(PackageGroup.namespace == None)\
.all()
base.user_groups = []
base.available_groups = []
if g.user:
user_groups = \
db.query(PackageGroup,
func.bool_or(PackageGroupRelation.base_id == base.id))\
.outerjoin(PackageGroupRelation)\
.join(GroupACL)\
.filter(GroupACL.user_id == g.user.id)\
.order_by(PackageGroup.namespace.nullsfirst(), PackageGroup.name)\
.group_by(PackageGroup.id)\
.distinct().all()
base.user_groups = [
group for group, checked in user_groups
if checked and group.namespace
]
base.available_groups = [
group for group, checked in user_groups if not checked
]
# History entry pagination pivot timestamp
# We only display entries older than this
last_seen_ts = request.args.get('last_seen_ts')
if last_seen_ts:
try:
last_seen_ts = int(last_seen_ts)
except ValueError:
abort(400)
def to_ts(col):
return cast(func.extract('EPOCH', col), Integer)
entries = None
if package:
# set current priority
package.current_priority = db.query(
Package.current_priority_expression(
collection=package.collection,
last_build=package.last_build,
)).filter(Package.id == package.id).scalar()
# prepare history entries - builds and resolution changes
builds = db.query(Build)\
.filter_by(package_id=package.id)\
.filter(to_ts(Build.started) < last_seen_ts
if last_seen_ts else true())\
.options(subqueryload(Build.dependency_changes),
subqueryload(Build.build_arch_tasks))\
.order_by(Build.started.desc())\
.limit(builds_per_page)\
.all()
resolutions = db.query(ResolutionChange)\
.filter_by(package_id=package.id)\
.filter(to_ts(ResolutionChange.timestamp) < last_seen_ts
if last_seen_ts else true())\
.options(joinedload(ResolutionChange.problems))\
.order_by(ResolutionChange.timestamp.desc())\
.limit(builds_per_page)\
.all()
entries = sorted(
builds + resolutions,
key=lambda x: getattr(x, 'started', None) or getattr(
x, 'timestamp'),
reverse=True,
)[:builds_per_page]
if not form:
form = forms.EditPackageForm(
tracked=package.tracked,
collection_id=package.collection_id,
manual_priority=package.manual_priority,
arch_override=(package.arch_override or '').split(' '),
skip_resolution=package.skip_resolution,
)
# Note: package might be None
return render_template(
"package-detail.html",
base=base,
package=package,
collection=collection,
form=form,
entries=entries,
all_packages=all_packages,
is_continuation=bool(last_seen_ts),
is_last=len(entries) < builds_per_page if package else True,
)
def edit_group(name, namespace=None):
group = db.query(PackageGroup)\
.options(joinedload(PackageGroup.packages))\
.filter_by(name=name, namespace=namespace).first_or_404()
return process_group_form(group=group)
def groups_overview():
groups = db.query(PackageGroup)\
.options(undefer(PackageGroup.package_count))\
.filter_by(namespace=None)\
.order_by(PackageGroup.name).all()
return render_template("list-groups.html", groups=groups)
def affected_by(dep_name):
"""
Display which packages are possibly affected by given dependency change.
"""
if len(g.current_collections) != 1:
abort(400)
collection = g.current_collections[0]
try:
evr1 = RpmEVR(int(request.args['epoch1']), request.args['version1'],
request.args['release1'])
evr2 = RpmEVR(int(request.args['epoch2']), request.args['version2'],
request.args['release2'])
except (KeyError, ValueError):
abort(400)
# Dependencies in the evr1 to evr2 interval
# Note that evr comparisons are overloaded custom comparators that invoke RPM-correct
# comparisons implemented in rpmvercmp.sql
deps_in = (db.query(Dependency.id).filter(
Dependency.name == dep_name).filter(Dependency.evr > evr1).filter(
Dependency.evr < evr2).cte('deps_in'))
# Dependencies with greater evr than evr2
deps_higher = (db.query(
Dependency.id).filter(Dependency.name == dep_name).filter(
Dependency.evr >= evr2).cte('deps_higher'))
# Dependencies with lesser evr than evr1
deps_lower = (db.query(
Dependency.id).filter(Dependency.name == dep_name).filter(
Dependency.evr <= evr1).cte('deps_lower'))
# Get only changes where the prev_evr to curr_evr interval overlaps with evr1 to evr2
filtered_changes = union(
# Changes with previous evr in the evr1 to evr2 interval
db.query(AppliedChange).filter(
AppliedChange.prev_dep_id.in_(db.query(deps_in))),
# Changes with current evr in the evr1 to evr2 interval
db.query(AppliedChange).filter(
AppliedChange.curr_dep_id.in_(db.query(deps_in))),
# Changes with both evrs "around" the evr1 to evr2 interval
db.query(AppliedChange).filter(
(AppliedChange.prev_dep_id.in_(db.query(deps_lower)))
& (AppliedChange.curr_dep_id.in_(db.query(deps_higher)))),
).alias('filtered_changes')
prev_build = aliased(Build)
# Get a subquery for previous build state
subq = db.query(prev_build.state.label('prev_state'))\
.order_by(prev_build.started.desc())\
.filter(prev_build.started < Build.started)\
.filter(prev_build.package_id == Build.package_id)\
.limit(1)\
.correlate().as_scalar()
prev_dep = aliased(Dependency)
curr_dep = aliased(Dependency)
failed = (
db.query(
prev_dep.name.label('dep_name'),
prev_dep.evr.label('prev_evr'),
curr_dep.evr.label('curr_evr'),
AppliedChange.distance,
Build.id.label('build_id'),
Build.state.label('build_state'),
Build.started.label('build_started'),
Package.name.label('package_name'),
Package.resolved.label('package_resolved'),
Package.last_complete_build_state.label('package_lb_state'),
subq.label('prev_build_state'),
).select_entity_from(filtered_changes).join(
prev_dep, AppliedChange.prev_dep).join(
curr_dep,
AppliedChange.curr_dep).join(AppliedChange.build).join(
Build.package).filter_by(blocked=False,
tracked=True,
collection_id=collection.id)
# Show only packages where the build after failed, but the previous one was ok
.filter(Build.state == Build.FAILED).filter(
subq != Build.FAILED).order_by(AppliedChange.distance,
Build.started.desc()).all())
# Auxiliary function to compute state string for the query row
def package_state(row):
return Package(
tracked=True,
blocked=False,
resolved=row.package_resolved,
last_complete_build_state=row.package_lb_state,
).state_string
return render_template("affected-by.html",
package_state=package_state,
dep_name=dep_name,
evr1=evr1,
evr2=evr2,
collection=collection,
failed=failed)
def edit_package(name):
"""
Edit package attributes or groups. Everyone can edit attributes, group membership
requires permissions.
"""
form = forms.EditPackageForm()
collection = g.collections_by_id.get(form.collection_id.data) or abort(400)
if not form.validate_or_flash():
return package_detail(name=name, form=form, collection=collection)
package = db.query(Package)\
.filter_by(name=name, collection_id=collection.id)\
.first_or_404()
# Interpret group checkboxes
for key, prev_val in request.form.items():
if key.startswith('group-prev-'):
group = db.query(PackageGroup).get_or_404(
int(key[len('group-prev-'):]))
new_val = request.form.get('group-{}'.format(group.id))
if bool(new_val) != (prev_val == 'true'):
if not group.editable:
abort(403)
if new_val:
data.set_group_content(session,
group, [package.name],
append=True)
else:
data.set_group_content(session,
group, [package.name],
delete=True)
# Using set_package_attribute to generate audit log events
if form.tracked.data is not None:
data.set_package_attribute(
session,
package,
'tracked',
form.tracked.data,
)
if form.manual_priority.data is not None:
data.set_package_attribute(
session,
package,
'manual_priority',
form.manual_priority.data,
)
if form.arch_override.data is not None:
data.set_package_attribute(
session,
package,
'arch_override',
' '.join(form.arch_override.data) or None,
)
if form.skip_resolution.data is not None:
data.set_package_attribute(
session,
package,
'skip_resolution',
form.skip_resolution.data,
)
if package.skip_resolution:
package.resolved = None
db.query(UnappliedChange).filter_by(package_id=package.id).delete()
flash_ack("Package modified")
db.commit()
return redirect(
url_for('package_detail', name=package.name) + "?collection=" +
collection.name)
def edit_group(name, namespace=None):
group = db.query(PackageGroup)\
.options(joinedload(PackageGroup.packages))\
.filter_by(name=name, namespace=namespace).first_or_404()
return process_group_form(group=group)
