def refresh_packages(self): """ Refresh packages from Koji: add packages not yet known by Koschei and update blocked flag. """ for collection in self.db.query(Collection): koji_packages = self.koji_sessions['secondary']\ .listPackages(tagID=collection.target_tag, inherited=True) whitelisted = { p['package_name'] for p in koji_packages if not p['blocked'] } packages = self.db.query(Package).filter_by( collection_id=collection.id).all() to_update = [ p.id for p in packages if p.blocked == (p.name in whitelisted) ] if to_update: self.db.query(Package).filter(Package.id.in_(to_update))\ .update({'blocked': ~Package.blocked}, synchronize_session=False) self.db.flush() existing_names = {p.name for p in packages} to_add = [ p for p in koji_packages if p['package_name'] not in existing_names ] if to_add: for p in to_add: pkg = Package(name=p['package_name'], collection_id=collection.id) pkg.blocked = p['blocked'] pkg.tracked = False self.db.add(pkg) self.db.flush() self.db.expire_all()
def refresh_packages(self): """ Refresh packages from Koji: add packages not yet known by Koschei and update blocked flag. """ source_tag = util.koji_config['source_tag'] koji_packages = self.koji_session.listPackages(tagID=source_tag, inherited=True) whitelisted = { p['package_name'] for p in koji_packages if not p['blocked'] } packages = self.db.query(Package).all() to_update = [ p.id for p in packages if p.blocked == (p.name in whitelisted) ] if to_update: self.db.query(Package).filter(Package.id.in_(to_update))\ .update({'blocked': ~Package.blocked}, synchronize_session=False) self.db.expire_all() self.db.flush() existing_names = {p.name for p in packages} to_add = [ p for p in koji_packages if p['package_name'] not in existing_names ] if to_add: for p in to_add: pkg = Package(name=p['package_name']) pkg.blocked = p['blocked'] pkg.tracked = False self.db.add(pkg) self.db.flush()
def refresh_packages(self): """ Refresh packages from Koji: add packages not yet known by Koschei and update blocked flag. """ for collection in self.db.query(Collection): koji_packages = self.koji_sessions['secondary']\ .listPackages(tagID=collection.target_tag, inherited=True) whitelisted = {p['package_name'] for p in koji_packages if not p['blocked']} packages = self.db.query(Package).filter_by(collection_id=collection.id).all() to_update = [p.id for p in packages if p.blocked == (p.name in whitelisted)] if to_update: self.db.query(Package).filter(Package.id.in_(to_update))\ .update({'blocked': ~Package.blocked}, synchronize_session=False) self.db.flush() existing_names = {p.name for p in packages} to_add = [p for p in koji_packages if p['package_name'] not in existing_names] if to_add: for p in to_add: pkg = Package(name=p['package_name'], collection_id=collection.id) pkg.blocked = p['blocked'] pkg.tracked = False self.db.add(pkg) self.db.flush() self.db.expire_all()
def state_icons(package_row): state_array = package_row.states packages = {} if len(state_array) > 4: # format is like this {"(2,t,,3)","(1,t,f,3)"} state_array = state_array[2:-2].split('","') for state in state_array: collection_id, tracked, resolved, build_state = state[1:-1].split( ',') resolved = None if resolved == '' else (resolved == 't') build_state = None if build_state == '' else int(build_state) tracked = tracked == 't' package = Package(name=package_row.name, blocked=False, tracked=tracked, last_complete_build_state=build_state, resolved=resolved) packages[int(collection_id)] = package out = "" for collection in g.collections: package = packages.get(collection.id) out += '<td>' if package: out += '<img src="{icon}" title="{title}"/>'\ .format(icon=package.state_icon, title=package.state_string) out += '</td>' return Markup(out)
def package_state(row): return Package( tracked=True, blocked=False, resolved=row.package_resolved, last_complete_build_state=row.package_lb_state, ).state_string
def refresh_user_packages(user): if not user.packages_retrieved: db = Session.object_session(user) names = query_users_packages(user.name) if names is not None: user.packages_retrieved = True existing = { p for [p] in db.query(Package.name).filter( Package.name.in_(names)) } for name in names: if name not in existing: pkg = Package(name=name, tracked=False) db.add(pkg) db.flush() packages = db.query(Package.id)\ .filter(Package.name.in_(names)).all() entries = [{ 'user_id': user.id, 'package_id': pkg.id } for pkg in packages] db.execute( delete(UserPackageRelation, UserPackageRelation.user_id == user.id)) db.execute(insert(UserPackageRelation, entries)) db.commit()
def get_priority_join(self, pkg): return self.db.query( Package.current_priority_expression( collection=Collection, last_build=Build, ) ).join(Package.collection)\ .join(Package.last_build)\ .filter(Package.id == pkg.id).scalar()
def collection_has_schedulable_package(db, collection): priority_threshold = get_config('priorities.build_threshold') priority_expr = Package.current_priority_expression(Collection, Build) return db.query(Package) \ .join(Package.collection) \ .join(Package.last_build) \ .filter(Package.collection_id == collection.id) \ .filter(priority_expr != None) \ .filter(priority_expr >= priority_threshold) \ .limit(1).count()
def get_priorities(self): priority_expr = Package.current_priority_expression( collection=Collection, last_build=Build, ) return self.db.query(Package.id, priority_expr)\ .join(Package.collection)\ .join(Package.last_build)\ .filter(priority_expr != None)\ .order_by(priority_expr.desc())\ .all()
def collection_package_view(template, query_fn=None, **template_args): """ Single-collection view of a list of packages. :param template: name of jinja2 template to be used :param query_fn: optional filter function of query -> filtered_query :param template_args: additional arguments passed to the template """ # should be called only when len(g.current_collections) == 1 collection = g.current_collections[0] # query current_priority separately as it's not a property of Package current_prio_expr = Package.current_priority_expression( collection=collection, last_build=Build, # package is outerjoined with last_build ) package_query = db.query(Package, current_prio_expr)\ .filter(Package.collection_id == collection.id) if query_fn: package_query = query_fn(package_query.join(BasePackage)) # whether to show untracked packages as well untracked = request.args.get('untracked') == '1' # determine correct ORDER BY order_name = request.args.get('order_by', 'running,state,name') order_map = { 'name': [Package.name], 'state': [Package.resolved, Reversed(Build.state)], 'running': [Package.last_complete_build_id == Package.last_build_id], 'task_id': [Build.task_id], 'started': [Build.started], 'current_priority': [NullsLastOrder(current_prio_expr)], } order_names, order = get_order(order_map, order_name) if not untracked: package_query = package_query.filter(Package.tracked == True) pkgs = package_query.filter(Package.blocked == False)\ .outerjoin(Package.last_build)\ .options(contains_eager(Package.last_build))\ .order_by(*order) page = pkgs.paginate(packages_per_page) # monkeypatch the priority as an attribute for ease of use for pkg, priority in page.items: pkg.current_priority = priority # extract only the package from the query results page.items = [pkg for pkg, _ in page.items] # monkeypatch visible package groups populate_package_groups(page.items) return render_template(template, packages=page.items, page=page, order=order_names, collection=collection, **template_args)
def refresh_packages(session): """ Refresh package list from Koji. Add packages not yet known by Koschei and update blocked flag of existing packages. """ bases = { base.name: base for base in session.db.query(BasePackage.id, BasePackage.name) } for collection in session.db.query(Collection.id, Collection.dest_tag, Collection.secondary_mode): koji_session = session.secondary_koji_for(collection) koji_packages = koji_session.listPackages(tagID=collection.dest_tag, inherited=True) whitelisted = { p['package_name'] for p in koji_packages if not p['blocked'] } packages = session.db.query(Package.id, Package.name, Package.blocked)\ .filter_by(collection_id=collection.id)\ .all() # Find packages which need to be blocked/unblocked to_update = [ p.id for p in packages if p.blocked == (p.name in whitelisted) ] if to_update: session.db.query(Package).filter(Package.id.in_(to_update))\ .update({'blocked': ~Package.blocked}, synchronize_session=False) existing_names = {p.name for p in packages} # Find packages to be added to_add = [] # Add PackageBases for pkg_dict in koji_packages: name = pkg_dict['package_name'] if name not in bases.keys(): base = BasePackage(name=name) bases[name] = base to_add.append(base) session.db.bulk_insert(to_add) to_add = [] # Add Packages for pkg_dict in koji_packages: name = pkg_dict['package_name'] if name not in existing_names: pkg = Package(name=name, base_id=bases.get(name).id, collection_id=collection.id, tracked=False, blocked=pkg_dict['blocked']) to_add.append(pkg) session.db.bulk_insert(to_add) session.db.expire_all()
def collection_package_view(template, query_fn=None, **template_args): """ Single-collection view of a list of packages. :param template: name of jinja2 template to be used :param query_fn: optional filter function of query -> filtered_query :param template_args: additional arguments passed to the template """ # should be called only when len(g.current_collections) == 1 collection = g.current_collections[0] # query current_priority separately as it's not a property of Package current_prio_expr = Package.current_priority_expression( collection=collection, last_build=Build, # package is outerjoined with last_build ) package_query = db.query(Package, current_prio_expr)\ .filter(Package.collection_id == collection.id) if query_fn: package_query = query_fn(package_query.join(BasePackage)) # whether to show untracked packages as well untracked = request.args.get('untracked') == '1' # determine correct ORDER BY order_name = request.args.get('order_by', 'running,state,name') order_map = { 'name': [Package.name], 'state': [Package.resolved, Reversed(Build.state)], 'running': [Package.last_complete_build_id == Package.last_build_id], 'task_id': [Build.task_id], 'started': [Build.started], 'current_priority': [NullsLastOrder(current_prio_expr)], } order_names, order = get_order(order_map, order_name) if not untracked: package_query = package_query.filter(Package.tracked == True) pkgs = package_query.filter(Package.blocked == False)\ .outerjoin(Package.last_build)\ .options(contains_eager(Package.last_build))\ .order_by(*order) page = pkgs.paginate(packages_per_page) # monkeypatch the priority as an attribute for ease of use for pkg, priority in page.items: pkg.current_priority = priority # extract only the package from the query results page.items = [pkg for pkg, _ in page.items] # monkeypatch visible package groups populate_package_groups(page.items) return render_template(template, packages=page.items, page=page, order=order_names, collection=collection, **template_args)
def prepare_package(self, name=None, collection=None, **kwargs): if 'collection_id' not in kwargs: if collection is None: collection = self.collection if not name: name = 'p{}'.format(self.pkg_name_counter) self.pkg_name_counter += 1 base = self.db.query(BasePackage).filter_by(name=name).first() if not base: base = BasePackage(name=name) pkg = Package(name=name, base=base, collection=collection, **kwargs) self.db.add(pkg) self.db.commit() return pkg
def __init__(self, row): self.name = row.name self.has_running_build = row.has_running_build self.base_id = row.base_id self.packages = [] for collection in g.current_collections: str_id = str(collection.id) package = Package( name=row.name, blocked=False, collection=collection, tracked=getattr(row, 'tracked' + str_id) or False, last_complete_build_state=getattr(row, 'state' + str_id), resolved=getattr(row, 'resolved' + str_id), ) self.packages.append(package)
def package_detail(name, form=None, collection=None): if not collection: collection = g.current_collections[0] g.current_collections = [collection] base = db.query(BasePackage).filter_by(name=name).first_or_404() # Get packages for all collections, so that we can display # "State in other collections" table packages = {p.collection_id: p for p in db.query(Package).filter_by(base_id=base.id)} # assign packages to collections in the right order package = None # the current package, may stay None all_packages = [] for coll in g.collections: p = packages.get(coll.id) if p: all_packages.append((coll, p)) if coll is collection: package = p # prepare group checkboxes base.global_groups = db.query(PackageGroup)\ .join(PackageGroupRelation)\ .filter(PackageGroupRelation.base_id == base.id)\ .filter(PackageGroup.namespace == None)\ .all() base.user_groups = [] base.available_groups = [] if g.user: user_groups = \ db.query(PackageGroup, func.bool_or(PackageGroupRelation.base_id == base.id))\ .outerjoin(PackageGroupRelation)\ .join(GroupACL)\ .filter(GroupACL.user_id == g.user.id)\ .order_by(PackageGroup.namespace.nullsfirst(), PackageGroup.name)\ .group_by(PackageGroup.id)\ .distinct().all() base.user_groups = [group for group, checked in user_groups if checked and group.namespace] base.available_groups = [group for group, checked in user_groups if not checked] # History entry pagination pivot timestamp # We only display entries older than this last_seen_ts = request.args.get('last_seen_ts') if last_seen_ts: try: last_seen_ts = int(last_seen_ts) except ValueError: abort(400) def to_ts(col): return cast(func.extract('EPOCH', col), Integer) entries = None if package: # set current priority package.current_priority = db.query( Package.current_priority_expression( collection=package.collection, last_build=package.last_build, ) ).filter(Package.id == package.id).scalar() # prepare history entries - builds and resolution changes builds = db.query(Build)\ .filter_by(package_id=package.id)\ .filter(to_ts(Build.started) < last_seen_ts if last_seen_ts else true())\ .options(subqueryload(Build.dependency_changes), subqueryload(Build.build_arch_tasks))\ .order_by(Build.started.desc())\ .limit(builds_per_page)\ .all() resolutions = db.query(ResolutionChange)\ .filter_by(package_id=package.id)\ .filter(to_ts(ResolutionChange.timestamp) < last_seen_ts if last_seen_ts else true())\ .options(joinedload(ResolutionChange.problems))\ .order_by(ResolutionChange.timestamp.desc())\ .limit(builds_per_page)\ .all() entries = sorted( builds + resolutions, key=lambda x: getattr(x, 'started', None) or getattr(x, 'timestamp'), reverse=True, )[:builds_per_page] if not form: form = forms.EditPackageForm( tracked=package.tracked, collection_id=package.collection_id, manual_priority=package.manual_priority, arch_override=(package.arch_override or '').split(' '), skip_resolution=package.skip_resolution, ) # Note: package might be None return render_template( "package-detail.html", base=base, package=package, collection=collection, form=form, entries=entries, all_packages=all_packages, is_continuation=bool(last_seen_ts), is_last=len(entries) < builds_per_page if package else True, )
def package_detail(name, form=None, collection=None): if not collection: collection = g.current_collections[0] g.current_collections = [collection] base = db.query(BasePackage).filter_by(name=name).first_or_404() # Get packages for all collections, so that we can display # "State in other collections" table packages = { p.collection_id: p for p in db.query(Package).filter_by(base_id=base.id) } # assign packages to collections in the right order package = None # the current package, may stay None all_packages = [] for coll in g.collections: p = packages.get(coll.id) if p: all_packages.append((coll, p)) if coll is collection: package = p # prepare group checkboxes base.global_groups = db.query(PackageGroup)\ .join(PackageGroupRelation)\ .filter(PackageGroupRelation.base_id == base.id)\ .filter(PackageGroup.namespace == None)\ .all() base.user_groups = [] base.available_groups = [] if g.user: user_groups = \ db.query(PackageGroup, func.bool_or(PackageGroupRelation.base_id == base.id))\ .outerjoin(PackageGroupRelation)\ .join(GroupACL)\ .filter(GroupACL.user_id == g.user.id)\ .order_by(PackageGroup.namespace.nullsfirst(), PackageGroup.name)\ .group_by(PackageGroup.id)\ .distinct().all() base.user_groups = [ group for group, checked in user_groups if checked and group.namespace ] base.available_groups = [ group for group, checked in user_groups if not checked ] # History entry pagination pivot timestamp # We only display entries older than this last_seen_ts = request.args.get('last_seen_ts') if last_seen_ts: try: last_seen_ts = int(last_seen_ts) except ValueError: abort(400) def to_ts(col): return cast(func.extract('EPOCH', col), Integer) entries = None if package: # set current priority package.current_priority = db.query( Package.current_priority_expression( collection=package.collection, last_build=package.last_build, )).filter(Package.id == package.id).scalar() # prepare history entries - builds and resolution changes builds = db.query(Build)\ .filter_by(package_id=package.id)\ .filter(to_ts(Build.started) < last_seen_ts if last_seen_ts else true())\ .options(subqueryload(Build.dependency_changes), subqueryload(Build.build_arch_tasks))\ .order_by(Build.started.desc())\ .limit(builds_per_page)\ .all() resolutions = db.query(ResolutionChange)\ .filter_by(package_id=package.id)\ .filter(to_ts(ResolutionChange.timestamp) < last_seen_ts if last_seen_ts else true())\ .options(joinedload(ResolutionChange.problems))\ .order_by(ResolutionChange.timestamp.desc())\ .limit(builds_per_page)\ .all() entries = sorted( builds + resolutions, key=lambda x: getattr(x, 'started', None) or getattr( x, 'timestamp'), reverse=True, )[:builds_per_page] if not form: form = forms.EditPackageForm( tracked=package.tracked, collection_id=package.collection_id, manual_priority=package.manual_priority, arch_override=(package.arch_override or '').split(' '), skip_resolution=package.skip_resolution, ) # Note: package might be None return render_template( "package-detail.html", base=base, package=package, collection=collection, form=form, entries=entries, all_packages=all_packages, is_continuation=bool(last_seen_ts), is_last=len(entries) < builds_per_page if package else True, )