def percpu_counter_sum(fbc: Object) -> int:
"""
Return the sum of a per-CPU counter.
:param fbc: ``struct percpu_counter *``
"""
ret = fbc.count.value_()
ptr = fbc.counters
for cpu in for_each_online_cpu(fbc.prog_):
ret += per_cpu_ptr(ptr, cpu)[0].value_()
return ret
def percpu_counter_sum(fbc):
"""
.. c:function:: s64 percpu_counter_sum(struct percpu_counter *fbc)
Return the sum of a per-CPU counter.
"""
ret = fbc.count.value_()
ptr = fbc.counters
for cpu in for_each_online_cpu(fbc.prog_):
ret += per_cpu_ptr(ptr, cpu)[0].value_()
return ret
def slab_cache_for_each_allocated_object(
slab_cache: Object, type: Union[str, Type]
) -> Iterator[Object]:
"""
Iterate over all allocated objects in a given slab cache.
Only the SLUB and SLAB allocators are supported; SLOB does not store enough
information to identify objects in a slab cache.
>>> dentry_cache = find_slab_cache(prog, "dentry")
>>> next(slab_cache_for_each_allocated_object(dentry_cache, "struct dentry"))
*(struct dentry *)0xffff905e41404000 = {
...
}
:param slab_cache: ``struct kmem_cache *``
:param type: Type of object in the slab cache.
:return: Iterator of ``type *`` objects.
"""
prog = slab_cache.prog_
slab_cache_size = slab_cache.size.value_()
pointer_type = prog.pointer_type(prog.type(type))
try:
freelist_type = prog.type("freelist_idx_t *")
slub = False
except LookupError:
slub = True
if slub:
try:
red_left_pad = slab_cache.red_left_pad.value_()
except AttributeError:
red_left_pad = 0
# In SLUB, the freelist is a linked list with the next pointer located
# at ptr + slab_cache->offset.
try:
freelist_offset = slab_cache.offset.value_()
except AttributeError:
raise ValueError("SLOB is not supported") from None
# If CONFIG_SLAB_FREELIST_HARDENED is enabled, then the next pointer is
# obfuscated using slab_cache->random.
try:
freelist_random = slab_cache.random.value_()
except AttributeError:
def _freelist_dereference(ptr_addr: int) -> int:
return prog.read_word(ptr_addr)
else:
ulong_size = sizeof(prog.type("unsigned long"))
def _freelist_dereference(ptr_addr: int) -> int:
# *ptr_addr ^ slab_cache->random ^ byteswap(ptr_addr)
return (
prog.read_word(ptr_addr)
^ freelist_random
^ int.from_bytes(ptr_addr.to_bytes(ulong_size, "little"), "big")
)
def _slub_get_freelist(freelist: Object, freelist_set: Set[int]) -> None:
ptr = freelist.value_()
while ptr:
freelist_set.add(ptr)
ptr = _freelist_dereference(ptr + freelist_offset)
cpu_freelists: Set[int] = set()
cpu_slab = slab_cache.cpu_slab.read_()
# Since Linux kernel commit bb192ed9aa71 ("mm/slub: Convert most struct
# page to struct slab by spatch") (in v5.17), the current slab for a
# CPU is `struct slab *slab`. Before that, it is `struct page *page`.
cpu_slab_attr = "slab" if hasattr(cpu_slab, "slab") else "page"
for cpu in for_each_online_cpu(prog):
this_cpu_slab = per_cpu_ptr(cpu_slab, cpu)
slab = getattr(this_cpu_slab, cpu_slab_attr).read_()
if slab and slab.slab_cache == slab_cache:
_slub_get_freelist(this_cpu_slab.freelist, cpu_freelists)
def _slab_page_objects(page: Object, slab: Object) -> Iterator[Object]:
freelist: Set[int] = set()
_slub_get_freelist(slab.freelist, freelist)
addr = page_to_virt(page).value_() + red_left_pad
end = addr + slab_cache_size * slab.objects
while addr < end:
if addr not in freelist and addr not in cpu_freelists:
yield Object(prog, pointer_type, value=addr)
addr += slab_cache_size
else:
try:
obj_offset = slab_cache.obj_offset.value_()
except AttributeError:
obj_offset = 0
slab_cache_num = slab_cache.num.value_()
cpu_cache = slab_cache.cpu_cache.read_()
cpu_caches_avail: Set[int] = set()
for cpu in for_each_online_cpu(prog):
ac = per_cpu_ptr(cpu_cache, cpu)
for i in range(ac.avail):
cpu_caches_avail.add(ac.entry[i].value_())
def _slab_freelist(slab: Object) -> Set[int]:
# In SLAB, the freelist is an array of free object indices.
freelist = cast(freelist_type, slab.freelist)
return {freelist[i].value_() for i in range(slab.active, slab_cache_num)}
def _slab_page_objects(page: Object, slab: Object) -> Iterator[Object]:
freelist = _slab_freelist(slab)
s_mem = slab.s_mem.value_()
for i in range(slab_cache_num):
if i in freelist:
continue
addr = s_mem + i * slab_cache_size + obj_offset
if addr in cpu_caches_avail:
continue
yield Object(prog, pointer_type, value=addr)
# Linux kernel commit d122019bf061cccc4583eb9ad40bf58c2fe517be ("mm: Split
# slab into its own type") (in v5.17) moved slab information from struct
# page to struct slab. The former can be casted to the latter.
try:
slab_type = prog.type("struct slab *")
except LookupError:
slab_type = prog.type("struct page *")
PG_slab_mask = 1 << prog.constant("PG_slab")
for page in for_each_page(prog):
try:
if not page.flags & PG_slab_mask:
continue
except FaultError:
continue
slab = cast(slab_type, page)
if slab.slab_cache == slab_cache:
yield from _slab_page_objects(page, slab)
def slab_cache_for_each_allocated_object(
slab_cache: Object, type: Union[str, Type]) -> Iterator[Object]:
"""
Iterate over all allocated objects in a given slab cache.
Only the SLUB and SLAB allocators are supported; SLOB does not store enough
information to identify objects in a slab cache.
>>> dentry_cache = find_slab_cache(prog, "dentry")
>>> next(slab_cache_for_each_allocated_object(dentry_cache, "struct dentry"))
*(struct dentry *)0xffff905e41404000 = {
...
}
:param slab_cache: ``struct kmem_cache *``
:param type: Type of object in the slab cache.
:return: Iterator of ``type *`` objects.
"""
prog = slab_cache.prog_
slab_cache_size = slab_cache.size.value_()
pointer_type = prog.pointer_type(prog.type(type))
try:
freelist_type = prog.type("freelist_idx_t *")
slub = False
except LookupError:
slub = True
if slub:
try:
red_left_pad = slab_cache.red_left_pad.value_()
except AttributeError:
red_left_pad = 0
try:
freelist_offset = slab_cache.offset.value_()
except AttributeError:
raise ValueError("SLOB is not supported") from None
def _slub_get_freelist(freelist: Object,
freelist_set: Set[int]) -> None:
# In SLUB, the freelist is a linked list with the next pointer
# located at ptr + slab_cache->offset.
ptr = freelist.value_()
while ptr:
freelist_set.add(ptr)
ptr = prog.read_word(ptr + freelist_offset)
cpu_freelists: Set[int] = set()
cpu_slab = slab_cache.cpu_slab.read_()
for cpu in for_each_online_cpu(prog):
_slub_get_freelist(
per_cpu_ptr(cpu_slab, cpu).freelist, cpu_freelists)
def _slab_page_objects(page: Object, slab: Object) -> Iterator[Object]:
freelist: Set[int] = set()
_slub_get_freelist(slab.freelist, freelist)
addr = page_to_virt(page).value_() + red_left_pad
end = addr + slab_cache_size * slab.objects
while addr < end:
if addr not in freelist and addr not in cpu_freelists:
yield Object(prog, pointer_type, value=addr)
addr += slab_cache_size
else:
try:
obj_offset = slab_cache.obj_offset.value_()
except AttributeError:
obj_offset = 0
slab_cache_num = slab_cache.num.value_()
cpu_cache = slab_cache.cpu_cache.read_()
cpu_caches_avail: Set[int] = set()
for cpu in for_each_online_cpu(prog):
ac = per_cpu_ptr(cpu_cache, cpu)
for i in range(ac.avail):
cpu_caches_avail.add(ac.entry[i].value_())
def _slab_freelist(slab: Object) -> Set[int]:
# In SLAB, the freelist is an array of free object indices.
freelist = cast(freelist_type, slab.freelist)
return {
freelist[i].value_()
for i in range(slab.active, slab_cache_num)
}
def _slab_page_objects(page: Object, slab: Object) -> Iterator[Object]:
freelist = _slab_freelist(slab)
s_mem = slab.s_mem.value_()
for i in range(slab_cache_num):
if i in freelist:
continue
addr = s_mem + i * slab_cache_size + obj_offset
if addr in cpu_caches_avail:
continue
yield Object(prog, pointer_type, value=addr)
# Linux kernel commit d122019bf061cccc4583eb9ad40bf58c2fe517be ("mm: Split
# slab into its own type") (in v5.17) moved slab information from struct
# page to struct slab. The former can be casted to the latter.
try:
slab_type = prog.type("struct slab *")
except LookupError:
slab_type = prog.type("struct page *")
PG_slab_mask = 1 << prog.constant("PG_slab")
for page in for_each_page(prog):
try:
if not page.flags & PG_slab_mask:
continue
except FaultError:
continue
slab = cast(slab_type, page)
if slab.slab_cache == slab_cache:
yield from _slab_page_objects(page, slab)