def create_from_source(cls, kernel_dir, output_dir, fun_kind=None,
setup_dir=True):
"""
Create a snapshot from a kernel source directory and prepare it for
snapshot directory generation.
:param kernel_dir: Source kernel directory.
:param output_dir: Snapshot output directory.
:param fun_kind: Snapshot function kind.
:param setup_dir: Whether to recreate the output directory.
:return: Desired instance of Snapshot.
"""
output_path = os.path.abspath(output_dir)
# Cleanup or create the output directory of the new snapshot.
if setup_dir:
if os.path.isdir(output_path):
shutil.rmtree(output_path)
os.mkdir(output_path)
# Prepare source representations for the new snapshot
kernel_source = KernelSource(kernel_dir, True)
snapshot_source = KernelSource(output_path)
kernel_snapshot = cls(kernel_source, snapshot_source, fun_kind)
# Add a new None group to the snapshot if the function kind is None.
if fun_kind is None:
kernel_snapshot._add_none_group()
return kernel_snapshot
def mod():
"""
Build LlvmSysctlModule for net.core.* sysctl options shared among tests.
"""
source = KernelSource("kernel/linux-3.10.0-862.el7", True)
kernel_module = source.get_module_from_source("net/core/sysctl_net_core.c")
yield LlvmSysctlModule(kernel_module, "net_core_table")
source.finalize()
def test_find_param_var():
"""
Test finding the name of a variable corresponding to a module parameter.
This is necessary since for parameters defined with module_param_named,
names of the parameter and of the variable differ.
"""
source = KernelSource("kernel/linux-3.10", True)
mod = source.get_module_from_source("net/rfkill/core.c")
assert mod.find_param_var("default_state").name == "rfkill_default_state"
def _from_yaml(self, yaml_file):
"""
Load the snaphot from its YAML representation. Paths are assumed to be
relative to the root directory.
:param yaml_file: Contents of the YAML file.
"""
yaml_file = yaml.safe_load(yaml_file)
yaml_dict = yaml_file[0]
self.created_time = yaml_dict["created_time"]
if os.path.isdir(yaml_dict["source_kernel_dir"]):
self.kernel_source = KernelSource(yaml_dict["source_kernel_dir"],
True)
for g in yaml_dict["list"]:
if "sysctl" in g:
self.kind = "sysctl"
group = g["sysctl"]
functions = g["functions"]
else:
group = None
functions = yaml_dict["list"]
self.fun_groups[group] = self.FunctionGroup()
for f in functions:
self.add_fun(
f["name"],
LlvmKernelModule(
os.path.join(
os.path.relpath(self.snapshot_source.kernel_dir),
f["llvm"])) if f["llvm"] else None, f["glob_var"],
f["tag"], group)
def __init__(self, spec, task_name, tasks_path, kernel_path):
self.old_kernel_dir = os.path.join(kernel_path, spec["old_kernel"])
self.new_kernel_dir = os.path.join(kernel_path, spec["new_kernel"])
self.name = task_name
self.task_dir = os.path.join(tasks_path, task_name)
if "control_flow_only" in spec:
self.control_flow_only = spec["control_flow_only"]
else:
self.control_flow_only = False
# Create LLVM sources and configuration
self.old_kernel = KernelSource(self.old_kernel_dir, True)
self.new_kernel = KernelSource(self.new_kernel_dir, True)
self.config = Config(self.old_kernel, self.new_kernel, False,
self.control_flow_only, False, None)
self.functions = dict()
def generate(args):
"""
Generate snapshot of sources of kernel functions.
This involves:
- find source code with functions definitions
- compile the source codes into LLVM IR
- copy LLVM and C source files into snapshot directory
- create YAML with list mapping functions to their LLVM sources
"""
source = KernelSource(args.kernel_dir, True)
args.output_dir = os.path.abspath(args.output_dir)
fun_list = FunctionList(args.output_dir)
# Cleanup or create the output directory
if os.path.isdir(args.output_dir):
shutil.rmtree(args.output_dir)
os.mkdir(args.output_dir)
# Build sources for functions from the list into LLVM IR
with open(args.functions_list, "r") as fun_list_file:
for line in fun_list_file.readlines():
fun = line.strip()
if not fun or not (fun[0].isalpha() or fun[0] == "_"):
continue
sys.stdout.write("{}: ".format(fun))
try:
llvm_mod = source.get_module_for_symbol(fun)
print(os.path.relpath(llvm_mod.llvm, args.kernel_dir))
fun_list.add(fun, llvm_mod)
except SourceNotFoundException:
print("source not found")
# Copy LLVM files to the snapshot
source.copy_source_files(fun_list.modules(), args.output_dir)
source.copy_cscope_files(args.output_dir)
# Create YAML with functions list
with open(os.path.join(args.output_dir, "functions.yaml"),
"w") as fun_list_yaml:
fun_list_yaml.write(fun_list.to_yaml())
source.finalize()
def load_from_dir(cls, snapshot_dir, config_file="snapshot.yaml"):
"""
Loads a snapshot from its directory.
:param snapshot_dir: Target snapshot directory.
:param config_file: Name of the snapshot configuration file.
:return: Desired instance of Snapshot.
"""
snapshot_source = KernelSource(snapshot_dir)
loaded_snapshot = cls(None, snapshot_source)
with open(os.path.join(snapshot_dir, config_file), "r") as \
snapshot_yaml:
loaded_snapshot._from_yaml(snapshot_yaml.read())
return loaded_snapshot
def test_syntax_diff():
f = "dio_iodone2_helper"
diff = ('*************** static void dio_iodone2_helper(struct dio *dio, l'
'off_t offset,\n*** 246,250 ***\n {\n! \tif (dio->end_io && dio->'
'result)\n! \t\tdio->end_io(dio->iocb, offset,\n! \t\t\t\ttransfer'
'red, dio->private, ret, is_async);\n \n--- 246,249 ---\n {\n! '
'\tif (dio->end_io)\n! \t\tdio->end_io(dio->iocb, offset, ret, dio'
'->private, 0, 0);\n \n')
source_first = KernelSource("kernel/linux-3.10.0-862.el7", True)
source_second = KernelSource("kernel/linux-3.10.0-957.el7", True)
config = Config(source_first, source_second, show_diff=True,
control_flow_only=True, verbosity=False, semdiff_tool=None)
first = source_first.get_module_for_symbol(f)
second = source_second.get_module_for_symbol(f)
fun_result = functions_diff(mod_first=first, mod_second=second,
fun_first=f, fun_second=f, glob_var=None,
config=config)
assert fun_result.inner[f].diff == diff
def source():
"""Create KernelSource shared among multiple tests."""
s = KernelSource("kernel/linux-3.10.0-957.el7", True)
yield s
s.finalize()
class TaskSpec:
"""
Task specification representing testing scenario.
Contains a list of functions to be compared with DiffKemp during the test.
"""
def __init__(self, spec, task_name, tasks_path, kernel_path):
self.old_kernel_dir = os.path.join(kernel_path, spec["old_kernel"])
self.new_kernel_dir = os.path.join(kernel_path, spec["new_kernel"])
self.name = task_name
self.task_dir = os.path.join(tasks_path, task_name)
if "control_flow_only" in spec:
self.control_flow_only = spec["control_flow_only"]
else:
self.control_flow_only = False
# Create LLVM sources and configuration
self.old_kernel = KernelSource(self.old_kernel_dir, True)
self.new_kernel = KernelSource(self.new_kernel_dir, True)
self.old_snapshot = Snapshot(self.old_kernel, self.old_kernel)
self.new_snapshot = Snapshot(self.new_kernel, self.new_kernel)
self.config = Config(self.old_snapshot, self.new_snapshot, False,
False, self.control_flow_only, False, False,
False, None)
self.functions = dict()
def finalize(self):
"""
Task finalization - should be called before the task is destroyed.
"""
self.old_kernel.finalize()
self.new_kernel.finalize()
def add_function_spec(self, fun, result):
"""Add a function comparison specification."""
self.functions[fun] = FunctionSpec(fun, result)
def build_modules_for_function(self, fun):
"""
Build LLVM modules containing definition of the compared function in
both kernels.
"""
# Since PyTest may share KernelSource objects among tasks, we need
# to explicitly initialize kernels.
self.old_kernel.initialize()
self.new_kernel.initialize()
mod_old = self.old_kernel.get_module_for_symbol(fun)
mod_new = self.new_kernel.get_module_for_symbol(fun)
self.functions[fun].old_module = mod_old
self.functions[fun].new_module = mod_new
return mod_old, mod_new
def _file_name(self, suffix, ext, name=None):
"""
Get name of a task file having the given name, suffix, and extension.
"""
return os.path.join(self.task_dir,
"{}_{}.{}".format(name or self.name, suffix, ext))
def old_llvm_file(self, name=None):
"""Name of the old LLVM file in the task dir."""
return self._file_name("old", "ll", name)
def new_llvm_file(self, name=None):
"""Name of the new LLVM file in the task dir."""
return self._file_name("new", "ll", name)
def old_src_file(self, name=None):
"""Name of the old C file in the task dir."""
return self._file_name("old", "c", name)
def new_src_file(self, name=None):
"""Name of the new C file in the task dir."""
return self._file_name("new", "c", name)
def prepare_dir(self, old_module, new_module, old_src, new_src, name=None):
"""
Create the task directory and copy the LLVM and the C files there.
:param old_module: Old LLVM module (instance of LlvmKernelModule).
:param old_src: C source from the old kernel version to be copied.
:param new_module: New LLVM module (instance of LlvmKernelModule).
:param new_src: C source from the new kernel version to be copied.
:param name: Optional parameter to specify the new file names. If None
then the spec name is used.
"""
if not os.path.isdir(self.task_dir):
os.mkdir(self.task_dir)
if not os.path.isfile(self.old_llvm_file(name)):
shutil.copyfile(old_module.llvm, self.old_llvm_file(name))
if old_src and not os.path.isfile(self.old_src_file(name)):
shutil.copyfile(old_src, self.old_src_file(name))
if not os.path.isfile(self.new_llvm_file(name)):
shutil.copyfile(new_module.llvm, self.new_llvm_file(name))
if new_src and not os.path.isfile(self.new_src_file(name)):
shutil.copyfile(new_src, self.new_src_file(name))
def source():
s = KernelSource("kernel/linux-3.10.0-957.el7", True)
yield s
s.finalize()
def compare(args):
old_functions = FunctionList(args.snapshot_dir_old)
with open(os.path.join(args.snapshot_dir_old, "functions.yaml"),
"r") as fun_list_yaml:
old_functions.from_yaml(fun_list_yaml.read())
old_source = KernelSource(args.snapshot_dir_old)
new_functions = FunctionList(args.snapshot_dir_new)
with open(os.path.join(args.snapshot_dir_new, "functions.yaml"),
"r") as fun_list_yaml:
new_functions.from_yaml(fun_list_yaml.read())
new_source = KernelSource(args.snapshot_dir_new)
if args.function:
old_functions.filter([args.function])
new_functions.filter([args.function])
config = Config(old_source, new_source, args.show_diff,
args.control_flow_only, args.verbose,
args.semdiff_tool)
result = Result(Result.Kind.NONE, args.snapshot_dir_old,
args.snapshot_dir_old)
for fun, old_mod in sorted(old_functions.functions.items()):
new_mod = new_functions.get_by_name(fun)
if not (old_mod.has_function(fun) and new_mod.has_function(fun)):
continue
fun_result = functions_diff(
mod_first=old_mod, mod_second=new_mod,
fun_first=fun, fun_second=fun,
glob_var=None, config=config)
if fun_result is not None:
if args.regex_filter is not None:
# Filter results by regex
pattern = re.compile(args.regex_filter)
for called_res in fun_result.inner.values():
if pattern.search(called_res.diff):
break
else:
fun_result.kind = Result.Kind.EQUAL_SYNTAX
result.add_inner(fun_result)
if fun_result.kind in [Result.Kind.ERROR, Result.Kind.UNKNOWN]:
print("{}: {}".format(fun, str(fun_result.kind)))
elif fun_result.kind == Result.Kind.NOT_EQUAL:
print_syntax_diff(args.snapshot_dir_old,
args.snapshot_dir_new,
fun, fun_result, False,
args.show_diff)
# Clean LLVM modules (allow GC to collect the occupied memory)
old_mod.clean_module()
new_mod.clean_module()
LlvmKernelModule.clean_all()
if args.report_stat:
print("")
print("Statistics")
print("----------")
result.report_stat(args.show_errors)
return 0
def test_get_child():
"""Test getting child of a sysctl definition."""
source = KernelSource("kernel/linux-3.10.0-862.el7", True)
kernel_module = source.get_module_from_source("kernel/sysctl.c")
sysctl_module = LlvmSysctlModule(kernel_module, "sysctl_base_table")
assert sysctl_module.get_child("vm").name == "vm_table"