def test_cdb(tmpdir, cmds_file, filter_opts):
cdb_json = os.path.join(str(tmpdir), "cdb.json")
c = Clade(tmpdir,
cmds_file,
conf={
"CDB.output": cdb_json,
"CDB.filter_opts": filter_opts
})
e = c.parse("CDB")
cdb = e.load_cdb()
assert cdb
cc = c.parse("CC")
assert len(cdb) >= len(list(cc.load_all_cmds()))
for cmd in cdb:
assert "directory" in cmd
assert "arguments" in cmd
assert "file" in cmd
for arg in cmd["arguments"]:
assert isinstance(arg, str)
if filter_opts:
assert "-fsyntax-only" not in cmd["arguments"]
def clade_api(tmpdir_factory, cmds_file):
tmpdir = tmpdir_factory.mktemp("Clade")
c = Clade(tmpdir, cmds_file)
c.parse_list(["CrossRef", "Variables", "Macros", "Typedefs", "CDB"])
yield c
def __init__(self,
conf,
logger,
parent_id,
callbacks,
mqs,
vals,
id=None,
work_dir=None,
attrs=None,
separate_from_parent=False,
include_child_resources=False,
program_fragment=None,
requirement=None,
resource_limits=None,
rerun=False):
super(VTGW,
self).__init__(conf, logger, parent_id, callbacks, mqs, vals, id,
work_dir, attrs, separate_from_parent,
include_child_resources)
self.program_fragment = program_fragment
self.requirement = requirement
self.abstract_task_desc_file = None
self.override_limits = resource_limits
self.rerun = rerun
self.session = core.session.Session(self.logger,
self.conf['Klever Bridge'],
self.conf['identifier'])
self.clade = Clade(self.conf['build base'])
def test_cc_empty_which_list(tmpdir, cmds_file):
conf = {"CC.which_list": []}
c = Clade(tmpdir, cmds_file, conf)
e = c.parse("CC")
assert len(list(e.load_all_cmds())) == 0
def test_files_to_add(tmpdir, cmds_file):
c = Clade(tmpdir, cmds_file, conf={"Storage.files_to_add": [__file__]})
c.parse("Storage")
storage_path = c.get_storage_path(__file__)
assert storage_path
assert os.path.exists(storage_path)
def __init__(self, logger, conf, tactic, pf_dir):
"""
The strategy needs a logger and configuration as the rest Klever components but also it requires Clade interface
object (uninitialized yet) and the description of the fragmentation set.
:param logger: logging Logger object.
:param conf: Dictionary.
:param tactic: Dictionary with options.
:param pf_dir: program fragments descriptions storage dir.
"""
# Simple attributes
self.logger = logger
self.conf = conf
self.tactic = tactic
self.pf_dir = pf_dir
self.files_to_keep = list()
self.project_attrs = list()
self.source_paths = self.conf['working source trees']
# Import clade
self.clade = Clade(work_dir=self.conf['build base'],
preset=self.CLADE_PRESET)
if not self.clade.work_dir_ok():
raise RuntimeError('Build base is not OK')
self.__get_project_attrs()
def __init__(self, conf, logger, parent_id, callbacks, mqs, vals, id=None, work_dir=None, attrs=None,
separate_from_parent=False, include_child_resources=False, qos_resource_limits=None, source_paths=None,
element=None):
# Read this in a callback
self.element = element
self.verdict = None
self.req_spec_id = None
self.program_fragment_id = None
self.envmodel = None
self.report_attrs = None
self.files_list_file = 'files list.txt'
self.task_error = None
self.source_paths = source_paths
self.results_key = None
self.additional_srcs = None
self.verification_task_files = None
self.__exception = None
self.__qos_resource_limit = qos_resource_limits
# Common initialization
super(RP, self).__init__(conf, logger, parent_id, callbacks, mqs, vals, id, work_dir, attrs,
separate_from_parent, include_child_resources)
self.clean_dir = True
self.session = klever.core.session.Session(self.logger, self.conf['Klever Bridge'], self.conf['identifier'])
# Obtain file prefixes that can be removed from file paths.
self.clade = Clade(self.conf['build base'])
if not self.clade.work_dir_ok():
raise RuntimeError('Build base is not OK')
self.search_dirs = klever.core.utils.get_search_dirs(self.conf['main working directory'], abs_paths=True)
self.verification_report_id = None
def __init__(self, logger, target_program_desc):
self.logger = logger
self.target_program_desc = target_program_desc
# Main working source tree where various build and auxiliary actions will be performed.
self.work_src_tree = self.target_program_desc['source code']
# Program attributes. We expect that architecture is always specified in the target program description while
# configuration and version can be either obtained during build somehow or remained unspecified.
self.architecture = self.target_program_desc['architecture']
self.configuration = None
self.version = self.target_program_desc.get('version')
# Working source trees are directories to be trimmed from file names.
self.work_src_trees = []
# Temporary directories that should be removed at the end of work.
self.tmp_dirs = []
# Path to the Clade cmds.txt file with intercepted commands
self.cmds_file = os.path.realpath(os.path.join(self.work_src_tree, 'cmds.txt'))
# Clade API object
clade_conf = dict(self._CLADE_CONF)
clade_conf.update(self.target_program_desc.get('extra Clade options', dict()))
self.clade = Clade(work_dir=self.target_program_desc['build base'],
cmds_file=self.cmds_file,
conf=clade_conf,
preset=self._CLADE_PRESET)
def build(self):
self._fetch_work_src_tree()
self._make_canonical_work_src_tree()
self._clean()
self._get_version()
if self.version:
self.logger.info('C program version is "{0}"'.format(self.version))
self._configure()
if self.configuration:
self.logger.info('C program configuration is "{0}"'.format(
self.configuration))
self._build()
if os.path.isdir(self.target_program_desc['build base']):
shutil.rmtree(self.target_program_desc['build base'])
if 'extra Clade options' in self.target_program_desc:
clade_conf = dict(self._CLADE_CONF)
clade_conf.update(self.target_program_desc['extra Clade options'])
else:
clade_conf = self._CLADE_CONF
clade = Clade(work_dir=self.target_program_desc['build base'],
cmds_file=os.path.join(self.work_src_tree, 'cmds.txt'),
conf=clade_conf,
preset=self._CLADE_PRESET)
clade.parse_list(
["CrossRef", "Callgraph", "Variables", "Typedefs", "Macros"])
self.logger.info(
'Save project attributes, working source trees and target program description to build base'
)
clade.add_meta_by_key('project attrs', [{
'name':
'project',
'value': [{
'name': 'name',
'value': type(self).__name__
}, {
'name': 'architecture',
'value': self.architecture
}, {
'name': 'version',
'value': self.version
}, {
'name': 'configuration',
'value': self.configuration
}]
}])
clade.add_meta_by_key('working source trees', self.work_src_trees)
clade.add_meta_by_key('target program description',
self.target_program_desc)
self.logger.info('Remove temporary directories')
for tmp_dir in self.tmp_dirs:
shutil.rmtree(tmp_dir)
def test_src_graph_empty_conf(tmpdir, cmds_file):
c = Clade(tmpdir, cmds_file)
e = c.parse("SrcGraph")
src_graph = e.load_src_graph()
assert src_graph
assert len(src_graph[test_file]["used_by"]) >= 1
def create_source_representation(logger, conf, abstract_task):
"""
Create Source object.
:param logger: Logger object.
:param conf: Conf dict.
:param abstract_task: Abstract task dict.
:return: Source object.
"""
# Initialize Clade client to make requests
clade = Clade(conf['build base'])
if not clade.work_dir_ok():
raise RuntimeError('Build base is not OK')
prefixes = _prefixes(conf, clade)
# Ask for dependencies for each CC
cfiles, dep_paths, files_map = _collect_file_dependencies(
clade, abstract_task)
# Read file with source analysis
collection = Source(cfiles, prefixes, dep_paths)
collection.c_full_paths = _c_full_paths(collection, cfiles)
_import_code_analysis(logger, conf, clade, files_map, collection)
if conf.get('dump types'):
dump_types('type collection.json')
if conf.get('dump source code analysis'):
collection.dump('vars.json', 'functions.json', 'macros.json')
return collection
def test_cmd_graph_empty_requires(tmpdir, cmds_file):
conf = {"CmdGraph.requires": []}
c = Clade(tmpdir, cmds_file, conf)
with pytest.raises(RuntimeError):
c.parse("CmdGraph")
def __init__(self, logger, conf, desc, pf_dir):
"""
The strategy needs a logger and configuration as the rest Klever components but also it requires Clade interface
object (uninitialized yet) and the description of the fragmentation set.
:param logger: logging Logger object.
:param conf: Dictionary.
:param desc: Dictionary.
:param clade: Clade interface.
:param pf_dir: program fragments descriptions storage dir.
"""
# Simple attributes
self.logger = logger
self.conf = conf
self.fragmentation_set_conf = desc
self.pf_dir = pf_dir
self.files_to_keep = list()
self.common_attributes = list()
# Import clade
self.clade = Clade(work_dir=self.conf['build base'], preset=self.CLADE_PRESET)
# Complex attributes
self.source_paths = self.__retrieve_source_paths()
self.attributes = self.__attributes()
def test_cc_preprocess(tmpdir, cmds_file):
conf = {"Compiler.preprocess_cmds": True}
c = Clade(tmpdir, cmds_file, conf)
e = c.parse("CC")
assert e.get_all_pre_files()
def _make(self,
*target,
opts=None,
env=None,
intercept_build_cmds=False,
get_output=False):
if opts is None:
opts = []
cmd = ['make', '-j', self.jobs] + opts + list(target)
if intercept_build_cmds:
clade = Clade(cmds_file=os.path.realpath(
os.path.join(self.work_src_tree, 'cmds.txt')))
r = clade.intercept(cmd, append=True, cwd=self.work_src_tree)
if r:
raise RuntimeError('Build failed')
return r
else:
return execute_cmd(self.logger,
*(cmd),
cwd=self.work_src_tree,
env=env,
get_output=get_output)
def test_pid_graph_as_picture(tmpdir, cmds_file, as_picture):
conf = {"PidGraph.as_picture": as_picture}
c = Clade(tmpdir, cmds_file, conf)
e = c.parse("PidGraph")
assert os.path.exists(e.graph_dot) == as_picture
assert os.path.exists(e.graph_dot + ".pdf") == as_picture
def __init__(self, work_dir1, work_dir2, log_level="INFO"):
self.work_dir1 = work_dir1
self.work_dir2 = work_dir2
self.cl1 = Clade(work_dir1)
self.cl2 = Clade(work_dir2)
logger.setLevel(log_level)
def test_intercept(tmpdir):
output = os.path.join(str(tmpdir), "cmds.txt")
c = Clade(cmds_file=output)
assert not c.intercept(command=test_project_make, use_wrappers=True)
assert os.path.isfile(output)
assert calculate_loc(output) > 1
def envs_file():
# Disable multiprocessing
os.environ["CLADE_DEBUG"] = "1"
c = Clade(work_dir=test_project + '/clade')
c.intercept(command=test_project_make,
use_wrappers=True,
intercept_envs=True)
yield os.path.join(c.work_dir, "envs.txt")
def test_cc_include_list(tmpdir, cmds_file, include_list):
conf = {
"Common.include_list": include_list
}
c = Clade(tmpdir, cmds_file, conf)
e = c.parse("CC")
assert len(list(e.load_all_cmds()))
def test_objcopy(tmpdir, cmds_file):
c = Clade(tmpdir, cmds_file)
e = c.parse("Objcopy")
cmds = e.load_all_cmds(with_opts=True, with_raw=True)
assert len(cmds) == 2
for cmd in cmds:
assert len(cmd["in"]) == 1
assert len(cmd["out"]) == 1
assert len(cmd["opts"]) == 1
def test_cc_load_deps_by_id(tmpdir, cmds_file):
c = Clade(tmpdir, cmds_file)
e = c.parse("CC")
for cmd in e.load_all_cmds(compile_only=True):
deps = e.load_deps_by_id(cmd["id"])
assert deps
for cmd_in in cmd["in"]:
assert cmd_in in deps
def test_storage_encoding(tmpdir, encoding):
c = Clade(tmpdir, conf={"Storage.convert_to_utf8": True})
bstr = "мир".encode("cp1251")
test_file = os.path.join(str(tmpdir), "test")
with open(test_file, "wb") as fh:
fh.write(bstr)
c.add_file_to_storage(test_file, encoding=encoding)
def test_cc_parallel(tmpdir, cmds_file):
del os.environ["CLADE_DEBUG"]
try:
c = Clade(tmpdir, cmds_file)
e = c.parse("CC")
assert e.load_all_cmds()
finally:
os.environ["CLADE_DEBUG"] = "1"
def test_ar(tmpdir, cmds_file):
c = Clade(tmpdir, cmds_file)
e = c.parse("AR")
cmds = e.load_all_cmds(with_opts=True, with_raw=True)
assert len(cmds) == 1
assert len(cmds[0]["in"]) == 2
assert len(cmds[0]["out"]) == 1
assert len(cmds[0]["opts"]) == 1
assert len(cmds[0]["command"]) == 5
def test_cmd_graph_as_picture(tmpdir, cmds_file, as_picture):
if not shutil.which("dot"):
return
conf = {"CmdGraph.as_picture": as_picture}
c = Clade(tmpdir, cmds_file, conf)
e = c.parse("CmdGraph")
assert os.path.exists(e.graph_dot) == as_picture
assert os.path.exists(e.graph_dot + ".pdf") == as_picture
def test_cc_exclude_list(tmpdir, cmds_file, exclude_list, exclude_list_in, exclude_list_out):
conf = {
"Common.exclude_list": exclude_list,
"Common.exclude_list_in": exclude_list_in,
"Common.exclude_list_out": exclude_list_out
}
c = Clade(tmpdir, cmds_file, conf)
e = c.parse("CC")
assert len(list(e.load_all_cmds()))
def test_cc_filter_deps(tmpdir, cmds_file, compile_only):
c = Clade(tmpdir, cmds_file)
e = c.parse("CC")
found_deps_opt = False
for cmd in e.load_all_cmds(with_opts=True, compile_only=compile_only):
if set(cc_preprocessor_opts).intersection(cmd["opts"]):
found_deps_opt = True
assert compile_only != found_deps_opt
def test_tracer(tmpdir, cmds_file):
c = Clade(tmpdir, cmds_file, preset="klever_linux_kernel")
c.parse_list(c.conf["extensions"])
print(c.work_dir)
t = Tracer(c.work_dir)
from_func = t.find_functions(["main"])[0]
to_func = t.find_functions(["printf"])[0]
trace = t.trace(from_func, to_func)
assert len(trace) == 2
def test_cc_filter(tmpdir, cmds_file, filter, filter_in, filter_out):
conf = {
"Common.filter": filter,
"Common.filter_in": filter_in,
"Common.filter_out": filter_out
}
c = Clade(tmpdir, cmds_file, conf)
e = c.parse("CC")
assert len(list(e.load_all_cmds()))