class VTGW(core.components.Component):
def __init__(self,
conf,
logger,
parent_id,
callbacks,
mqs,
vals,
id=None,
work_dir=None,
attrs=None,
separate_from_parent=False,
include_child_resources=False,
program_fragment=None,
requirement=None,
resource_limits=None,
rerun=False):
super(VTGW,
self).__init__(conf, logger, parent_id, callbacks, mqs, vals, id,
work_dir, attrs, separate_from_parent,
include_child_resources)
self.program_fragment = program_fragment
self.requirement = requirement
self.abstract_task_desc_file = None
self.override_limits = resource_limits
self.rerun = rerun
self.session = core.session.Session(self.logger,
self.conf['Klever Bridge'],
self.conf['identifier'])
self.clade = Clade(self.conf['build base'])
def tasks_generator_worker(self):
files_list_file = 'files list.txt'
with open(files_list_file, 'w', encoding='utf8') as fp:
fp.writelines('\n'.join(
sorted(f for grp in self.program_fragment['grps']
for f in grp['files'])))
core.utils.report(self.logger,
'attrs', {
'id':
self.id,
'attrs': [{
"name": "Program fragment",
"value": self.program_fragment['id'],
"data": files_list_file,
"compare": True,
"associate": True
}]
},
self.mqs['report files'],
self.vals['report id'],
self.conf['main working directory'],
data_files=[files_list_file])
try:
self.generate_abstact_verification_task_desc(
self.program_fragment, self.requirement)
if not self.vals['task solving flag'].value:
with self.vals['task solving flag'].get_lock():
self.vals['task solving flag'].value = 1
except Exception:
self.plugin_fail_processing()
raise
finally:
self.session.sign_out()
main = tasks_generator_worker
def generate_abstact_verification_task_desc(self, program_fragment_desc,
requirement_desc):
"""Has a callback!"""
self.logger.info(
"Start generating tasks for program fragment {!r} and requirement {!r}"
.format(program_fragment_desc['id'], requirement_desc['id']))
program_fragment = program_fragment_desc['id']
self.requirement = requirement_desc['id']
# Prepare pilot workdirs if it will be possible to reuse data
requirement_class = resolve_requirement_class(requirement_desc['id'])
pilot_requirement = _requirement_classes[requirement_class][0]['id']
pilot_plugins_work_dir = os.path.join(os.path.pardir,
pilot_requirement)
# Initial abstract verification task looks like corresponding program fragment.
initial_abstract_task_desc = copy.deepcopy(program_fragment_desc)
initial_abstract_task_desc['id'] = '{0}/{1}'.format(
program_fragment, self.requirement)
initial_abstract_task_desc['fragment'] = program_fragment
initial_abstract_task_desc['attrs'] = ()
for grp in initial_abstract_task_desc['grps']:
grp['Extra CCs'] = []
for cc in grp['CCs']:
in_file = self.clade.get_cmd(cc)['in'][0]
grp['Extra CCs'].append({'CC': cc, 'in file': in_file})
del (grp['CCs'])
initial_abstract_task_desc_file = 'initial abstract task.json'
self.logger.debug(
'Put initial abstract verification task description to file "{0}"'.
format(initial_abstract_task_desc_file))
with open(initial_abstract_task_desc_file, 'w', encoding='utf8') as fp:
json.dump(initial_abstract_task_desc,
fp,
ensure_ascii=False,
sort_keys=True,
indent=4)
# Invoke all plugins one by one.
cur_abstract_task_desc_file = initial_abstract_task_desc_file
out_abstract_task_desc_file = None
if self.rerun:
# Get only the last, and note that the last one prepares tasks and otherwise rerun should not be set
plugins = [requirement_desc['plugins'][-1]]
else:
plugins = requirement_desc['plugins']
for plugin_desc in plugins:
# Here plugin will put modified abstract verification task description.
plugin_work_dir = plugin_desc['name'].lower()
out_abstract_task_desc_file = '{0} abstract task.json'.format(
plugin_desc['name'].lower())
if self.rerun:
self.logger.info(
"Instead of running the {!r} plugin for the {!r} requirement in the same dir obtain "
"results for the original run".format(
plugin_desc['name'], self.requirement))
cur_abstract_task_desc_file = os.path.join(
os.pardir, out_abstract_task_desc_file)
os.symlink(
os.path.relpath(cur_abstract_task_desc_file,
os.path.curdir),
out_abstract_task_desc_file)
if requirement_desc['id'] not in [c[0]['id'] for c in _requirement_classes.values()] and \
plugin_desc['name'] in ['SA', 'EMG']:
# Expect that there is a work directory which has all prepared
# Make symlinks to the pilot requirement work dir
self.logger.info(
"Instead of running the {!r} plugin for the {!r} requirement lets use already obtained"
" results for the {!r} requirement".format(
plugin_desc['name'], self.requirement,
pilot_requirement))
pilot_plugin_work_dir = os.path.join(
pilot_plugins_work_dir, plugin_desc['name'].lower())
pilot_abstract_task_desc_file = os.path.join(
pilot_plugins_work_dir, '{0} abstract task.json'.format(
plugin_desc['name'].lower()))
os.symlink(
os.path.relpath(pilot_abstract_task_desc_file,
os.path.curdir),
out_abstract_task_desc_file)
os.symlink(
os.path.relpath(pilot_plugin_work_dir, os.path.curdir),
plugin_work_dir)
else:
self.logger.info('Launch plugin {0}'.format(
plugin_desc['name']))
# Get plugin configuration on the basis of common configuration, plugin options specific for requirement
# specification and information on requirement itself. In addition put either initial or
# current description of abstract verification task into plugin configuration.
plugin_conf = copy.deepcopy(self.conf)
if 'options' in plugin_desc:
plugin_conf.update(plugin_desc['options'])
if 'bug kinds' in requirement_desc:
plugin_conf.update(
{'bug kinds': requirement_desc['bug kinds']})
plugin_conf['in abstract task desc file'] = os.path.relpath(
cur_abstract_task_desc_file,
self.conf['main working directory'])
plugin_conf['out abstract task desc file'] = os.path.relpath(
out_abstract_task_desc_file,
self.conf['main working directory'])
plugin_conf['solution class'] = self.requirement
plugin_conf['override resource limits'] = self.override_limits
plugin_conf_file = '{0} conf.json'.format(
plugin_desc['name'].lower())
self.logger.debug(
'Put configuration of plugin "{0}" to file "{1}"'.format(
plugin_desc['name'], plugin_conf_file))
with open(plugin_conf_file, 'w', encoding='utf8') as fp:
json.dump(plugin_conf,
fp,
ensure_ascii=False,
sort_keys=True,
indent=4)
try:
p = plugin_desc['plugin'](plugin_conf,
self.logger,
self.id,
self.callbacks,
self.mqs,
self.vals,
plugin_desc['name'],
plugin_work_dir,
separate_from_parent=True,
include_child_resources=True)
p.start()
p.join()
except core.components.ComponentError:
self.plugin_fail_processing()
break
if self.requirement in [c[0]['id'] for c in _requirement_classes.values()] and \
plugin_desc['name'] == 'EMG':
self.logger.debug(
"Signal to VTG that the cache preapred for the requirement {!r} is ready for the "
"further use".format(pilot_requirement))
self.mqs['prepared verification tasks'].put(
(program_fragment, self.requirement))
cur_abstract_task_desc_file = out_abstract_task_desc_file
else:
final_abstract_task_desc_file = 'final abstract task.json'
if not self.rerun:
self.logger.debug(
'Put final abstract verification task description to file "{0}"'
.format(final_abstract_task_desc_file))
# Final abstract verification task description equals to abstract verification task description received
# from last plugin.
os.symlink(
os.path.relpath(out_abstract_task_desc_file,
os.path.curdir),
final_abstract_task_desc_file)
# VTG will consume this abstract verification task description file.
self.abstract_task_desc_file = out_abstract_task_desc_file
if os.path.isfile(os.path.join(plugin_work_dir, 'task.json')) and \
os.path.isfile(os.path.join(plugin_work_dir, 'task files.zip')):
task_id = self.session.schedule_task(
os.path.join(plugin_work_dir, 'task.json'),
os.path.join(plugin_work_dir, 'task files.zip'))
with open(self.abstract_task_desc_file, 'r',
encoding='utf8') as fp:
final_task_data = json.load(fp)
# Plan for checking status
self.mqs['pending tasks'].put([[
str(task_id), final_task_data["result processing"],
self.program_fragment, self.requirement,
final_task_data['verifier']
], self.rerun])
self.logger.info(
"Submitted successfully verification task {} for solution".
format(os.path.join(plugin_work_dir, 'task.json')))
else:
self.logger.warning(
"There is no verification task generated by the last plugin, expect {}"
.format(os.path.join(plugin_work_dir, 'task.json')))
self.mqs['processed tasks'].put(
(program_fragment, self.requirement, [None, None, None]))
def plugin_fail_processing(self):
"""The function has a callback in sub-job processing!"""
self.logger.debug("VTGW that processed {!r}, {!r} failed".format(
self.program_fragment['id'], self.requirement))
self.mqs['processed tasks'].put(
(self.program_fragment['id'], self.requirement, [None, None,
None]))
def join(self, timeout=None, stopped=False):
try:
ret = super(VTGW, self).join(timeout, stopped)
finally:
if not self.conf['keep intermediate files'] and not self.is_alive(
):
self.logger.debug(
"Indicate that the working directory can be deleted for: {!r}, {!r}"
.format(self.program_fragment['id'],
self.requirement['id']))
self.mqs['delete dir'].put(
[self.program_fragment['id'], self.requirement['id']])
return ret
def weave(self):
self.abstract_task_desc.setdefault('extra C files', dict())
clade = Clade(self.conf['build base'])
if not clade.work_dir_ok():
raise RuntimeError('Build base is not OK')
meta = clade.get_meta()
# This is required to get compiler (Aspectator) specific stdarg.h since kernel C files are compiled
# with "-nostdinc" option and system stdarg.h couldn't be used.
aspectator_search_dir = '-isystem' + klever.core.utils.execute(
self.logger, (klever.core.vtg.utils.get_cif_or_aspectator_exec(
self.conf, 'aspectator'), '-print-file-name=include'),
collect_all_stdout=True)[0]
env = dict(os.environ)
# Print stubs instead of inline Assembler since verifiers do not interpret it and even can fail.
env['LDV_INLINE_ASM_STUB'] = ''
for grp in self.abstract_task_desc['grps']:
self.logger.info('Weave in C files of group "{0}"'.format(
grp['id']))
for extra_cc in grp['Extra CCs']:
# Each CC is either pair (compiler command identifier, compiler command type) or JSON file name
# with compiler command description.
if isinstance(extra_cc['CC'], list):
cc = clade.get_cmd(*extra_cc['CC'], with_opts=True)
if "in file" in extra_cc:
# This is for CC commands with several input files
infile = extra_cc["in file"]
else:
infile = cc["in"][0]
infile = clade.get_storage_path(infile)
if meta['conf'].get('Compiler.preprocess_cmds', False):
infile = infile('.c')[0] + '.i'
else:
with open(os.path.join(self.conf['main working directory'],
extra_cc['CC']),
encoding='utf8') as fp:
cc = json.load(fp)
infile = cc["in"][0]
# Distinguish source files having the same names.
outfile_unique = '{0}.c'.format(
klever.core.utils.unique_file_name(
os.path.splitext(os.path.basename(infile))[0], '.c'))
# This is used for storing/getting to/from cache where uniqueness is guaranteed by other means.
outfile = '{0}.c'.format(
os.path.splitext(os.path.basename(infile))[0])
self.logger.info('Weave in C file "{0}"'.format(infile))
# Produce aspect to be weaved in.
if 'plugin aspects' in extra_cc:
self.logger.info(
'Concatenate all aspects of all plugins together')
# Resulting aspect.
aspect = 'aspect'
# Get all aspects. Place RSG aspects at beginning since they can instrument entities added by
# aspects of other plugins while corresponding function declarations still need be at beginning
# of file.
aspects = []
for plugin_aspects in extra_cc['plugin aspects']:
if plugin_aspects['plugin'] == 'RSG':
aspects[0:0] = plugin_aspects['aspects']
else:
aspects.extend(plugin_aspects['aspects'])
# Concatenate aspects.
with open(aspect, 'w',
encoding='utf8') as fout, fileinput.input(
[
os.path.join(
self.conf['main working directory'],
aspect) for aspect in aspects
],
openhook=fileinput.hook_encoded(
'utf8')) as fin:
for line in fin:
fout.write(line)
else:
# Instrumentation is not required when there is no aspects. But we will still pass source files
# through C-backend to make resulting code to look similarly and thus to avoid different issues
# at merging source files and models together.
aspect = None
if aspect:
self.logger.info(
'Aspect to be weaved in is "{0}"'.format(aspect))
else:
self.logger.info(
'C file will be passed through C Back-end only')
cwd = clade.get_storage_path(cc['cwd'])
is_model = (grp['id'] == 'models')
# Original sources should be woven in and we do not need to get cross references for them since this
# was already done before.
if not is_model:
self.__weave(infile, cc['opts'], aspect, outfile_unique,
clade, env, cwd, aspectator_search_dir,
is_model)
# For generated models we need to weave them in (actually, just pass through C Back-end) and to get
# cross references always since most likely they all are different.
elif 'generated' in extra_cc:
self.__weave(infile, cc['opts'], aspect, outfile_unique,
clade, env, cwd, aspectator_search_dir,
is_model)
if self.conf[
'code coverage details'] != 'Original C source files':
self.__get_cross_refs(infile, cc['opts'],
outfile_unique, clade, cwd,
aspectator_search_dir)
# For non-generated models use results cache in addition.
else:
cache_dir = os.path.join(
self.conf['cache directory'],
klever.core.utils.get_file_name_checksum(infile))
with klever.core.utils.LockedOpen(cache_dir + '.tmp', 'w'):
if os.path.exists(cache_dir):
self.logger.info('Get woven in C file from cache')
self.abstract_task_desc['extra C files'].append({
'C file':
os.path.relpath(
os.path.join(cache_dir,
os.path.basename(outfile)),
self.conf['main working directory'])
})
if self.conf[
'code coverage details'] != 'Original C source files':
self.logger.info(
'Get cross references from cache')
self.__merge_additional_srcs(
os.path.join(cache_dir,
'additional sources'))
else:
os.makedirs(cache_dir)
self.__weave(infile, cc['opts'], aspect,
outfile_unique, clade, env, cwd,
aspectator_search_dir, is_model)
self.logger.info('Store woven in C file to cache')
shutil.copy(outfile_unique,
os.path.join(cache_dir, outfile))
if self.conf[
'code coverage details'] != 'Original C source files':
self.__get_cross_refs(infile, cc['opts'],
outfile_unique, clade,
cwd,
aspectator_search_dir)
self.logger.info(
'Store cross references to cache')
shutil.copytree(
outfile_unique + ' additional sources',
os.path.join(cache_dir,
'additional sources'))
# For auxiliary files there is no cross references since it is rather hard to get them from Aspectator. But
# there still highlighting.
if self.conf['code coverage details'] == 'All source files':
for aux_file in glob.glob('*.aux'):
new_file = os.path.join(
'additional sources', 'generated models',
os.path.relpath(aux_file,
self.conf['main working directory']))
os.makedirs(os.path.dirname(new_file), exist_ok=True)
shutil.copy(aux_file, new_file)
cross_refs = CrossRefs(self.conf, self.logger, clade, aux_file,
new_file, self.search_dirs)
cross_refs.get_cross_refs()
self.abstract_task_desc['additional sources'] = os.path.relpath('additional sources',
self.conf['main working directory']) \
if os.path.isdir('additional sources') else None
# Copy additional sources for total code coverage.
if self.conf['code coverage details'] != 'Original C source files':
with klever.core.utils.Cd('additional sources'):
for root, dirs, files in os.walk(os.path.curdir):
for file in files:
# These files are handled below in addition to corresponding source files.
if file.endswith('.json'):
continue
if self.conf['code coverage details'] == 'C source files including models' \
and not file.endswith('.c'):
continue
file = os.path.join(root, file)
new_file = os.path.join(
self.conf['additional sources directory'], file)
os.makedirs(os.path.dirname(new_file), exist_ok=True)
with klever.core.utils.LockedOpen(
new_file + '.tmp', 'w'):
if os.path.isfile(new_file):
os.remove(new_file + '.tmp')
continue
shutil.copy(file, new_file)
shutil.copy(file + '.idx.json',
new_file + '.idx.json')
os.remove(new_file + '.tmp')
# These sections won't be refereed any more.
del (self.abstract_task_desc['grps'])
del (self.abstract_task_desc['deps'])
def request_arg_signs(self):
self.logger.info('Request argument signatures')
clade = Clade(work_dir=self.conf['build base'])
for request_aspect in self.conf['request aspects']:
request_aspect = core.vtg.utils.find_file_or_dir(
self.logger, self.conf['main working directory'],
request_aspect)
self.logger.debug('Request aspect is "{0}"'.format(request_aspect))
# This is required to get compiler (Aspectator) specific stdarg.h since kernel C files are compiled with
# "-nostdinc" option and system stdarg.h couldn't be used.
aspectator_search_dir = '-isystem' + core.utils.execute(
self.logger, ('aspectator', '-print-file-name=include'),
collect_all_stdout=True)[0]
for grp in self.abstract_task_desc['grps']:
self.logger.info(
'Request argument signatures for C files of group "{0}"'.
format(grp['id']))
for extra_cc in grp['Extra CCs']:
self.logger.info(
'Request argument signatures for C file "{0}"'.format(
extra_cc['in file']))
cc = clade.get_cmd(extra_cc['CC'], with_opts=True)
env = dict(os.environ)
env['LDV_ARG_SIGNS_FILE'] = os.path.realpath(
os.path.splitext(
os.path.splitext(
os.path.basename(request_aspect))[0])[0])
# Add plugin aspects produced thus far (by EMG) since they can include additional headers for which
# additional argument signatures should be extracted. Like in Weaver.
if 'plugin aspects' in extra_cc:
self.logger.info(
'Concatenate all aspects of all plugins together')
# Resulting request aspect.
aspect = '{0}.aspect'.format(
core.utils.unique_file_name(
os.path.splitext(os.path.basename(
cc['out'][0]))[0], '.aspect'))
# Get all aspects. Place original request aspect at beginning since it can instrument entities
# added by aspects of other plugins while corresponding function declarations still need be at
# beginning of file.
aspects = [
os.path.relpath(
request_aspect,
self.conf['main working directory'])
]
for plugin_aspects in extra_cc['plugin aspects']:
aspects.extend(plugin_aspects['aspects'])
# Concatenate aspects.
with open(
aspect, 'w', encoding='utf8'
) as fout, fileinput.input(
[
os.path.join(
self.conf['main working directory'],
aspect) for aspect in aspects
],
openhook=fileinput.hook_encoded(
'utf8')) as fin:
for line in fin:
fout.write(line)
else:
aspect = request_aspect
core.utils.execute(
self.logger,
tuple([
'cif', '--in',
clade.get_storage_path(cc['in'][0]), '--aspect',
os.path.realpath(aspect), '--stage',
'instrumentation', '--out',
os.path.realpath('{0}.c'.format(
core.utils.unique_file_name(
os.path.splitext(
os.path.basename(cc['out'][0]))[0],
'.c.aux'))), '--debug', 'DEBUG'
] + (
['--keep'] if self.
conf['keep intermediate files'] else []
) + ['--'] + core.vtg.utils.prepare_cif_opts(
self.conf, cc['opts'], clade.storage_dir
) + [
# Besides header files specific for requirements will be
# searched for.
'-I' + os.path.realpath(
os.path.dirname(self.conf['requirements DB'])),
aspectator_search_dir
]),
env,
cwd=clade.get_storage_path(cc['cwd']),
timeout=0.01,
filter_func=core.vtg.utils.CIFErrorFilter())
class Source:
"""
Representation of a collection with the data collected by a source code analysis. The collection contains
information about functions, variable initializations, a functions call graph, macros.
"""
def __init__(self, logger, conf, abstract_task):
"""
Setup initial attributes and get logger object.
:param logger: logging object.
:param conf: Source code analysis configuration.
:param abstract_task: Abstract verification task dictionary (given by VTG).
:param conf: Configuration properties dictionary.
"""
self.logger = logger
self._conf = conf
self._source_functions = dict()
self._source_vars = dict()
self._macros = dict()
self.__function_calls_cache = dict()
# Initialize Clade cient to make requests
self._clade = Clade(self._conf['build base'])
# Ask for dependencies for each CC
cfiles, files_map = self._collect_file_dependencies(abstract_task)
# Read file with source analysis
self._import_code_analysis(cfiles, files_map)
@property
def source_functions(self):
"""
Return a list of function names.
:return: function names list.
"""
return list(self._source_functions.keys())
def get_source_function(self, name, path=None, declaration=None):
"""
Provides the function by a given name from the collection.
:param name: Function name.
:param path: File where the function should be declared or defined.
:param declaration: Declaration object representing the function of interest.
:return: Function object or None.
"""
name = self.refined_name(name)
if name and name in self._source_functions:
if path and path in self._source_functions[name]:
return self._source_functions[name][path]
else:
functions = self.get_source_functions(name,
declaration=declaration)
if len(functions) == 1:
return functions[0]
elif len(functions) > 1:
raise ValueError(
"There are several definitions of function {!r} in provided code you must specify "
"scope".format(name))
return None
def get_source_functions(self, name, declaration=None):
"""
Provides all functions found by a given name from the collection.
:param name: Function name.
:param declaration: Declaration object representing the function of interest.
:return: List with Function objects.
"""
name = self.refined_name(name)
result = []
if name and name in self._source_functions:
for func in self._source_functions[name].values():
if func not in result and (
not declaration or
(declaration and declaration.compare(func.declaration))):
result.append(func)
return result
def set_source_function(self, new_obj, path):
"""
Replace an Function object in the collection.
:param new_obj: Function object.
:param path: File where the function should be declared or defined.
:return: None.
"""
if new_obj.name not in self._source_functions:
self._source_functions[new_obj.name] = dict()
self._source_functions[new_obj.name][path] = new_obj
def remove_source_function(self, name):
"""
Delete the function from the collection.
:param name: Function name.
:return: None.
"""
del self._source_functions[name]
@property
def source_variables(self):
"""
Return list of global variables.
:return: Variable names list.
"""
return list(self._source_vars.keys())
def get_source_variable(self, name, path=None):
"""
Provides a gloabal variable by a given name and scope file from the collection.
:param name: Variable name.
:param path: File with the variable declaration or initialization.
:return: Variable object or None.
"""
name = self.refined_name(name)
if name and name in self._source_vars:
if path and path in self._source_vars[name]:
return self._source_vars[name][path]
else:
variables = self.get_source_variables(name)
if len(variables) == 1:
return variables[0]
return None
def get_source_variables(self, name):
"""
Provides all global variables by a given name from the collection.
:param name: Variable name.
:return: List with Variable objects.
"""
name = self.refined_name(name)
result = []
if name and name in self._source_vars:
for var in self._source_vars[name].values():
if var not in result:
result.append(var)
return result
def set_source_variable(self, new_obj, path):
"""
Replace an object in global variables collection.
:param new_obj: Variable object.
:param path: File with the variable declaration or initialization.
:return: None.
"""
if new_obj.name not in self._source_vars:
self._source_vars[new_obj.name] = dict()
self._source_vars[new_obj.name][path] = new_obj
def remove_source_variable(self, name):
"""
Delete the global variable from the collection.
:param name: Variable name.
:return: None.
"""
del self._source_vars[name]
def get_macro(self, name):
"""
Provides a macro by a given name from the collection.
:param name: Macro name.
:return: Macro object or None.
"""
if name in self._macros:
return self._macros[name]
else:
return None
def set_macro(self, new_obj):
"""
Set or replace an object in macros collection.
:param new_obj: Macro object.
:return: None.
"""
self._macros[new_obj.name] = new_obj
def remove_macro(self, name):
"""
Delete the macro from the collection.
:param name: Macro name.
:return: None.
"""
del self._macros[name]
@staticmethod
def refined_name(call):
"""
Resolve function name from simple expressions which contains explicit function name like '& myfunc', '(myfunc)',
'(& myfunc)' or 'myfunc'.
:param call: An expression string.
:return: Extracted function name string.
"""
name_re = re.compile("\(?\s*&?\s*(\w+)\s*\)?$")
if name_re.fullmatch(call):
return name_re.fullmatch(call).group(1)
else:
return None
def _import_code_analysis(self, cfiles, dependencies):
"""
Read global variables, functions and macros to fill up the collection.
:param source_analysis: Dictionary with the content of source analysis.
:param files_map: Dictionary to resolve main file by an included file.
:return: None.
"""
# Import typedefs if there are provided
self.logger.info("Extract complete types definitions")
typedef = self._clade.get_typedefs(
set(dependencies.keys()).union(cfiles))
if typedef:
import_typedefs(typedef, dependencies)
variables = self._clade.get_variables(cfiles)
if variables:
self.logger.info("Import global variables initializations")
for path, vals in variables.items():
for variable in vals:
variable_name = extract_name(variable['declaration'])
if not variable_name:
raise ValueError('Global variable without a name')
var = Variable(variable_name, variable['declaration'])
# Here we know, that if we met a variable in an another file then it is an another variable because
# a program should contain a single global variable initialization
self.set_source_variable(var, path)
var.declaration_files.add(path)
var.initialization_file = path
var.static = is_static(variable['declaration'])
if 'value' in variable:
var.value = variable['value']
# Variables which are used in variables initalizations
self.logger.info("Import source functions")
vfunctions = self._clade.get_used_in_vars_functions()
# Get functions defined in dependencies and in the main functions and have calls
cg = self._clade.get_callgraph(set(dependencies.keys()))
# Function scope definitions
# todo: maybe this should be fixed in Clade
# As we will not get definitions for library functions if there are in compiled parts we should add all scopes
# that are given for all function called from outside of the code we analyze
for scope in (s for s in cfiles if s in cg):
for func in (f for f in cg[scope] if cg[scope][f].get('calls')):
for dep in cg[scope][func].get('calls'):
dependencies.setdefault(dep, set())
dependencies[dep].add(scope)
fs = self._clade.get_functions_by_file(
set(dependencies.keys()).union(cfiles))
# Add called functions
for scope in cg:
for func in cg[scope]:
desc = cg[scope][func]
if scope in cfiles:
# Definition of the function is in the code of interest
self._add_function(func, scope, fs, dependencies, cfiles)
# Add called functions
for def_scope, cf_desc in desc.get('calls',
dict()).items():
if def_scope not in cfiles:
for called_func in (
f for f in cf_desc
if def_scope in fs and f in fs[def_scope]):
self._add_function(called_func, def_scope, fs,
dependencies, cfiles)
elif ('called_in' in desc
and set(desc['called_in'].keys()).intersection(cfiles)
) or func in vfunctions:
if scope in fs and func in fs[scope]:
# Function is called in the target code but defined in dependencies
self._add_function(func, scope, fs, dependencies,
cfiles)
elif scope != 'unknown':
self.logger.warning(
"There is no information on declarations of function {!r} from {!r} scope"
.format(func, scope))
# Add functions missed in the call graph
for scope in (s for s in fs if s in cfiles):
for func in fs[scope]:
func_intf = self.get_source_function(func, scope)
if not func_intf:
self._add_function(func, scope, fs, dependencies, cfiles)
for func in self.source_functions:
for obj in self.get_source_functions(func):
scopes = set(obj.declaration_files).union(set(
obj.header_files))
if not obj.definition_file:
# It is likely be this way
scopes.add('unknown')
for scope in (s for s in scopes
if cg.get(s, dict()).get(func)):
for cscope, desc in ((s, d)
for s, d in cg[scope][func].get(
'called_in', {}).items()
if s in cfiles):
for caller in desc:
for line in desc[caller]:
params = desc[caller][line].get('args')
caller_intf = self.get_source_function(
caller, cscope)
obj.add_call(caller, cscope)
if params:
# Here can be functions which are not defined or visible
for _, passed_func in list(params):
passed_obj = self.get_source_function(
passed_func, cscope)
if not passed_obj:
passed_scope = self._search_function(
passed_func, cscope, fs)
if passed_scope:
self._add_function(
passed_func, passed_scope,
fs, dependencies, cfiles)
else:
self.logger.warning(
"Cannot find function {!r} from scope {!r}"
.format(
passed_func, cscope))
# Ignore this call since model will not be correct without signature
params = None
break
caller_intf.call_in_function(obj, params)
macros_file = get_conf_property(self._conf['source analysis'],
'macros white list')
if macros_file:
macros_file = find_file_or_dir(
self.logger, self._conf['main working directory'], macros_file)
with open(macros_file, 'r', encoding='utf8') as fp:
white_list = ujson.load(fp)
if white_list:
macros = self._clade.get_macros_expansions(cfiles, white_list)
for path, macros in macros.items():
for macro, desc in macros.items():
obj = self.get_macro(macro)
if not obj:
obj = Macro(macro)
for call in desc.get('args', []):
obj.add_parameters(path, call)
self.set_macro(obj)
def _search_function(self, func_name, some_scope, fs):
# Be aware of this funciton - it is costly
if some_scope in fs and func_name in fs[some_scope]:
return some_scope
elif 'unknown' in fs and func_name in fs['unknown']:
return 'unknown'
else:
for s in (s for s in fs if func_name in fs[s]):
return s
return None
def _add_function(self, func, scope, fs, deps, cfiles):
fs_desc = fs[scope][func]
if scope == 'unknown':
key = list(fs_desc['declarations'].keys())[0]
signature = fs_desc['declarations'][key]['signature']
func_intf = Function(func, signature)
# Do not set definition file since it is out of scope of the target program fragment
else:
signature = fs_desc.get('signature')
func_intf = Function(func, signature)
func_intf.definition_file = scope
# Set static
if fs_desc.get('type') == "static":
func_intf.static = True
else:
func_intf.static = False
# Add declarations
files = {func_intf.definition_file
} if func_intf.definition_file else set()
if fs_desc['declarations']:
files.update({
f
for f in fs_desc['declarations']
if f != 'unknown' and f in deps
})
for file in files:
if file not in cfiles and file not in func_intf.header_files:
func_intf.header_files.append(file)
for cfile in deps[file]:
self.set_source_function(func_intf, cfile)
func_intf.declaration_files.add(cfile)
def _collect_file_dependencies(self, abstract_task):
"""
Collect for each included header file or c file its "main" file to which it was included. This is required
since we cannot write aspects and instrument files which have no CC command so me build this map.
:param abstract_task: Abstract task dictionary.
:return: Collection dictionary {included file: {files that include this one}}.
"""
collection = dict()
c_files = set()
def _collect_cc_deps(cfile, deps):
# Collect for each file CC entry to which it is included
for file in deps:
if file not in collection:
collection[file] = set()
collection[file].add(cfile)
# Read each CC description and import map of files to in files
for group in abstract_task['grps']:
for desc in group['Extra CCs']:
cc_desc = self._clade.get_cmd(desc['CC'])
c_file = cc_desc['in'][0]
# Now read deps
_collect_cc_deps(c_file, self._clade.get_cmd_deps(desc['CC']))
c_files.add(c_file)
return c_files, collection
def weave(self):
self.abstract_task_desc['extra C files'] = []
clade = Clade(self.conf['build base'])
# This is required to get compiler (Aspectator) specific stdarg.h since kernel C files are compiled
# with "-nostdinc" option and system stdarg.h couldn't be used.
aspectator_search_dir = '-isystem' + core.utils.execute(
self.logger, ('aspectator', '-print-file-name=include'),
collect_all_stdout=True)[0]
env = dict(os.environ)
# Print stubs instead of inline Assembler since verifiers do not interpret it and even can fail.
env['LDV_INLINE_ASM_STUB'] = ''
for grp in self.abstract_task_desc['grps']:
self.logger.info('Weave in C files of group "{0}"'.format(
grp['id']))
for extra_cc in grp['Extra CCs']:
if 'CC' in extra_cc:
if extra_cc['CC'].isdigit():
cc = clade.get_cmd(extra_cc['CC'], with_opts=True)
else:
with open(os.path.join(
self.conf['main working directory'],
extra_cc['CC']),
encoding='utf8') as fp:
cc = json.load(fp)
# extra_cc is a cc command that is not from Clade
# Thus paths in it need to be converted to be absolute
# like in other Clade commands
if "cwd" in cc and "in" in cc:
cc["in"] = [
os.path.join(cc["cwd"], cc_in)
for cc_in in cc["in"]
]
if "cwd" in cc and "out" in cc:
cc["out"] = [
os.path.join(cc["cwd"], cc_out)
for cc_out in cc["out"]
]
self.logger.info('Weave in C file "{0}"'.format(
cc['in'][0]))
cc['out'][0] = '{0}.c'.format(
core.utils.unique_file_name(
os.path.splitext(os.path.basename(
cc['out'][0]))[0], '.abs-paths.i'))
# Produce aspect to be weaved in.
if 'plugin aspects' in extra_cc:
self.logger.info(
'Concatenate all aspects of all plugins together')
# Resulting aspect.
aspect = 'aspect'
# Get all aspects. Place RSG aspects at beginning since they can instrument entities added by
# aspects of other plugins while corresponding function declarations still need be at beginning
# of file.
aspects = []
for plugin_aspects in extra_cc['plugin aspects']:
if plugin_aspects['plugin'] == 'RSG':
aspects[0:0] = plugin_aspects['aspects']
else:
aspects.extend(plugin_aspects['aspects'])
# Concatenate aspects.
with open(
aspect, 'w', encoding='utf8'
) as fout, fileinput.input(
[
os.path.join(
self.conf['main working directory'],
aspect) for aspect in aspects
],
openhook=fileinput.hook_encoded(
'utf8')) as fin:
for line in fin:
fout.write(line)
else:
# Simulate resulting aspect.
aspect = '/dev/null'
self.logger.debug(
'Aspect to be weaved in is "{0}"'.format(aspect))
core.utils.execute(
self.logger,
tuple([
'cif',
'--in',
clade.get_storage_path(cc['in'][0]),
'--aspect',
os.path.realpath(aspect),
# Besides header files specific for requirements specifications will be searched for.
'--general-opts',
'-I' + os.path.realpath(
os.path.dirname(self.conf['requirements DB'])),
'--aspect-preprocessing-opts',
' '.join(self.conf['aspect preprocessing options']
) if 'aspect preprocessing options' in
self.conf else '',
'--out',
os.path.realpath(cc['out'][0]),
'--back-end',
'src',
'--debug',
'DEBUG'
] + (['--keep'] if self.
conf['keep intermediate files'] else []) +
['--'] + core.vtg.utils.prepare_cif_opts(
self.conf, cc['opts'], clade.storage_dir) +
[aspectator_search_dir]),
env=env,
cwd=clade.get_storage_path(cc['cwd']),
timeout=0.01,
filter_func=core.vtg.utils.CIFErrorFilter())
self.logger.debug('C file "{0}" was weaved in'.format(
cc['in'][0]))
# In addition preprocess output files since CIF outputs a bit unpreprocessed files.
preprocessed_c_file = '{}.i'.format(
os.path.splitext(cc['out'][0])[0])
core.utils.execute(
self.logger, ('aspectator', '-E', '-x', 'c',
cc['out'][0], '-o', preprocessed_c_file),
timeout=0.01)
if not self.conf['keep intermediate files']:
os.remove(cc['out'][0])
self.logger.debug(
'Preprocessed weaved C file was put to "{0}"'.format(
preprocessed_c_file))
abs_paths_c_file = '{0}.abs-paths.i'.format(
os.path.splitext(cc['out'][0])[0])
with open(preprocessed_c_file,
encoding='utf8') as fp_in, open(
abs_paths_c_file, 'w',
encoding='utf8') as fp_out:
# Print preprocessor header as is.
first_line = fp_in.readline()
fp_out.write(first_line)
for line in fp_in:
fp_out.write(line)
if line == first_line:
break
# Replace relative file paths with absolute ones for line directives in other lines.
for line in fp_in:
match = re.match(r'(# \d+ ")(.+)("\n)', line)
if match:
file = match.group(2)
if not os.path.isabs(file):
# All relative file paths are relative to CC working directory.
file = os.path.abspath(
os.path.join(
os.path.realpath(clade.storage_dir)
+ cc['cwd'], file))
fp_out.write(
match.group(1) + file + match.group(3))
else:
fp_out.write(line)
if not self.conf['keep intermediate files']:
os.remove(preprocessed_c_file)
self.logger.debug(
'Preprocessed weaved C file with absolute paths was put to "{0}"'
.format(abs_paths_c_file))
extra_c_file = {
'C file':
os.path.relpath(abs_paths_c_file,
self.conf['main working directory'])
}
else:
extra_c_file = {}
if 'requirement id' in extra_cc:
extra_c_file['requirement id'] = extra_cc['requirement id']
if 'bug kinds' in extra_cc:
extra_c_file['bug kinds'] = extra_cc['bug kinds']
self.abstract_task_desc['extra C files'].append(extra_c_file)
# These sections won't be reffered any more.
del (self.abstract_task_desc['grps'])
del (self.abstract_task_desc['deps'])
def request_arg_signs(self):
self.logger.info('Request argument signatures')
clade = Clade(work_dir=self.conf['build base'])
if not clade.work_dir_ok():
raise RuntimeError('Build base is not OK')
meta = clade.get_meta()
for request_aspect in self.conf['request aspects']:
request_aspect = klever.core.vtg.utils.find_file_or_dir(self.logger, self.conf['main working directory'],
request_aspect)
self.logger.debug('Request aspect is "{0}"'.format(request_aspect))
for grp in self.abstract_task_desc['grps']:
self.logger.info('Request argument signatures for C files of group "{0}"'.format(grp['id']))
for extra_cc in grp['Extra CCs']:
infile = extra_cc['in file']
self.logger.info('Request argument signatures for C file "{0}"'.format(infile))
cc = clade.get_cmd(*extra_cc['CC'], with_opts=True)
env = dict(os.environ)
env['LDV_ARG_SIGNS_FILE'] = os.path.realpath(
os.path.splitext(os.path.splitext(os.path.basename(request_aspect))[0])[0])
self.logger.debug('Argument signature file is "{0}"'
.format(os.path.relpath(env['LDV_ARG_SIGNS_FILE'])))
# Add plugin aspects produced thus far (by EMG) since they can include additional headers for which
# additional argument signatures should be extracted. Like in Weaver.
if 'plugin aspects' in extra_cc:
self.logger.info('Concatenate all aspects of all plugins together')
# Resulting request aspect.
aspect = '{0}.aspect'.format(
klever.core.utils.unique_file_name(os.path.splitext(os.path.basename(infile))[0],
'.aspect'))
# Get all aspects. Place original request aspect at beginning since it can instrument entities
# added by aspects of other plugins while corresponding function declarations still need be at
# beginning of file.
aspects = [os.path.relpath(request_aspect, self.conf['main working directory'])]
for plugin_aspects in extra_cc['plugin aspects']:
aspects.extend(plugin_aspects['aspects'])
# Concatenate aspects.
with open(aspect, 'w', encoding='utf-8') as fout, fileinput.input(
[os.path.join(self.conf['main working directory'], aspect) for aspect in aspects],
openhook=fileinput.hook_encoded('utf-8')) as fin:
for line in fin:
fout.write(line)
else:
aspect = request_aspect
storage_path = clade.get_storage_path(infile)
if meta['conf'].get('Compiler.preprocess_cmds', False) and \
'klever-core-work-dir' not in storage_path:
storage_path = storage_path.split('.c')[0] + '.i'
opts = cc['opts']
# Like in Weaver.
opts.append(klever.core.vtg.utils.define_arch_dependent_macro(self.conf))
klever.core.utils.execute(
self.logger,
tuple(
[
klever.core.vtg.utils.get_cif_or_aspectator_exec(self.conf, 'cif'),
'--in', storage_path,
'--aspect', os.path.realpath(aspect),
'--stage', 'instrumentation',
'--out', os.path.realpath('{0}.c'.format(klever.core.utils.unique_file_name(
os.path.splitext(os.path.basename(infile))[0], '.c.aux'))),
'--debug', 'DEBUG'
] +
(['--keep'] if self.conf['keep intermediate files'] else []) +
['--'] +
klever.core.vtg.utils.prepare_cif_opts(opts, clade) +
# Like in Weaver.
['-I' + os.path.join(os.path.dirname(self.conf['specifications base']), 'include')]
),
env,
cwd=clade.get_storage_path(cc['cwd']),
timeout=0.01,
filter_func=klever.core.vtg.utils.CIFErrorFilter())
class Diff:
def __init__(self, work_dir1, work_dir2, log_level="INFO"):
self.work_dir1 = work_dir1
self.work_dir2 = work_dir2
self.cl1 = Clade(work_dir1)
self.cl2 = Clade(work_dir2)
logger.setLevel(log_level)
def compare(self):
self.compare_extension_lists()
self.compare_pid_graphs()
self.compare_cmds()
self.compare_storages()
self.compare_cmd_graphs()
self.compare_src_graphs()
self.compare_functions()
self.compare_macros()
self.compare_callgraphs()
pass
def compare_extension_lists(self):
a = self.__get_extension_list(self.work_dir1)
b = self.__get_extension_list(self.work_dir2)
if a == b:
logger.info("Sets of extensions are the same: {!r}".format(
", ".join(a)))
else:
logger.error("Sets of extensions are different")
logger.error("First set: {!r}".format(", ".join(a)))
logger.error("Second set: {!r}".format(", ".join(b)))
def compare_pid_graphs(self):
if not self.__ext_work_dirs_exist("PidGraph"):
return
keys_are_same = True
direct_parents_are_same = True
indirect_parents_are_same = True
pid_by_id1 = self.cl1.pid_by_id
pid_by_id2 = self.cl2.pid_by_id
keys1 = pid_by_id1.keys()
keys2 = pid_by_id2.keys()
if keys1 == keys2:
logger.debug("IDs of intercepted commands are the same")
common_ids = keys1
else:
common_ids = set(keys1) & set(keys2)
keys_are_same = False
removed = [x for x in keys1 if x not in common_ids]
added = [x for x in keys2 if x not in common_ids]
if removed:
logger.error(
"{!r} ids were removed from the pid graph".format(removed))
if added:
logger.error(
"{!r} ids were added to the pid graph".format(added))
for key in common_ids:
if pid_by_id1[key] != pid_by_id2[key]:
logger.error(
"ID {!r} has different direct parents: {!r} vs {!r}".
format(key, pid_by_id1[key], pid_by_id2[key]))
direct_parents_are_same = False
if direct_parents_are_same and keys_are_same:
logger.debug("All ids have the same direct parents")
elif direct_parents_are_same and not keys_are_same:
logger.debug("All common ids have the same direct parents")
pid_graph1 = self.cl1.pid_graph
pid_graph2 = self.cl2.pid_graph
for key in common_ids:
if pid_graph1[key] != pid_graph2[key]:
logger.error(
"ID {!r} has different indirect parents: {!r} vs {!r}".
format(key, pid_graph1[key], pid_graph2[key]))
indirect_parents_are_same = False
if indirect_parents_are_same and keys_are_same:
logger.debug("All ids have the same indirect parents")
elif indirect_parents_are_same and not keys_are_same:
logger.debug("All common ids have the same indirect parents")
if (keys_are_same and direct_parents_are_same
and indirect_parents_are_same):
logger.info("Pid graphs are the same")
else:
logger.info("Pid graphs are different")
def compare_cmds(self):
if not self.__ext_work_dirs_exist("CmdGraph"):
return
keys_are_same = True
inputs_are_same = True
outputs_are_same = True
deps_are_same = True
cmds1 = self.cl1.cmds
cmds2 = self.cl2.cmds
cmd_ids1 = [x["id"] for x in cmds1]
cmd_ids2 = [x["id"] for x in cmds2]
if cmd_ids1 == cmd_ids2:
logger.debug("Sets of parsed commands are the same")
common_ids = cmd_ids1
else:
common_ids = set(cmd_ids1) & set(cmd_ids2)
keys_are_same = False
removed = [x for x in cmd_ids1 if x not in common_ids]
added = [x for x in cmd_ids2 if x not in common_ids]
for cmd_id in removed:
logger.error("{!r} command with ID={!r} was removed".format(
self.cl1.get_cmd_type(cmd_id), cmd_id))
for cmd_id in added:
logger.error("{!r} command with ID={!r} was added".format(
self.cl2.get_cmd_type(cmd_id), cmd_id))
for cmd_id in common_ids:
cmd1 = self.cl1.get_cmd(cmd_id)
cmd2 = self.cl2.get_cmd(cmd_id)
if cmd1["in"] != cmd2["in"]:
logger.error(
"{!r} command with ID={!r} changed its input files: {!r} vs {!r}"
.format(
self.cl1.get_cmd_type(cmd_id),
cmd_id,
cmd1["in"],
cmd2["in"],
))
inputs_are_same = False
if cmd1["out"] != cmd2["out"]:
logger.error(
"{!r} command with ID={!r} changed its output files: {!r} vs {!r}"
.format(
self.cl1.get_cmd_type(cmd_id),
cmd_id,
cmd1["out"],
cmd2["out"],
))
outputs_are_same = False
common_compilation_cmds_ids = [
x["id"] for x in self.cl2.compilation_cmds if x["id"] in common_ids
]
for cmd_id in common_compilation_cmds_ids:
cmd_deps1 = set(self.cl1.get_cmd(cmd_id, with_deps=True)["deps"])
cmd_deps2 = set(self.cl2.get_cmd(cmd_id, with_deps=True)["deps"])
if cmd_deps1 != cmd_deps2:
removed = cmd_deps1 - cmd_deps2
added = cmd_deps2 - cmd_deps1
for dep in removed:
logger.error(
"{!r} command with ID={!r} no longer has dependency: {!r}"
.format(self.cl1.get_cmd_type(cmd_id), cmd_id, dep))
for dep in added:
logger.error(
"{!r} command with ID={!r} has new dependency: {!r}".
format(self.cl1.get_cmd_type(cmd_id), cmd_id, dep))
deps_are_same = False
if (keys_are_same and inputs_are_same and outputs_are_same
and deps_are_same):
logger.info("Parsed commands are the same")
else:
logger.info("Parsed commands are different")
def compare_storages(self):
if not self.__ext_work_dirs_exist("Storage"):
return
storage_files1 = set()
storage_files2 = set()
for root, _, filenames in os.walk(self.cl1.storage_dir):
for filename in filenames:
storage_files1.add(
os.path.relpath(
os.path.join(root, filename),
start=self.cl1.storage_dir,
))
for root, _, filenames in os.walk(self.cl2.storage_dir):
for filename in filenames:
storage_files2.add(
os.path.relpath(
os.path.join(root, filename),
start=self.cl2.storage_dir,
))
if storage_files1 == storage_files2:
logger.info("Files in the Storage are the same")
else:
removed = storage_files1 - storage_files2
added = storage_files2 - storage_files1
for file in removed:
logger.error(
"{!r} file was removed from the Storage".format(file))
for file in added:
logger.error("{!r} file was added to the Storage".format(file))
def compare_cmd_graphs(self):
if not self.__ext_work_dirs_exist("CmdGraph"):
return
cmd_graph1 = self.cl1.cmd_graph
cmd_graph2 = self.cl2.cmd_graph
used_by_are_same = True
using_are_same = True
cmds_ids1 = set(cmd_graph1.keys())
cmds_ids2 = set(cmd_graph2.keys())
if cmds_ids1 == cmds_ids2:
common_ids = cmds_ids1
else:
common_ids = cmds_ids1 & cmds_ids2
for cmd_id in common_ids:
used_by1 = set(cmd_graph1[cmd_id]["used_by"])
used_by2 = set(cmd_graph2[cmd_id]["used_by"])
if used_by1 != used_by2:
removed = used_by1 - used_by2
added = used_by2 - used_by1
if removed:
logger.error(
"{!r} command with ID={!r} is no longer used by: {!r}".
format(
cmd_graph1[cmd_id]["type"],
cmd_id,
", ".join(removed),
))
if added:
logger.error(
"{!r} command with ID={!r} is now used by: {!r}".
format(
cmd_graph1[cmd_id]["type"],
cmd_id,
", ".join(added),
))
used_by_are_same = False
using1 = set(cmd_graph1[cmd_id]["using"])
using2 = set(cmd_graph2[cmd_id]["using"])
if using1 != using2:
removed = using1 - using2
added = using2 - using1
if removed:
logger.error(
"{!r} command with ID={!r} is no longer using: {!r}".
format(
cmd_graph1[cmd_id]["type"],
cmd_id,
", ".join(removed),
))
if added:
logger.error(
"{!r} command with ID={!r} is now using: {!r}".format(
cmd_graph1[cmd_id]["type"],
cmd_id,
", ".join(added),
))
using_are_same = False
if used_by_are_same and using_are_same:
logger.info("Cmd graphs are the same")
else:
logger.info("Cmd graphs are different")
def compare_src_graphs(self):
if not self.__ext_work_dirs_exist("SrcGraph"):
return
src_graph1 = self.cl1.src_graph
src_graph2 = self.cl2.src_graph
keys_are_same = True
used_by_are_same = True
compiled_in_are_same = True
files1 = set(src_graph1.keys())
files2 = set(src_graph2.keys())
if files1 == files2:
logger.info("Files in the source graph are the same")
common_files = files1
else:
common_files = files1 & files2
removed = files1 - files2
added = files2 - files1
for file in removed:
logger.error(
"{!r} file was removed from the source graph".format(file))
for file in added:
logger.error(
"{!r} file was added to the source graph".format(file))
keys_are_same = False
for file in common_files:
used_by1 = set(src_graph1[file]["used_by"])
used_by2 = set(src_graph2[file]["used_by"])
if used_by1 != used_by2:
removed = used_by1 - used_by2
added = used_by2 - used_by1
if removed:
logger.error(
"{!r} file is no longer used by {!r} commands".format(
file, ", ".join(removed)))
if added:
logger.error(
"{!r} file is now used by {!r} commands".format(
file, ", ".join(added)))
used_by_are_same = False
compiled_in1 = set(src_graph1[file]["compiled_in"])
compiled_in2 = set(src_graph1[file]["compiled_in"])
if compiled_in1 != compiled_in2:
removed = compiled_in1 - compiled_in2
added = compiled_in2 - compiled_in1
if removed:
logger.error(
"{!r} file is no longer compiled in {!r} commands".
format(file, ", ".join(removed)))
if added:
logger.error(
"{!r} file is now compiled in {!r} commands".format(
file, ", ".join(added)))
compiled_in_are_same = False
if keys_are_same and used_by_are_same and compiled_in_are_same:
logger.info("Source graphs are the same")
else:
logger.info("Source graphs are different")
def compare_functions(self):
if not self.__ext_work_dirs_exist("Functions"):
return
keys_are_same = True
types_are_same = True
lines_are_same = True
signatures_are_same = True
decls_are_same = True
f1 = self.cl1.functions_by_file
f2 = self.cl2.functions_by_file
keys1 = set()
keys2 = set()
for file in f1:
for func in f1[file]:
keys1.add((file, func))
for file in f2:
for func in f2[file]:
keys2.add((file, func))
if keys1 == keys2:
common_keys = keys2
else:
common_keys = keys1 & keys2
removed = keys1 - keys2
added = keys2 - keys1
for file, func in removed:
logger.error("{!r} function from {!r} file was removed".format(
func, file))
for file, func in added:
logger.error("{!r} function from {!r} file was added".format(
func, file))
keys_are_same = False
for file, func in common_keys:
type1 = f1[file][func]["type"]
type2 = f2[file][func]["type"]
if type1 != type2:
logger.error(
"{!r} function from {!r} file changed its type from {!r} to {!r}"
.format(func, file, type1, type2))
types_are_same = False
line1 = f1[file][func]["line"]
line2 = f2[file][func]["line"]
if line1 != line2:
logger.error(
"{!r} function from {!r} file changed its definition line from {!r} to {!r}"
.format(func, file, line1, line2))
lines_are_same = False
signature1 = f1[file][func]["signature"]
signature2 = f2[file][func]["signature"]
if signature1 != signature2:
logger.error(
"{!r} function from {!r} file changed its signature from {!r} to {!r}"
.format(func, file, signature1, signature2))
signatures_are_same = False
if f1[file][func]["declarations"]:
decl_files1 = set(f1[file][func]["declarations"].keys())
else:
decl_files1 = set()
if f2[file][func]["declarations"]:
decl_files2 = set(f2[file][func]["declarations"].keys())
else:
decl_files2 = set()
if decl_files1 == decl_files2:
common_decl_files = decl_files2
else:
common_decl_files = decl_files1 & decl_files2
removed = decl_files1 - decl_files2
added = decl_files2 - decl_files1
for decl_file in removed:
logger.error(
"{!r} function from {!r} file no longer has declaration in {!r}"
.format(func, file, decl_file))
for decl_file in added:
logger.error(
"{!r} function from {!r} file has new declaration in {!r}"
.format(func, file, decl_file))
decls_are_same = False
for decl_file in common_decl_files:
decl1 = f1[file][func]["declarations"][decl_file]
decl2 = f2[file][func]["declarations"][decl_file]
line1 = decl1["line"]
line2 = decl2["line"]
if line1 != line2:
logger.error(
"{!r} function from {!r} file changed its declaration line from {!r} to {!r}"
.format(func, file, line1, line2))
decls_are_same = False
signature1 = decl1["signature"]
signature2 = decl2["signature"]
if signature1 != signature2:
logger.error(
"{!r} function from {!r} file changed its declaration signature from {!r} to {!r}"
.format(func, file, signature1, signature2))
decls_are_same = False
type1 = decl1["type"]
type2 = decl2["type"]
if type1 != type2:
logger.error(
"{!r} function from {!r} file changed its declaration type from {!r} to {!r}"
.format(func, file, type1, type2))
decls_are_same = False
if (keys_are_same and types_are_same and lines_are_same
and signatures_are_same and decls_are_same):
logger.info("Functions are the same")
else:
logger.info("Functions are different")
def compare_macros(self):
if not self.__ext_work_dirs_exist("Macros"):
return
exp_files_are_same = True
exp_names_are_same = True
exp_args_are_same = True
exp1 = self.cl1.get_macros_expansions()
exp2 = self.cl2.get_macros_expansions()
exp_files1 = set(exp1)
exp_files2 = set(exp2)
if exp_files1 == exp_files2:
common_exp_files = exp_files2
else:
common_exp_files = exp_files1 & exp_files2
removed = exp_files1 - exp_files2
added = exp_files2 - exp_files1
for exp_file in removed:
logger.error(
"{!r} file was removed from macros expansions".format(
exp_file))
for exp_file in added:
logger.error("{!r} file was added to macros expansions".format(
exp_file))
exp_files_are_same = False
for exp_file in common_exp_files:
exp_names1 = set(exp1[exp_file])
exp_names2 = set(exp2[exp_file])
if exp_names1 == exp_names2:
common_exp_names = exp_names2
else:
common_exp_names = exp_names1 & exp_names2
removed = exp_names1 - exp_names2
added = exp_names2 - exp_names1
for exp_name in removed:
logger.error(
"Expansion of macro {!r} from {!r} file was removed".
format(exp_name, exp_file))
for exp_name in added:
logger.error(
"Expansion of macro {!r} from {!r} file was added".
format(exp_name, exp_file))
exp_names_are_same = False
for exp_name in common_exp_names:
args1 = set([
x for sublist in exp1[exp_file][exp_name]["args"]
for x in sublist
])
args2 = set([
x for sublist in exp2[exp_file][exp_name]["args"]
for x in sublist
])
if args1 != args2:
removed = args1 - args2
added = args2 - args1
for args in removed:
logger.error(
"Macro {!r} from {!r} file no longer has these expansion args: {!r}"
.format(exp_name, exp_file, args))
for args in added:
logger.error(
"Macro {!r} from {!r} file now has new expansion args: {!r}"
.format(exp_name, exp_file, args))
exp_args_are_same = False
if exp_files_are_same and exp_names_are_same and exp_args_are_same:
logger.info("Macros expansions are the same")
else:
logger.info("Macros expansions are different")
def1 = self.cl1.get_macros_definitions()
def2 = self.cl2.get_macros_definitions()
def_files_are_same = True
def_names_are_same = True
def_lines_are_same = True
def_files1 = set(def1)
def_files2 = set(def2)
if def_files1 == def_files2:
common_def_files = def_files2
else:
common_def_files = def_files1 & def_files2
removed = def_files1 - def_files2
added = def_files2 - def_files1
for def_file in removed:
logger.error(
"{!r} file was removed from macros definitions".format(
def_file))
for def_file in added:
logger.error(
"{!r} file was added to macros definitions".format(
def_file))
def_files_are_same = False
for def_file in common_def_files:
def_names1 = set(def1[def_file])
def_names2 = set(def1[def_file])
if def_names1 == def_names2:
common_def_names = def_names2
else:
common_def_names = def_names1 & def_names2
removed = def_names1 - def_names2
added = def_names2 - def_names1
for def_name in removed:
logger.error(
"Definition of macro {!r} from {!r} file was removed".
format(def_name, def_file))
for def_name in added:
logger.error(
"Definition of macro {!r} from {!r} file was added".
format(def_name, def_file))
def_names_are_same = False
for def_name in common_def_names:
lines1 = def1[def_file][def_name]
lines2 = def1[def_file][def_name]
if lines1 != lines2:
removed = lines1 - lines2
added = lines2 - lines1
for line in removed:
logger.error(
"Macro {!r} from {!r} file no longer has definition on line {!r}"
.format(def_name, def_file, line))
for line in added:
logger.error(
"Macro {!r} from {!r} file now has definition on line {!r}"
.format(def_name, def_file, line))
def_lines_are_same = False
if def_files_are_same and def_names_are_same and def_lines_are_same:
logger.info("Macros definitions are the same")
else:
logger.info("Macros definitions are different")
def compare_callgraphs(self):
if not self.__ext_work_dirs_exist("Callgraph"):
return
files_are_same = True
funcs_are_same = True
called_in_files_are_same = True
called_in_funcs_are_same = True
call_lines_are_same = True
match_types_are_same = True
c1 = self.cl1.callgraph
c2 = self.cl2.callgraph
files1 = set(c1)
files2 = set(c2)
if files1 == files2:
common_files = files2
else:
common_files = files1 & files2
removed = files1 - files2
added = files2 - files1
for file in removed:
logger.error(
"{!r} file was removed from the callgraph".format(file))
for file in added:
logger.error(
"{!r} file was added to the callgraph".format(file))
files_are_same = False
for file in common_files:
funcs1 = set(c1[file])
funcs2 = set(c2[file])
if funcs1 == funcs2:
common_funcs = funcs2
else:
common_funcs = funcs1 & funcs2
removed = funcs1 - funcs2
added = funcs2 - funcs1
for func in removed:
logger.error(
"{!r} function from {!r} file was removed from the callgraph"
.format(func, file))
for func in added:
logger.error(
"{!r} function from {!r} file was added to the callgraph"
.format(func, file))
for func in common_funcs:
called_in_files1 = set(c1[file][func].get("called_in", []))
called_in_files2 = set(c2[file][func].get("called_in", []))
if called_in_files1 == called_in_files2:
common_called_in_files = called_in_files2
else:
common_called_in_files = (called_in_files1
& called_in_files2)
removed = called_in_files1 - called_in_files2
added = called_in_files2 - called_in_files1
for called_in_file in removed:
logger.error(
"{!r} function from {!r} file is no longer called in {!r} file"
.format(func, file, called_in_file))
for called_in_file in added:
logger.error(
"{!r} function from {!r} file is now called in {!r} file"
.format(func, file, called_in_file))
called_in_files_are_same = False
for called_in_file in common_called_in_files:
called_in_funcs1 = set(
c1[file][func]["called_in"][called_in_file])
called_in_funcs2 = set(
c2[file][func]["called_in"][called_in_file])
if called_in_funcs1 == called_in_funcs2:
common_called_in_funcs = called_in_funcs2
else:
common_called_in_funcs = (called_in_funcs1
& called_in_funcs2)
removed = called_in_funcs1 - called_in_funcs2
added = called_in_funcs2 - called_in_funcs1
for called_in_func in removed:
logger.error(
"{!r} function from {!r} file is no longer called in {!r} func from {!r} file"
.format(func, file, called_in_func,
called_in_file))
for called_in_func in added:
logger.error(
"{!r} function from {!r} file is now called in {!r} func from {!r} file"
.format(func, file, called_in_func,
called_in_file))
called_in_funcs_are_same = False
for called_in_func in common_called_in_funcs:
call_lines1 = set(c1[file][func]["called_in"]
[called_in_file][called_in_func])
call_lines2 = set(c2[file][func]["called_in"]
[called_in_file][called_in_func])
if call_lines1 == call_lines2:
common_call_lines = call_lines2
else:
common_call_lines = call_lines1 & call_lines2
removed = call_lines1 - call_lines2
added = call_lines2 - call_lines1
for call_line in removed:
logger.error(
"{!r} function from {!r} file is no longer called in {!r} func from {!r} file on line {!r}"
.format(
func,
file,
called_in_func,
called_in_file,
call_line,
))
for call_line in added:
logger.error(
"{!r} function from {!r} file is now called in {!r} func from {!r} file on line {!r}"
.format(
func,
file,
called_in_func,
called_in_file,
call_line,
))
call_lines_are_same = False
if common_call_lines:
call_line = list(common_call_lines)[0]
match_type1 = c1[file][func]["called_in"][
called_in_file][called_in_func][call_line][
"match_type"]
match_type2 = c2[file][func]["called_in"][
called_in_file][called_in_func][call_line][
"match_type"]
if match_type1 != match_type2:
logger.error(
"Match type of {!r} ({!r}) call in {!r} ({!r}) was changed from {!r} to {!r}"
.format(
func,
file,
called_in_func,
called_in_file,
match_type1,
match_type2,
))
match_types_are_same = False
if (files_are_same and funcs_are_same and called_in_files_are_same
and called_in_funcs_are_same and call_lines_are_same
and match_types_are_same):
logger.info("Callgraphs are the same")
else:
logger.info("Callgraphs are different")
@staticmethod
def __get_extension_list(work_dir):
return [
f for f in os.listdir(work_dir)
if os.path.isdir(os.path.join(work_dir, f))
]
def __ext_work_dirs_exist(self, ext_name):
a = os.path.join(self.work_dir1, ext_name)
b = os.path.join(self.work_dir2, ext_name)
if os.path.exists(a) and os.path.exists(b):
return True
else:
return False