def test_files_to_add(tmpdir, cmds_file):
c = Clade(tmpdir, cmds_file, conf={"Storage.files_to_add": [__file__]})
c.parse("Storage")
storage_path = c.get_storage_path(__file__)
assert storage_path
assert os.path.exists(storage_path)
def test_storage(tmpdir):
c = Clade(tmpdir)
c.add_file_to_storage(__file__)
c.add_file_to_storage("do_not_exist.c")
assert os.path.exists(os.path.join(c.storage_dir, __file__))
assert c.get_storage_path(__file__)
# Test possible race condition
with unittest.mock.patch("shutil.copyfile") as copyfile_mock:
copyfile_mock.side_effect = shutil.SameFileError
c.add_file_to_storage(test_file)
def test_storage(tmpdir):
c = Clade(tmpdir)
returned_storage_path = c.add_file_to_storage(__file__)
c.add_file_to_storage("do_not_exist.c")
storage_path = c.get_storage_path(__file__)
assert storage_path
assert os.path.exists(storage_path)
assert storage_path.startswith(c.storage_dir)
assert returned_storage_path == storage_path
# Test possible race condition
with unittest.mock.patch("shutil.copyfile") as copyfile_mock:
copyfile_mock.side_effect = shutil.SameFileError
c.add_file_to_storage(test_file)
def weave(self):
self.abstract_task_desc.setdefault('extra C files', dict())
clade = Clade(self.conf['build base'])
if not clade.work_dir_ok():
raise RuntimeError('Build base is not OK')
meta = clade.get_meta()
# This is required to get compiler (Aspectator) specific stdarg.h since kernel C files are compiled
# with "-nostdinc" option and system stdarg.h couldn't be used.
aspectator_search_dir = '-isystem' + klever.core.utils.execute(
self.logger, (klever.core.vtg.utils.get_cif_or_aspectator_exec(
self.conf, 'aspectator'), '-print-file-name=include'),
collect_all_stdout=True)[0]
env = dict(os.environ)
# Print stubs instead of inline Assembler since verifiers do not interpret it and even can fail.
env['LDV_INLINE_ASM_STUB'] = ''
for grp in self.abstract_task_desc['grps']:
self.logger.info('Weave in C files of group "{0}"'.format(
grp['id']))
for extra_cc in grp['Extra CCs']:
# Each CC is either pair (compiler command identifier, compiler command type) or JSON file name
# with compiler command description.
if isinstance(extra_cc['CC'], list):
cc = clade.get_cmd(*extra_cc['CC'], with_opts=True)
if "in file" in extra_cc:
# This is for CC commands with several input files
infile = extra_cc["in file"]
else:
infile = cc["in"][0]
infile = clade.get_storage_path(infile)
if meta['conf'].get('Compiler.preprocess_cmds', False):
infile = infile('.c')[0] + '.i'
else:
with open(os.path.join(self.conf['main working directory'],
extra_cc['CC']),
encoding='utf8') as fp:
cc = json.load(fp)
infile = cc["in"][0]
# Distinguish source files having the same names.
outfile_unique = '{0}.c'.format(
klever.core.utils.unique_file_name(
os.path.splitext(os.path.basename(infile))[0], '.c'))
# This is used for storing/getting to/from cache where uniqueness is guaranteed by other means.
outfile = '{0}.c'.format(
os.path.splitext(os.path.basename(infile))[0])
self.logger.info('Weave in C file "{0}"'.format(infile))
# Produce aspect to be weaved in.
if 'plugin aspects' in extra_cc:
self.logger.info(
'Concatenate all aspects of all plugins together')
# Resulting aspect.
aspect = 'aspect'
# Get all aspects. Place RSG aspects at beginning since they can instrument entities added by
# aspects of other plugins while corresponding function declarations still need be at beginning
# of file.
aspects = []
for plugin_aspects in extra_cc['plugin aspects']:
if plugin_aspects['plugin'] == 'RSG':
aspects[0:0] = plugin_aspects['aspects']
else:
aspects.extend(plugin_aspects['aspects'])
# Concatenate aspects.
with open(aspect, 'w',
encoding='utf8') as fout, fileinput.input(
[
os.path.join(
self.conf['main working directory'],
aspect) for aspect in aspects
],
openhook=fileinput.hook_encoded(
'utf8')) as fin:
for line in fin:
fout.write(line)
else:
# Instrumentation is not required when there is no aspects. But we will still pass source files
# through C-backend to make resulting code to look similarly and thus to avoid different issues
# at merging source files and models together.
aspect = None
if aspect:
self.logger.info(
'Aspect to be weaved in is "{0}"'.format(aspect))
else:
self.logger.info(
'C file will be passed through C Back-end only')
cwd = clade.get_storage_path(cc['cwd'])
is_model = (grp['id'] == 'models')
# Original sources should be woven in and we do not need to get cross references for them since this
# was already done before.
if not is_model:
self.__weave(infile, cc['opts'], aspect, outfile_unique,
clade, env, cwd, aspectator_search_dir,
is_model)
# For generated models we need to weave them in (actually, just pass through C Back-end) and to get
# cross references always since most likely they all are different.
elif 'generated' in extra_cc:
self.__weave(infile, cc['opts'], aspect, outfile_unique,
clade, env, cwd, aspectator_search_dir,
is_model)
if self.conf[
'code coverage details'] != 'Original C source files':
self.__get_cross_refs(infile, cc['opts'],
outfile_unique, clade, cwd,
aspectator_search_dir)
# For non-generated models use results cache in addition.
else:
cache_dir = os.path.join(
self.conf['cache directory'],
klever.core.utils.get_file_name_checksum(infile))
with klever.core.utils.LockedOpen(cache_dir + '.tmp', 'w'):
if os.path.exists(cache_dir):
self.logger.info('Get woven in C file from cache')
self.abstract_task_desc['extra C files'].append({
'C file':
os.path.relpath(
os.path.join(cache_dir,
os.path.basename(outfile)),
self.conf['main working directory'])
})
if self.conf[
'code coverage details'] != 'Original C source files':
self.logger.info(
'Get cross references from cache')
self.__merge_additional_srcs(
os.path.join(cache_dir,
'additional sources'))
else:
os.makedirs(cache_dir)
self.__weave(infile, cc['opts'], aspect,
outfile_unique, clade, env, cwd,
aspectator_search_dir, is_model)
self.logger.info('Store woven in C file to cache')
shutil.copy(outfile_unique,
os.path.join(cache_dir, outfile))
if self.conf[
'code coverage details'] != 'Original C source files':
self.__get_cross_refs(infile, cc['opts'],
outfile_unique, clade,
cwd,
aspectator_search_dir)
self.logger.info(
'Store cross references to cache')
shutil.copytree(
outfile_unique + ' additional sources',
os.path.join(cache_dir,
'additional sources'))
# For auxiliary files there is no cross references since it is rather hard to get them from Aspectator. But
# there still highlighting.
if self.conf['code coverage details'] == 'All source files':
for aux_file in glob.glob('*.aux'):
new_file = os.path.join(
'additional sources', 'generated models',
os.path.relpath(aux_file,
self.conf['main working directory']))
os.makedirs(os.path.dirname(new_file), exist_ok=True)
shutil.copy(aux_file, new_file)
cross_refs = CrossRefs(self.conf, self.logger, clade, aux_file,
new_file, self.search_dirs)
cross_refs.get_cross_refs()
self.abstract_task_desc['additional sources'] = os.path.relpath('additional sources',
self.conf['main working directory']) \
if os.path.isdir('additional sources') else None
# Copy additional sources for total code coverage.
if self.conf['code coverage details'] != 'Original C source files':
with klever.core.utils.Cd('additional sources'):
for root, dirs, files in os.walk(os.path.curdir):
for file in files:
# These files are handled below in addition to corresponding source files.
if file.endswith('.json'):
continue
if self.conf['code coverage details'] == 'C source files including models' \
and not file.endswith('.c'):
continue
file = os.path.join(root, file)
new_file = os.path.join(
self.conf['additional sources directory'], file)
os.makedirs(os.path.dirname(new_file), exist_ok=True)
with klever.core.utils.LockedOpen(
new_file + '.tmp', 'w'):
if os.path.isfile(new_file):
os.remove(new_file + '.tmp')
continue
shutil.copy(file, new_file)
shutil.copy(file + '.idx.json',
new_file + '.idx.json')
os.remove(new_file + '.tmp')
# These sections won't be refereed any more.
del (self.abstract_task_desc['grps'])
del (self.abstract_task_desc['deps'])
def request_arg_signs(self):
self.logger.info('Request argument signatures')
clade = Clade(work_dir=self.conf['build base'])
for request_aspect in self.conf['request aspects']:
request_aspect = core.vtg.utils.find_file_or_dir(
self.logger, self.conf['main working directory'],
request_aspect)
self.logger.debug('Request aspect is "{0}"'.format(request_aspect))
# This is required to get compiler (Aspectator) specific stdarg.h since kernel C files are compiled with
# "-nostdinc" option and system stdarg.h couldn't be used.
aspectator_search_dir = '-isystem' + core.utils.execute(
self.logger, ('aspectator', '-print-file-name=include'),
collect_all_stdout=True)[0]
for grp in self.abstract_task_desc['grps']:
self.logger.info(
'Request argument signatures for C files of group "{0}"'.
format(grp['id']))
for extra_cc in grp['Extra CCs']:
self.logger.info(
'Request argument signatures for C file "{0}"'.format(
extra_cc['in file']))
cc = clade.get_cmd(extra_cc['CC'], with_opts=True)
env = dict(os.environ)
env['LDV_ARG_SIGNS_FILE'] = os.path.realpath(
os.path.splitext(
os.path.splitext(
os.path.basename(request_aspect))[0])[0])
# Add plugin aspects produced thus far (by EMG) since they can include additional headers for which
# additional argument signatures should be extracted. Like in Weaver.
if 'plugin aspects' in extra_cc:
self.logger.info(
'Concatenate all aspects of all plugins together')
# Resulting request aspect.
aspect = '{0}.aspect'.format(
core.utils.unique_file_name(
os.path.splitext(os.path.basename(
cc['out'][0]))[0], '.aspect'))
# Get all aspects. Place original request aspect at beginning since it can instrument entities
# added by aspects of other plugins while corresponding function declarations still need be at
# beginning of file.
aspects = [
os.path.relpath(
request_aspect,
self.conf['main working directory'])
]
for plugin_aspects in extra_cc['plugin aspects']:
aspects.extend(plugin_aspects['aspects'])
# Concatenate aspects.
with open(
aspect, 'w', encoding='utf8'
) as fout, fileinput.input(
[
os.path.join(
self.conf['main working directory'],
aspect) for aspect in aspects
],
openhook=fileinput.hook_encoded(
'utf8')) as fin:
for line in fin:
fout.write(line)
else:
aspect = request_aspect
core.utils.execute(
self.logger,
tuple([
'cif', '--in',
clade.get_storage_path(cc['in'][0]), '--aspect',
os.path.realpath(aspect), '--stage',
'instrumentation', '--out',
os.path.realpath('{0}.c'.format(
core.utils.unique_file_name(
os.path.splitext(
os.path.basename(cc['out'][0]))[0],
'.c.aux'))), '--debug', 'DEBUG'
] + (
['--keep'] if self.
conf['keep intermediate files'] else []
) + ['--'] + core.vtg.utils.prepare_cif_opts(
self.conf, cc['opts'], clade.storage_dir
) + [
# Besides header files specific for requirements will be
# searched for.
'-I' + os.path.realpath(
os.path.dirname(self.conf['requirements DB'])),
aspectator_search_dir
]),
env,
cwd=clade.get_storage_path(cc['cwd']),
timeout=0.01,
filter_func=core.vtg.utils.CIFErrorFilter())
def add_models(self, generated_models):
self.logger.info(
'Add models to abstract verification task description')
models = {}
if 'environment model' in self.abstract_task_desc:
rel_path = os.path.relpath(
os.path.join(self.conf['main working directory'],
self.abstract_task_desc['environment model']),
os.path.curdir)
models[rel_path] = {}
# Get common and requirement specific models.
if 'common models' in self.conf and 'models' in self.conf:
for common_model_c_file in self.conf['common models']:
if common_model_c_file in self.conf['models']:
raise KeyError(
'C file "{0}" is specified in both common and requirement specific models'
)
if 'models' in self.conf:
for model_c_file in self.conf['models']:
# Specify additional settings for generated models that have not any settings.
if model_c_file.startswith('$'):
is_generated_model_c_file_found = False
for generated_model_c_file in generated_models:
if generated_model_c_file.endswith(model_c_file[1:]):
models[generated_model_c_file] = self.conf[
'models'][model_c_file]
is_generated_model_c_file_found = True
if not is_generated_model_c_file_found:
raise KeyError(
'Model C file "{0}" was not generated'.format(
model_c_file[1:]))
# Like common models processed below.
for model_c_file in self.conf['models']:
if not model_c_file.startswith('$'):
model_c_file_realpath = core.vtg.utils.find_file_or_dir(
self.logger, self.conf['main working directory'],
model_c_file)
self.logger.debug('Get model with C file "{0}"'.format(
model_c_file_realpath))
models[model_c_file_realpath] = self.conf['models'][
model_c_file]
if 'common models' in self.conf:
for common_model_c_file in self.conf['common models']:
common_model_c_file_realpath = core.vtg.utils.find_file_or_dir(
self.logger, self.conf['main working directory'],
common_model_c_file)
self.logger.debug('Get common model with C file "{0}"'.format(
common_model_c_file_realpath))
models[common_model_c_file_realpath] = self.conf[
'common models'][common_model_c_file]
self.logger.debug('Resulting models are: {0}'.format(models))
if not models:
self.logger.warning('No models are specified')
return
# CC extra full description files will be put to this directory as well as corresponding intermediate and final
# output files.
os.makedirs('models'.encode('utf8'))
self.logger.info(
'Add aspects to abstract verification task description')
aspects = []
for model_c_file in models:
aspect = '{}.aspect'.format(os.path.splitext(model_c_file)[0])
if not os.path.isfile(aspect):
continue
self.logger.debug('Get aspect "{0}"'.format(aspect))
aspects.append(aspect)
# Sort aspects to apply them in the deterministic order.
aspects.sort()
for grp in self.abstract_task_desc['grps']:
self.logger.info('Add aspects to C files of group "{0}"'.format(
grp['id']))
for extra_cc in grp['Extra CCs']:
if 'plugin aspects' not in extra_cc:
extra_cc['plugin aspects'] = []
extra_cc['plugin aspects'].append({
'plugin':
self.name,
'aspects': [
os.path.relpath(aspect,
self.conf['main working directory'])
for aspect in aspects
]
})
for model_c_file in models:
model = models[model_c_file]
if 'bug kinds' in model:
self.logger.info(
'Preprocess bug kinds for model with C file "{0}"'.format(
model_c_file))
# Collect all bug kinds specified in model to check that valid bug kinds are specified in requirement
# model description.
bug_kinds = set()
lines = []
with open(model_c_file, encoding='utf8') as fp:
for line in fp:
# Bug kinds are specified in form of strings like in requirements DB as first actual
# parameters of ldv_assert().
match = re.search(r'ldv_assert\("([^"]+)"', line)
if match:
bug_kind, = match.groups()
bug_kinds.add(bug_kind)
# Include bug kinds in names of ldv_assert().
lines.append(
re.sub(
r'ldv_assert\("([^"]+)", ?',
r'ldv_assert_{0}('.format(
re.sub(r'\W', '_', bug_kind)), line))
else:
lines.append(line)
for bug_kind in model['bug kinds']:
if bug_kind not in bug_kinds:
raise KeyError(
'Invalid bug kind "{0}" is specified in requirement model description'
.format(bug_kind))
preprocessed_model_c_file = '{0}.bk.c'.format(
core.utils.unique_file_name(
os.path.join(
'models',
os.path.splitext(
os.path.basename(model_c_file))[0]), '.bk.c'))
with open(preprocessed_model_c_file, 'w',
encoding='utf8') as fp:
# Create ldv_assert*() function declarations to avoid compilation warnings. These functions will
# be defined later somehow by VTG.
for bug_kind in sorted(bug_kinds):
fp.write('extern void ldv_assert_{0}(int);\n'.format(
re.sub(r'\W', '_', bug_kind)))
# Specify original location to avoid references to *.bk.c files in error traces.
fp.write('# 1 "{0}"\n'.format(
os.path.abspath(model_c_file)))
for line in lines:
fp.write(line)
model[
'bug kinds preprocessed C file'] = preprocessed_model_c_file
self.logger.debug(
'Preprocessed bug kinds for model with C file "{0}" was placed to "{1}"'
.format(model_c_file, preprocessed_model_c_file))
else:
model['bug kinds preprocessed C file'] = model_c_file
# Generate CC full description file per each model and add it to abstract task description.
# First of all obtain CC options to be used to compile models.
clade = Clade(self.conf['build base'])
# Relative path to source file which CC options to be used is specified in configuration. Clade needs absolute
# path. The former is relative to one of source paths.
for path in self.conf['source paths']:
opts_file = os.path.join(path, self.conf['opts file'])
try:
empty_cc = list(clade.get_compilation_cmds_by_file(opts_file))
except KeyError:
pass
if not empty_cc:
raise RuntimeError("There is not of cc commands for {!r}".format(
self.conf['project']['opts file']))
elif len(empty_cc) > 1:
self.logger.warning(
"There are more than one cc command for {!r}".format(
self.conf['project']['opts file']))
empty_cc = empty_cc.pop()
empty_cc['opts'] = clade.get_cmd_opts(empty_cc['id'])
model_grp = {'id': 'models', 'Extra CCs': []}
for model_c_file in sorted(models):
model = models[model_c_file]
extra_cc = {}
if 'bug kinds preprocessed C file' in model:
file, ext = os.path.splitext(
os.path.join(
'models',
os.path.basename(
model['bug kinds preprocessed C file'])))
base_name = core.utils.unique_file_name(
file, '{0}.json'.format(ext))
full_desc_file = '{0}{1}.json'.format(base_name, ext)
out_file = '{0}.c'.format(base_name)
self.logger.debug(
'Dump CC full description to file "{0}"'.format(
full_desc_file))
with open(full_desc_file, 'w', encoding='utf8') as fp:
json.dump(
{
'cwd':
empty_cc['cwd'],
'in': [
os.path.relpath(
model['bug kinds preprocessed C file'],
os.path.realpath(
clade.get_storage_path(
empty_cc['cwd'])))
],
'out': [os.path.realpath(out_file)],
'opts':
empty_cc['opts'] + [
'-DLDV_SETS_MODEL_' +
(model['sets model'] if 'sets model' in model
else self.conf['common sets model']).upper()
]
},
fp,
ensure_ascii=False,
sort_keys=True,
indent=4)
extra_cc['CC'] = os.path.relpath(
full_desc_file, self.conf['main working directory'])
if 'bug kinds' in model:
extra_cc['bug kinds'] = model['bug kinds']
if extra_cc:
model_grp['Extra CCs'].append(extra_cc)
self.abstract_task_desc['grps'].append(model_grp)
for dep in self.abstract_task_desc['deps'].values():
dep.append(model_grp['id'])
def test_storage(tmpdir, cmds_file):
c = Clade(tmpdir, cmds_file)
c.add_file_to_storage(__file__)
assert os.path.exists(os.path.join(c.storage_dir, __file__))
assert c.get_storage_path(__file__)
class Job(klever.core.components.Component):
CORE_COMPONENTS = ['PFG', 'VTG', 'VRP']
def __init__(self,
conf,
logger,
parent_id,
callbacks,
mqs,
vals,
id=None,
work_dir=None,
attrs=None,
separate_from_parent=True,
include_child_resources=False,
components_common_conf=None):
super(Job,
self).__init__(conf, logger, parent_id, callbacks, mqs, vals, id,
work_dir, attrs, separate_from_parent,
include_child_resources)
self.common_components_conf = components_common_conf
if work_dir:
self.common_components_conf[
'additional sources directory'] = os.path.join(
os.path.realpath(work_dir), 'additional sources')
self.clade = None
self.components = []
self.component_processes = []
def decide_job_or_sub_job(self):
self.logger.info('Decide job/sub-job "{0}"'.format(self.id))
# This is required to associate verification results with particular sub-jobs.
# Skip leading "/" since this identifier is used in os.path.join() that returns absolute path otherwise.
self.common_components_conf['sub-job identifier'] = self.id[1:]
# Check and set build base here since many Core components need it.
self.__set_build_base()
self.clade = Clade(self.common_components_conf['build base'])
if not self.clade.work_dir_ok():
raise RuntimeError('Build base is not OK')
self.__retrieve_working_src_trees()
self.__get_original_sources_basic_info()
self.__upload_original_sources()
# Create directory where files will be cached and remember absolute path to it for components.
os.mkdir('cache')
self.common_components_conf['cache directory'] = os.path.realpath(
'cache')
if self.common_components_conf['keep intermediate files']:
self.logger.debug(
'Create components configuration file "conf.json"')
with open('conf.json', 'w', encoding='utf8') as fp:
json.dump(self.common_components_conf,
fp,
ensure_ascii=False,
sort_keys=True,
indent=4)
self.__get_job_or_sub_job_components()
self.callbacks = klever.core.components.get_component_callbacks(
self.logger, [type(self)] + self.components)
self.launch_sub_job_components()
self.clean_dir = True
self.logger.info("All components finished")
if self.conf.get('collect total code coverage', None):
self.logger.debug('Waiting for a collecting coverage')
while not self.vals['coverage_finished'].get(
self.common_components_conf['sub-job identifier'], True):
time.sleep(1)
self.logger.debug("Coverage collected")
main = decide_job_or_sub_job
def __set_build_base(self):
if 'build base' not in self.common_components_conf:
raise KeyError(
"Provide 'build base' configuration option to start verification"
)
common_advice = 'please, fix "job.json" (attribute "build base")'
common_advice += ' or/and deployment configuration file (attribute "Klever Build Bases")'
# Try to find specified build base either in normal way or additionally in directory "build bases" that is
# convenient to use when working with many build bases.
try:
build_base = klever.core.utils.find_file_or_dir(
self.logger, os.path.curdir,
self.common_components_conf['build base'])
except FileNotFoundError:
try:
build_base = klever.core.utils.find_file_or_dir(
self.logger, os.path.curdir,
os.path.join('build bases',
self.common_components_conf['build base']))
except FileNotFoundError:
raise FileNotFoundError(
'Specified build base "{0}" does not exist, {1}'.format(
self.common_components_conf['build base'],
common_advice)) from None
# Extract build base from archive. There should not be any intermediate directories in archives.
if os.path.isfile(build_base) and (tarfile.is_tarfile(build_base)
or zipfile.is_zipfile(build_base)):
if tarfile.is_tarfile(build_base):
self.logger.debug(
'Build base "{0}" is provided in form of TAR archive'.
format(build_base))
with tarfile.open(build_base) as TarFile:
TarFile.extractall('build base')
else:
self.logger.debug(
'Build base "{0}" is provided in form of ZIP archive'.
format(build_base))
with zipfile.ZipFile(build_base) as zfp:
zfp.extractall('build base')
# Directory contains extracted build base.
extracted_from = ' extracted from "{0}"'.format(
os.path.realpath(build_base))
build_base = 'build base'
else:
extracted_from = ''
# We need to specify absolute path to build base since it will be used in different Klever components. Besides,
# this simplifies troubleshooting.
build_base = os.path.realpath(build_base)
# TODO: fix after https://github.com/17451k/clade/issues/108.
if not os.path.isdir(build_base):
raise FileExistsError(
'Build base "{0}" is not a directory, {1}'.format(
build_base, extracted_from, common_advice))
if not os.path.isfile(os.path.join(build_base, 'meta.json')):
raise FileExistsError(
'Directory "{0}"{1} is not a build base since it does not contain file "meta.json", {2}'
.format(build_base, extracted_from, common_advice))
self.common_components_conf['build base'] = build_base
self.logger.debug('Klever components will use build base "{0}"'.format(
self.common_components_conf['build base']))
# Klever will try to cut off either working source trees (if specified) or at least build directory (otherwise)
# from referred file names. Sometimes this is rather optional like for source files referred by error traces, but,
# say, for program fragment identifiers this is strictly necessary, e.g. because of otherwise expert assessment will
# not work as expected.
def __retrieve_working_src_trees(self):
clade_meta = self.clade.get_meta()
self.common_components_conf['working source trees'] = clade_meta['working source trees'] \
if 'working source trees' in clade_meta else [clade_meta['build_dir']]
def __refer_original_sources(self, src_id):
klever.core.utils.report(self.logger, 'patch', {
'identifier': self.id,
'original_sources': src_id
}, self.mqs['report files'], self.vals['report id'],
self.conf['main working directory'])
def __process_source_files(self):
for file_name in self.clade.src_info:
self.mqs['file names'].put(file_name)
for i in range(self.workers_num):
self.mqs['file names'].put(None)
def __process_source_file(self):
while True:
file_name = self.mqs['file names'].get()
if not file_name:
return
src_file_name = klever.core.utils.make_relative_path(
self.common_components_conf['working source trees'], file_name)
if src_file_name != file_name:
src_file_name = os.path.join('source files', src_file_name)
new_file_name = os.path.join('original sources',
src_file_name.lstrip(os.path.sep))
os.makedirs(os.path.dirname(new_file_name), exist_ok=True)
shutil.copy(self.clade.get_storage_path(file_name), new_file_name)
cross_refs = CrossRefs(
self.common_components_conf, self.logger, self.clade,
file_name, new_file_name,
self.common_components_conf['working source trees'],
'source files')
cross_refs.get_cross_refs()
def __get_original_sources_basic_info(self):
self.logger.info(
'Get information on original sources for following visualization of uncovered source files'
)
# For each source file we need to know the total number of lines and places where functions are defined.
src_files_info = dict()
for file_name, file_size in self.clade.src_info.items():
src_file_name = klever.core.utils.make_relative_path(
self.common_components_conf['working source trees'], file_name)
# Skip non-source files.
if src_file_name == file_name:
continue
src_file_name = os.path.join('source files', src_file_name)
src_files_info[src_file_name] = list()
# Store source file size.
src_files_info[src_file_name].append(file_size['loc'])
# Store source file function definition lines.
func_def_lines = list()
funcs = self.clade.get_functions_by_file([file_name], False)
if funcs:
for func_name, func_info in list(funcs.values())[0].items():
func_def_lines.append(int(func_info['line']))
src_files_info[src_file_name].append(sorted(func_def_lines))
# Dump obtain information (huge data!) to load it when reporting total code coverage if everything will be okay.
with open('original sources basic information.json', 'w') as fp:
klever.core.utils.json_dump(src_files_info, fp,
self.conf['keep intermediate files'])
def __upload_original_sources(self):
# Use Clade UUID to distinguish various original sources. It is pretty well since this UUID is uuid.uuid4().
src_id = self.clade.get_uuid()
session = klever.core.session.Session(self.logger,
self.conf['Klever Bridge'],
self.conf['identifier'])
if session.check_original_sources(src_id):
self.logger.info('Original sources were uploaded already')
self.__refer_original_sources(src_id)
return
self.logger.info(
'Cut off working source trees or build directory from original source file names and convert index data'
)
os.makedirs('original sources')
self.mqs['file names'] = multiprocessing.Queue()
self.workers_num = klever.core.utils.get_parallel_threads_num(
self.logger, self.conf)
subcomponents = [('PSFS', self.__process_source_files)]
for i in range(self.workers_num):
subcomponents.append(('RSF', self.__process_source_file))
self.launch_subcomponents(False, *subcomponents)
self.mqs['file names'].close()
self.logger.info('Compress original sources')
klever.core.utils.ArchiveFiles(['original sources'
]).make_archive('original sources.zip')
self.logger.info('Upload original sources')
try:
session.upload_original_sources(src_id, 'original sources.zip')
# Do not fail if there are already original sources. There may be complex data races because of checking and
# uploading original sources archive are not atomic.
except klever.core.session.BridgeError:
if "original sources with this identifier already exists." not in list(
session.error.values())[0]:
raise
self.__refer_original_sources(src_id)
if not self.conf['keep intermediate files']:
shutil.rmtree('original sources')
os.remove('original sources.zip')
def __get_job_or_sub_job_components(self):
self.logger.info('Get components for sub-job "{0}"'.format(self.id))
self.components = [
getattr(
importlib.import_module('.{0}'.format(component.lower()),
'klever.core'), component)
for component in self.CORE_COMPONENTS
]
self.logger.debug('Components to be launched: "{0}"'.format(', '.join(
[component.__name__ for component in self.components])))
def launch_sub_job_components(self):
"""Has callbacks"""
self.logger.info('Launch components for sub-job "{0}"'.format(self.id))
for component in self.components:
p = component(self.common_components_conf,
self.logger,
self.id,
self.callbacks,
self.mqs,
self.vals,
separate_from_parent=True)
self.component_processes.append(p)
klever.core.components.launch_workers(self.logger,
self.component_processes)
def add_models(self, generated_models):
self.logger.info('Add models to abstract verification task description')
models = []
if 'environment model' in self.abstract_task_desc:
models.append({
'model': os.path.relpath(os.path.join(self.conf['main working directory'],
self.abstract_task_desc['environment model']),
os.path.curdir),
'options': {},
'generated': True
})
if 'extra C files' in self.abstract_task_desc:
self.abstract_task_desc['extra C files'] = []
for c_file in (extra_c_file["C file"] for extra_c_file in self.abstract_task_desc['extra C files']
if "C file" in extra_c_file):
models.append(os.path.relpath(os.path.join(self.conf['main working directory'], c_file),
os.path.curdir))
def get_model_c_file(model):
# Model may be a C file or a dictionary with model file and option attributes.
if isinstance(model, dict):
return model['model']
else:
return model
# Get common and requirement specific models.
if 'common models' in self.conf and 'models' in self.conf:
for common_model_c_file in self.conf['common models']:
for model in self.conf['models']:
if common_model_c_file == get_model_c_file(model):
raise KeyError('C file "{0}" is specified in both common and requirement specific models'
.format(common_model_c_file))
if 'models' in self.conf:
# Find out actual C files.
for model in self.conf['models']:
model_c_file = get_model_c_file(model)
# Handle generated models which C files start with "$".
if model_c_file.startswith('$'):
is_generated_model_c_file_found = False
for generated_model_c_file in generated_models:
if generated_model_c_file.endswith(model_c_file[1:]):
if isinstance(model, dict):
# Specify model options for generated models that can not have model options themselves.
models.append({
'model': generated_model_c_file,
'options': model['options'],
'generated': True
})
else:
models.append({
'model': generated_model_c_file,
'options': {},
'generated': True
})
is_generated_model_c_file_found = True
if not is_generated_model_c_file_found:
raise KeyError('Model C file "{0}" was not generated'.format(model_c_file[1:]))
# Handle non-generated models.
else:
model_c_file_realpath = klever.core.vtg.utils.find_file_or_dir(
self.logger, self.conf['main working directory'], model_c_file)
self.logger.debug('Get model with C file "{0}"'.format(model_c_file_realpath))
if isinstance(model, dict):
models.append({
'model': model_c_file_realpath,
'options': model['options']
})
else:
models.append(model_c_file_realpath)
# Like for models above except for common models are always C files without any model settings.
if 'common models' in self.conf:
for common_model_c_file in self.conf['common models']:
common_model_c_file_realpath = klever.core.vtg.utils.find_file_or_dir(
self.logger, self.conf['main working directory'], common_model_c_file)
self.logger.debug('Get common model with C file "{0}"'.format(common_model_c_file_realpath))
models.append(common_model_c_file_realpath)
self.logger.debug('Resulting models are: {0}'.format(models))
if not models:
self.logger.warning('No models are specified')
return
# CC extra full description files will be put to this directory as well as corresponding intermediate and final
# output files.
os.makedirs('models'.encode('utf8'))
self.logger.info('Add aspects to abstract verification task description')
aspects = []
for model in models:
aspect = '{}.aspect'.format(os.path.splitext(get_model_c_file(model))[0])
if not os.path.isfile(aspect):
continue
self.logger.debug('Get aspect "{0}"'.format(aspect))
aspects.append(aspect)
# Sort aspects to apply them in the deterministic order.
aspects.sort()
for grp in self.abstract_task_desc['grps']:
self.logger.info('Add aspects to C files of group "{0}"'.format(grp['id']))
for extra_cc in grp['Extra CCs']:
if 'plugin aspects' not in extra_cc:
extra_cc['plugin aspects'] = []
extra_cc['plugin aspects'].append({
'plugin': self.name,
'aspects': [os.path.relpath(aspect, self.conf['main working directory']) for aspect in aspects]
})
# Generate CC full description file per each model and add it to abstract task description.
# First of all obtain CC options to be used to compile models.
clade = Clade(self.conf['build base'])
if not clade.work_dir_ok():
raise RuntimeError('Build base is not OK')
meta = clade.get_meta()
if not meta['conf'].get('Compiler.preprocess_cmds', False):
# Model compiler input file represents input file which compiler options and CWD should be used for
# compiling models. This input file is relative to one of source paths.
compiler_cmds = None
for path in self.conf['working source trees']:
try:
compiler_cmds = list(clade.get_compilation_cmds_by_file(
os.path.join(path, self.conf['model compiler input file'])))
except KeyError:
pass
if not compiler_cmds:
raise RuntimeError("There is no compiler commands for {!r}"
.format(self.conf['model compiler input file']))
elif len(compiler_cmds) > 1:
self.logger.warning("There are more than one compiler command for {!r}".
format(self.conf['model compiler input file']))
model_compiler_opts = clade.get_cmd_opts(compiler_cmds[0]['id'])
model_compiler_cwd = compiler_cmds[0]['cwd']
else:
# No specific compiler options are necessary for models.
model_compiler_opts = []
if len(self.conf['working source trees']) != 1:
raise NotImplementedError('There are several working source trees!')
model_compiler_cwd = self.conf['working source trees'][0]
model_grp = {'id': 'models', 'Extra CCs': []}
for model in sorted(models, key=get_model_c_file):
model_c_file = get_model_c_file(model)
file, ext = os.path.splitext(os.path.join('models', os.path.basename(model_c_file)))
base_name = klever.core.utils.unique_file_name(file, '{0}.json'.format(ext))
full_desc_file = '{0}{1}.json'.format(base_name, ext)
out_file = '{0}.c'.format(base_name)
# Always specify either specific model sets model or common one.
opts = ['-DLDV_SETS_MODEL_' + (model['options']['sets model']
if isinstance(model, dict) and 'sets model' in model['options']
else self.conf['common sets model']).upper()]
self.logger.debug('Dump CC full description to file "{0}"'.format(full_desc_file))
with open(full_desc_file, 'w', encoding='utf8') as fp:
klever.core.utils.json_dump({
'cwd': model_compiler_cwd,
'in': [os.path.relpath(model_c_file, os.path.realpath(clade.get_storage_path(model_compiler_cwd)))],
'out': [os.path.realpath(out_file)],
'opts': model_compiler_opts + opts
}, fp, self.conf['keep intermediate files'])
extra_cc = {'CC': os.path.relpath(full_desc_file, self.conf['main working directory'])}
if 'generated' in model:
extra_cc['generated'] = True
model_grp['Extra CCs'].append(extra_cc)
self.abstract_task_desc['grps'].append(model_grp)
for dep in self.abstract_task_desc['deps'].values():
dep.append(model_grp['id'])
def weave(self):
self.abstract_task_desc['extra C files'] = []
clade = Clade(self.conf['build base'])
# This is required to get compiler (Aspectator) specific stdarg.h since kernel C files are compiled
# with "-nostdinc" option and system stdarg.h couldn't be used.
aspectator_search_dir = '-isystem' + core.utils.execute(
self.logger, ('aspectator', '-print-file-name=include'),
collect_all_stdout=True)[0]
env = dict(os.environ)
# Print stubs instead of inline Assembler since verifiers do not interpret it and even can fail.
env['LDV_INLINE_ASM_STUB'] = ''
for grp in self.abstract_task_desc['grps']:
self.logger.info('Weave in C files of group "{0}"'.format(
grp['id']))
for extra_cc in grp['Extra CCs']:
if 'CC' in extra_cc:
if extra_cc['CC'].isdigit():
cc = clade.get_cmd(extra_cc['CC'], with_opts=True)
else:
with open(os.path.join(
self.conf['main working directory'],
extra_cc['CC']),
encoding='utf8') as fp:
cc = json.load(fp)
# extra_cc is a cc command that is not from Clade
# Thus paths in it need to be converted to be absolute
# like in other Clade commands
if "cwd" in cc and "in" in cc:
cc["in"] = [
os.path.join(cc["cwd"], cc_in)
for cc_in in cc["in"]
]
if "cwd" in cc and "out" in cc:
cc["out"] = [
os.path.join(cc["cwd"], cc_out)
for cc_out in cc["out"]
]
self.logger.info('Weave in C file "{0}"'.format(
cc['in'][0]))
cc['out'][0] = '{0}.c'.format(
core.utils.unique_file_name(
os.path.splitext(os.path.basename(
cc['out'][0]))[0], '.abs-paths.i'))
# Produce aspect to be weaved in.
if 'plugin aspects' in extra_cc:
self.logger.info(
'Concatenate all aspects of all plugins together')
# Resulting aspect.
aspect = 'aspect'
# Get all aspects. Place RSG aspects at beginning since they can instrument entities added by
# aspects of other plugins while corresponding function declarations still need be at beginning
# of file.
aspects = []
for plugin_aspects in extra_cc['plugin aspects']:
if plugin_aspects['plugin'] == 'RSG':
aspects[0:0] = plugin_aspects['aspects']
else:
aspects.extend(plugin_aspects['aspects'])
# Concatenate aspects.
with open(
aspect, 'w', encoding='utf8'
) as fout, fileinput.input(
[
os.path.join(
self.conf['main working directory'],
aspect) for aspect in aspects
],
openhook=fileinput.hook_encoded(
'utf8')) as fin:
for line in fin:
fout.write(line)
else:
# Simulate resulting aspect.
aspect = '/dev/null'
self.logger.debug(
'Aspect to be weaved in is "{0}"'.format(aspect))
core.utils.execute(
self.logger,
tuple([
'cif',
'--in',
clade.get_storage_path(cc['in'][0]),
'--aspect',
os.path.realpath(aspect),
# Besides header files specific for requirements specifications will be searched for.
'--general-opts',
'-I' + os.path.realpath(
os.path.dirname(self.conf['requirements DB'])),
'--aspect-preprocessing-opts',
' '.join(self.conf['aspect preprocessing options']
) if 'aspect preprocessing options' in
self.conf else '',
'--out',
os.path.realpath(cc['out'][0]),
'--back-end',
'src',
'--debug',
'DEBUG'
] + (['--keep'] if self.
conf['keep intermediate files'] else []) +
['--'] + core.vtg.utils.prepare_cif_opts(
self.conf, cc['opts'], clade.storage_dir) +
[aspectator_search_dir]),
env=env,
cwd=clade.get_storage_path(cc['cwd']),
timeout=0.01,
filter_func=core.vtg.utils.CIFErrorFilter())
self.logger.debug('C file "{0}" was weaved in'.format(
cc['in'][0]))
# In addition preprocess output files since CIF outputs a bit unpreprocessed files.
preprocessed_c_file = '{}.i'.format(
os.path.splitext(cc['out'][0])[0])
core.utils.execute(
self.logger, ('aspectator', '-E', '-x', 'c',
cc['out'][0], '-o', preprocessed_c_file),
timeout=0.01)
if not self.conf['keep intermediate files']:
os.remove(cc['out'][0])
self.logger.debug(
'Preprocessed weaved C file was put to "{0}"'.format(
preprocessed_c_file))
abs_paths_c_file = '{0}.abs-paths.i'.format(
os.path.splitext(cc['out'][0])[0])
with open(preprocessed_c_file,
encoding='utf8') as fp_in, open(
abs_paths_c_file, 'w',
encoding='utf8') as fp_out:
# Print preprocessor header as is.
first_line = fp_in.readline()
fp_out.write(first_line)
for line in fp_in:
fp_out.write(line)
if line == first_line:
break
# Replace relative file paths with absolute ones for line directives in other lines.
for line in fp_in:
match = re.match(r'(# \d+ ")(.+)("\n)', line)
if match:
file = match.group(2)
if not os.path.isabs(file):
# All relative file paths are relative to CC working directory.
file = os.path.abspath(
os.path.join(
os.path.realpath(clade.storage_dir)
+ cc['cwd'], file))
fp_out.write(
match.group(1) + file + match.group(3))
else:
fp_out.write(line)
if not self.conf['keep intermediate files']:
os.remove(preprocessed_c_file)
self.logger.debug(
'Preprocessed weaved C file with absolute paths was put to "{0}"'
.format(abs_paths_c_file))
extra_c_file = {
'C file':
os.path.relpath(abs_paths_c_file,
self.conf['main working directory'])
}
else:
extra_c_file = {}
if 'requirement id' in extra_cc:
extra_c_file['requirement id'] = extra_cc['requirement id']
if 'bug kinds' in extra_cc:
extra_c_file['bug kinds'] = extra_cc['bug kinds']
self.abstract_task_desc['extra C files'].append(extra_c_file)
# These sections won't be reffered any more.
del (self.abstract_task_desc['grps'])
del (self.abstract_task_desc['deps'])
class Job(klever.core.components.Component):
CORE_COMPONENTS = [
'PFG',
'VTG',
'VRP'
]
def __init__(self, conf, logger, parent_id, callbacks, mqs, vals, id=None, work_dir=None, attrs=None,
separate_from_parent=True, include_child_resources=False, components_common_conf=None):
super(Job, self).__init__(conf, logger, parent_id, callbacks, mqs, vals, id, work_dir, attrs,
separate_from_parent, include_child_resources)
self.common_components_conf = components_common_conf
if work_dir:
self.common_components_conf['additional sources directory'] = os.path.join(os.path.realpath(work_dir),
'additional sources')
self.clade = None
self.components = []
self.component_processes = []
def decide_job_or_sub_job(self):
self.logger.info('Decide job/sub-job "{0}"'.format(self.id))
# This is required to associate verification results with particular sub-jobs.
# Skip leading "/" since this identifier is used in os.path.join() that returns absolute path otherwise.
self.common_components_conf['sub-job identifier'] = self.id[1:]
self.logger.info('Get specifications set')
if 'specifications set' in self.common_components_conf:
spec_set = self.common_components_conf['specifications set']
else:
raise KeyError('Specify attribute "specifications set" within job.json')
self.logger.debug('Specifications set is "{0}"'.format(spec_set))
# Check that specifications set is supported.
with open(self.common_components_conf['specifications base'], encoding='utf-8') as fp:
req_spec_base = json.load(fp)
spec_set = self.common_components_conf['specifications set']
if spec_set not in req_spec_base['specification sets']:
raise ValueError("Klever does not support specifications set {!r} yet, available options are: {}"
.format(spec_set, ', '.join(req_spec_base['specification sets'])))
# Check and set build base here since many Core components need it.
self.__set_build_base()
self.clade = Clade(self.common_components_conf['build base'])
if not self.clade.work_dir_ok():
raise RuntimeError(f'Build base "{self.common_components_conf["build base"]}" is not OK')
self.__retrieve_working_src_trees()
self.__get_original_sources_basic_info()
self.__upload_original_sources()
# Create directory where files will be cached and remember absolute path to it for components.
os.mkdir('cache')
self.common_components_conf['cache directory'] = os.path.realpath('cache')
if self.common_components_conf['keep intermediate files']:
self.logger.debug('Create components configuration file "conf.json"')
with open('conf.json', 'w', encoding='utf-8') as fp:
json.dump(self.common_components_conf, fp, ensure_ascii=False, sort_keys=True, indent=4)
self.__get_job_or_sub_job_components()
self.callbacks = klever.core.components.get_component_callbacks(self.logger, [type(self)] + self.components)
self.launch_sub_job_components()
self.clean_dir = True
self.logger.info("All components finished")
if self.conf.get('collect total code coverage', None):
self.logger.debug('Waiting for a collecting coverage')
while not self.vals['coverage_finished'].get(self.common_components_conf['sub-job identifier'], True):
time.sleep(1)
self.logger.debug("Coverage collected")
main = decide_job_or_sub_job
def __set_build_base(self):
if 'build base' not in self.common_components_conf:
raise KeyError("Provide 'build base' configuration option to start verification")
common_advice = 'please, fix "job.json" (attribute "build base")'
common_advice += ' or/and deployment configuration file (attribute "Klever Build Bases")'
# Try to find specified build base either in normal way or additionally in directory "build bases" that is
# convenient to use when working with many build bases.
try:
build_base = klever.core.utils.find_file_or_dir(self.logger,
self.common_components_conf['main working directory'],
self.common_components_conf['build base'])
except FileNotFoundError as e:
self.logger.warning('Failed to find build base:\n{}'.format(traceback.format_exc().rstrip()))
try:
build_base = klever.core.utils.find_file_or_dir(
self.logger, self.common_components_conf['main working directory'],
os.path.join('build bases', self.common_components_conf['build base']))
except FileNotFoundError as e:
self.logger.warning('Failed to find build base:\n{}'.format(traceback.format_exc().rstrip()))
raise FileNotFoundError(
'Specified build base "{0}" does not exist, {1}'.format(self.common_components_conf['build base'],
common_advice)) from None
# Extract build base from archive. There should not be any intermediate directories in archives.
if os.path.isfile(build_base) and (tarfile.is_tarfile(build_base) or zipfile.is_zipfile(build_base)):
if tarfile.is_tarfile(build_base):
self.logger.debug('Build base "{0}" is provided in form of TAR archive'.format(build_base))
with tarfile.open(build_base) as TarFile:
TarFile.extractall('build base')
else:
self.logger.debug('Build base "{0}" is provided in form of ZIP archive'.format(build_base))
with zipfile.ZipFile(build_base) as zfp:
zfp.extractall('build base')
# Directory contains extracted build base.
extracted_from = ' extracted from "{0}"'.format(os.path.realpath(build_base))
build_base = 'build base'
else:
extracted_from = ''
# We need to specify absolute path to build base since it will be used in different Klever components. Besides,
# this simplifies troubleshooting.
build_base = os.path.realpath(build_base)
# TODO: fix after https://github.com/17451k/clade/issues/108.
if not os.path.isdir(build_base):
raise FileExistsError('Build base "{0}" is not a directory, {1}'
.format(build_base, extracted_from, common_advice))
if not os.path.isfile(os.path.join(build_base, 'meta.json')):
raise FileExistsError(
'Directory "{0}"{1} is not a build base since it does not contain file "meta.json", {2}'
.format(build_base, extracted_from, common_advice))
self.common_components_conf['build base'] = build_base
self.logger.debug('Klever components will use build base "{0}"'
.format(self.common_components_conf['build base']))
# Klever will try to cut off either working source trees (if specified) or maximum common paths of CC/CL input files
# and LD/Link output files (otherwise) from referred file names. Sometimes this is rather optional like for source
# files referred by error traces, but, say, for program fragment identifiers this is strictly necessary, e.g.
# because of otherwise expert assessment will not work as expected.
def __retrieve_working_src_trees(self):
clade_meta = self.clade.get_meta()
# Best of all if users specify working source trees in build bases manually themselves. It is a most accurate
# approach.
if 'working source trees' in clade_meta:
work_src_trees = clade_meta['working source trees']
# Otherwise try to find out them automatically as described above.
else:
in_files = []
for cmd in self.clade.get_all_cmds_by_type("CC") + self.clade.get_all_cmds_by_type("CL"):
if cmd['in']:
for in_file in cmd['in']:
# Sometimes some auxiliary stuff is built in addition to normal C source files that are most
# likely located in a place we would like to get.
if not in_file.startswith('/tmp') and in_file != '/dev/null':
in_files.append(os.path.join(cmd['cwd'], in_file))
in_files_prefix = os.path.dirname(os.path.commonprefix(in_files))
self.logger.info('Common prefix of CC/CL input files is "{0}"'.format(in_files_prefix))
out_files = []
for cmd in self.clade.get_all_cmds_by_type("LD") + self.clade.get_all_cmds_by_type("Link"):
if cmd['out']:
for out_file in cmd['out']:
# Like above.
if not out_file.startswith('/tmp') and out_file != '/dev/null':
out_files.append(os.path.join(cmd['cwd'], out_file))
out_files_prefix = os.path.dirname(os.path.commonprefix(out_files))
self.logger.info('Common prefix of LD/Link output files is "{0}"'.format(out_files_prefix))
# Meaningful paths look like "/dir...".
meaningful_paths = []
for path in (in_files_prefix, out_files_prefix):
if path and path != os.path.sep and path not in meaningful_paths:
meaningful_paths.append(path)
if meaningful_paths:
work_src_trees = meaningful_paths
# At least consider build directory as working source tree if the automatic procedure fails.
else:
self.logger.warning(
'Consider build directory "{0}" as working source tree.'
'This may be dangerous and we recommend to specify appropriate working source trees manually!'
.format(clade_meta['build_dir']))
work_src_trees = [clade_meta['build_dir']]
# Consider minimal path if it is common prefix for other ones. For instance, if we have "/dir1/dir2" and "/dir1"
# then "/dir1" will become the only working source tree.
if len(work_src_trees) > 1:
min_work_src_tree = min(work_src_trees)
if os.path.commonprefix(work_src_trees) == min_work_src_tree:
work_src_trees = [min_work_src_tree]
self.logger.info(
'Working source trees to be used are as follows:\n{0}'
.format('\n'.join([' {0}'.format(t) for t in work_src_trees])))
self.common_components_conf['working source trees'] = work_src_trees
def __refer_original_sources(self, src_id):
klever.core.utils.report(
self.logger,
'patch',
{
'identifier': self.id,
'original_sources': src_id
},
self.mqs['report files'],
self.vals['report id'],
self.conf['main working directory']
)
def __process_source_files(self):
for file_name in self.clade.src_info:
self.mqs['file names'].put(file_name)
for i in range(self.workers_num):
self.mqs['file names'].put(None)
def __process_source_file(self):
while True:
file_name = self.mqs['file names'].get()
if not file_name:
return
src_file_name = klever.core.utils.make_relative_path(self.common_components_conf['working source trees'],
file_name)
if src_file_name != file_name:
src_file_name = os.path.join('source files', src_file_name)
new_file_name = os.path.join('original sources', src_file_name.lstrip(os.path.sep))
os.makedirs(os.path.dirname(new_file_name), exist_ok=True)
shutil.copy(self.clade.get_storage_path(file_name), new_file_name)
cross_refs = CrossRefs(self.common_components_conf, self.logger, self.clade,
file_name, new_file_name,
self.common_components_conf['working source trees'], 'source files')
cross_refs.get_cross_refs()
def __get_original_sources_basic_info(self):
self.logger.info('Get information on original sources for following visualization of uncovered source files')
# For each source file we need to know the total number of lines and places where functions are defined.
src_files_info = dict()
for file_name, file_size in self.clade.src_info.items():
src_file_name = klever.core.utils.make_relative_path(self.common_components_conf['working source trees'],
file_name)
# Skip non-source files.
if src_file_name == file_name:
continue
src_file_name = os.path.join('source files', src_file_name)
src_files_info[src_file_name] = list()
# Store source file size.
src_files_info[src_file_name].append(file_size['loc'])
# Store source file function definition lines.
func_def_lines = list()
funcs = self.clade.get_functions_by_file([file_name], False)
if funcs:
for func_name, func_info in list(funcs.values())[0].items():
func_def_lines.append(int(func_info['line']))
src_files_info[src_file_name].append(sorted(func_def_lines))
# Dump obtain information (huge data!) to load it when reporting total code coverage if everything will be okay.
with open('original sources basic information.json', 'w') as fp:
klever.core.utils.json_dump(src_files_info, fp, self.conf['keep intermediate files'])
def __upload_original_sources(self):
# Use Clade UUID to distinguish various original sources. It is pretty well since this UUID is uuid.uuid4().
src_id = self.clade.get_uuid()
# In addition, take into account a meta content as we like to change it manually often. In this case it may be
# necessary to re-index the build base. It is not clear if this is the case actually, so, do this in case of
# any changes in meta.
src_id += '-' + klever.core.utils.get_file_name_checksum(json.dumps(self.clade.get_meta()))[:12]
session = klever.core.session.Session(self.logger, self.conf['Klever Bridge'], self.conf['identifier'])
if session.check_original_sources(src_id):
self.logger.info('Original sources were uploaded already')
self.__refer_original_sources(src_id)
return
self.logger.info(
'Cut off working source trees or build directory from original source file names and convert index data')
os.makedirs('original sources')
self.mqs['file names'] = multiprocessing.Queue()
self.workers_num = klever.core.utils.get_parallel_threads_num(self.logger, self.conf)
subcomponents = [('PSFS', self.__process_source_files)]
for i in range(self.workers_num):
subcomponents.append(('PSF', self.__process_source_file))
self.launch_subcomponents(False, *subcomponents)
self.mqs['file names'].close()
self.logger.info('Compress original sources')
klever.core.utils.ArchiveFiles(['original sources']).make_archive('original sources.zip')
self.logger.info('Upload original sources')
try:
session.upload_original_sources(src_id, 'original sources.zip')
# Do not fail if there are already original sources. There may be complex data races because of checking and
# uploading original sources archive are not atomic.
except klever.core.session.BridgeError:
if "original sources with this identifier already exists." not in list(session.error.values())[0]:
raise
self.__refer_original_sources(src_id)
if not self.conf['keep intermediate files']:
shutil.rmtree('original sources')
os.remove('original sources.zip')
def __get_job_or_sub_job_components(self):
self.logger.info('Get components for sub-job "{0}"'.format(self.id))
self.components = [getattr(importlib.import_module('.{0}'.format(component.lower()), 'klever.core'), component)
for component in self.CORE_COMPONENTS]
self.logger.debug('Components to be launched: "{0}"'.format(
', '.join([component.__name__ for component in self.components])))
def launch_sub_job_components(self):
"""Has callbacks"""
self.logger.info('Launch components for sub-job "{0}"'.format(self.id))
for component in self.components:
p = component(self.common_components_conf, self.logger, self.id, self.callbacks, self.mqs,
self.vals, separate_from_parent=True)
self.component_processes.append(p)
klever.core.components.launch_workers(self.logger, self.component_processes)
def request_arg_signs(self):
self.logger.info('Request argument signatures')
clade = Clade(work_dir=self.conf['build base'])
if not clade.work_dir_ok():
raise RuntimeError('Build base is not OK')
meta = clade.get_meta()
for request_aspect in self.conf['request aspects']:
request_aspect = klever.core.vtg.utils.find_file_or_dir(self.logger, self.conf['main working directory'],
request_aspect)
self.logger.debug('Request aspect is "{0}"'.format(request_aspect))
for grp in self.abstract_task_desc['grps']:
self.logger.info('Request argument signatures for C files of group "{0}"'.format(grp['id']))
for extra_cc in grp['Extra CCs']:
infile = extra_cc['in file']
self.logger.info('Request argument signatures for C file "{0}"'.format(infile))
cc = clade.get_cmd(*extra_cc['CC'], with_opts=True)
env = dict(os.environ)
env['LDV_ARG_SIGNS_FILE'] = os.path.realpath(
os.path.splitext(os.path.splitext(os.path.basename(request_aspect))[0])[0])
self.logger.debug('Argument signature file is "{0}"'
.format(os.path.relpath(env['LDV_ARG_SIGNS_FILE'])))
# Add plugin aspects produced thus far (by EMG) since they can include additional headers for which
# additional argument signatures should be extracted. Like in Weaver.
if 'plugin aspects' in extra_cc:
self.logger.info('Concatenate all aspects of all plugins together')
# Resulting request aspect.
aspect = '{0}.aspect'.format(
klever.core.utils.unique_file_name(os.path.splitext(os.path.basename(infile))[0],
'.aspect'))
# Get all aspects. Place original request aspect at beginning since it can instrument entities
# added by aspects of other plugins while corresponding function declarations still need be at
# beginning of file.
aspects = [os.path.relpath(request_aspect, self.conf['main working directory'])]
for plugin_aspects in extra_cc['plugin aspects']:
aspects.extend(plugin_aspects['aspects'])
# Concatenate aspects.
with open(aspect, 'w', encoding='utf-8') as fout, fileinput.input(
[os.path.join(self.conf['main working directory'], aspect) for aspect in aspects],
openhook=fileinput.hook_encoded('utf-8')) as fin:
for line in fin:
fout.write(line)
else:
aspect = request_aspect
storage_path = clade.get_storage_path(infile)
if meta['conf'].get('Compiler.preprocess_cmds', False) and \
'klever-core-work-dir' not in storage_path:
storage_path = storage_path.split('.c')[0] + '.i'
opts = cc['opts']
# Like in Weaver.
opts.append(klever.core.vtg.utils.define_arch_dependent_macro(self.conf))
klever.core.utils.execute(
self.logger,
tuple(
[
klever.core.vtg.utils.get_cif_or_aspectator_exec(self.conf, 'cif'),
'--in', storage_path,
'--aspect', os.path.realpath(aspect),
'--stage', 'instrumentation',
'--out', os.path.realpath('{0}.c'.format(klever.core.utils.unique_file_name(
os.path.splitext(os.path.basename(infile))[0], '.c.aux'))),
'--debug', 'DEBUG'
] +
(['--keep'] if self.conf['keep intermediate files'] else []) +
['--'] +
klever.core.vtg.utils.prepare_cif_opts(opts, clade) +
# Like in Weaver.
['-I' + os.path.join(os.path.dirname(self.conf['specifications base']), 'include')]
),
env,
cwd=clade.get_storage_path(cc['cwd']),
timeout=0.01,
filter_func=klever.core.vtg.utils.CIFErrorFilter())