def build(self, build_commands_file: str):
if not self.is_build:
return
os.chdir(self.source_dir)
# Remove Clade working directory and temporary files
tmp_path = os.path.join(self.source_dir, CLADE_WORK_DIR)
if os.path.exists(tmp_path):
shutil.rmtree(tmp_path)
tmp_path = os.path.join(self.source_dir, CLADE_BASE_FILE)
if os.path.exists(tmp_path):
os.remove(tmp_path)
if self.command_caller(self.make_clean_command):
self.logger.warning("Make clean failed")
if os.path.exists(self.env_path):
os.environ["PATH"] += os.pathsep + self.env_path
self.logger.debug("Using Clade tool to build sources with '{}'".format(
self.make_command))
try:
# noinspection PyUnresolvedReferences
from clade import Clade
c = Clade(CLADE_WORK_DIR, CLADE_BASE_FILE, conf=self.clade_conf)
c.intercept(str(self.make_command).split())
c.parse("SrcGraph")
cmds = c.compilation_cmds
for cmd in cmds:
identifier = cmd['id']
cmd['command'] = c.get_cmd_raw(identifier)[0]
cmd['opts'] = c.get_cmd_opts(identifier)
with open(build_commands_file, "w") as fd:
json.dump(cmds, fd, sort_keys=True, indent="\t")
except Exception:
error_msg = "Building has failed due to: {}".format(
traceback.format_exc())
if self.fail_if_failure:
sys.exit(error_msg)
else:
self.logger.warning(error_msg)
os.chdir(self.work_dir)
self.logger.info("Sources has been successfully built")
def add_models(self, generated_models):
self.logger.info(
'Add models to abstract verification task description')
models = {}
if 'environment model' in self.abstract_task_desc:
rel_path = os.path.relpath(
os.path.join(self.conf['main working directory'],
self.abstract_task_desc['environment model']),
os.path.curdir)
models[rel_path] = {}
# Get common and requirement specific models.
if 'common models' in self.conf and 'models' in self.conf:
for common_model_c_file in self.conf['common models']:
if common_model_c_file in self.conf['models']:
raise KeyError(
'C file "{0}" is specified in both common and requirement specific models'
)
if 'models' in self.conf:
for model_c_file in self.conf['models']:
# Specify additional settings for generated models that have not any settings.
if model_c_file.startswith('$'):
is_generated_model_c_file_found = False
for generated_model_c_file in generated_models:
if generated_model_c_file.endswith(model_c_file[1:]):
models[generated_model_c_file] = self.conf[
'models'][model_c_file]
is_generated_model_c_file_found = True
if not is_generated_model_c_file_found:
raise KeyError(
'Model C file "{0}" was not generated'.format(
model_c_file[1:]))
# Like common models processed below.
for model_c_file in self.conf['models']:
if not model_c_file.startswith('$'):
model_c_file_realpath = core.vtg.utils.find_file_or_dir(
self.logger, self.conf['main working directory'],
model_c_file)
self.logger.debug('Get model with C file "{0}"'.format(
model_c_file_realpath))
models[model_c_file_realpath] = self.conf['models'][
model_c_file]
if 'common models' in self.conf:
for common_model_c_file in self.conf['common models']:
common_model_c_file_realpath = core.vtg.utils.find_file_or_dir(
self.logger, self.conf['main working directory'],
common_model_c_file)
self.logger.debug('Get common model with C file "{0}"'.format(
common_model_c_file_realpath))
models[common_model_c_file_realpath] = self.conf[
'common models'][common_model_c_file]
self.logger.debug('Resulting models are: {0}'.format(models))
if not models:
self.logger.warning('No models are specified')
return
# CC extra full description files will be put to this directory as well as corresponding intermediate and final
# output files.
os.makedirs('models'.encode('utf8'))
self.logger.info(
'Add aspects to abstract verification task description')
aspects = []
for model_c_file in models:
aspect = '{}.aspect'.format(os.path.splitext(model_c_file)[0])
if not os.path.isfile(aspect):
continue
self.logger.debug('Get aspect "{0}"'.format(aspect))
aspects.append(aspect)
# Sort aspects to apply them in the deterministic order.
aspects.sort()
for grp in self.abstract_task_desc['grps']:
self.logger.info('Add aspects to C files of group "{0}"'.format(
grp['id']))
for extra_cc in grp['Extra CCs']:
if 'plugin aspects' not in extra_cc:
extra_cc['plugin aspects'] = []
extra_cc['plugin aspects'].append({
'plugin':
self.name,
'aspects': [
os.path.relpath(aspect,
self.conf['main working directory'])
for aspect in aspects
]
})
for model_c_file in models:
model = models[model_c_file]
if 'bug kinds' in model:
self.logger.info(
'Preprocess bug kinds for model with C file "{0}"'.format(
model_c_file))
# Collect all bug kinds specified in model to check that valid bug kinds are specified in requirement
# model description.
bug_kinds = set()
lines = []
with open(model_c_file, encoding='utf8') as fp:
for line in fp:
# Bug kinds are specified in form of strings like in requirements DB as first actual
# parameters of ldv_assert().
match = re.search(r'ldv_assert\("([^"]+)"', line)
if match:
bug_kind, = match.groups()
bug_kinds.add(bug_kind)
# Include bug kinds in names of ldv_assert().
lines.append(
re.sub(
r'ldv_assert\("([^"]+)", ?',
r'ldv_assert_{0}('.format(
re.sub(r'\W', '_', bug_kind)), line))
else:
lines.append(line)
for bug_kind in model['bug kinds']:
if bug_kind not in bug_kinds:
raise KeyError(
'Invalid bug kind "{0}" is specified in requirement model description'
.format(bug_kind))
preprocessed_model_c_file = '{0}.bk.c'.format(
core.utils.unique_file_name(
os.path.join(
'models',
os.path.splitext(
os.path.basename(model_c_file))[0]), '.bk.c'))
with open(preprocessed_model_c_file, 'w',
encoding='utf8') as fp:
# Create ldv_assert*() function declarations to avoid compilation warnings. These functions will
# be defined later somehow by VTG.
for bug_kind in sorted(bug_kinds):
fp.write('extern void ldv_assert_{0}(int);\n'.format(
re.sub(r'\W', '_', bug_kind)))
# Specify original location to avoid references to *.bk.c files in error traces.
fp.write('# 1 "{0}"\n'.format(
os.path.abspath(model_c_file)))
for line in lines:
fp.write(line)
model[
'bug kinds preprocessed C file'] = preprocessed_model_c_file
self.logger.debug(
'Preprocessed bug kinds for model with C file "{0}" was placed to "{1}"'
.format(model_c_file, preprocessed_model_c_file))
else:
model['bug kinds preprocessed C file'] = model_c_file
# Generate CC full description file per each model and add it to abstract task description.
# First of all obtain CC options to be used to compile models.
clade = Clade(self.conf['build base'])
# Relative path to source file which CC options to be used is specified in configuration. Clade needs absolute
# path. The former is relative to one of source paths.
for path in self.conf['source paths']:
opts_file = os.path.join(path, self.conf['opts file'])
try:
empty_cc = list(clade.get_compilation_cmds_by_file(opts_file))
except KeyError:
pass
if not empty_cc:
raise RuntimeError("There is not of cc commands for {!r}".format(
self.conf['project']['opts file']))
elif len(empty_cc) > 1:
self.logger.warning(
"There are more than one cc command for {!r}".format(
self.conf['project']['opts file']))
empty_cc = empty_cc.pop()
empty_cc['opts'] = clade.get_cmd_opts(empty_cc['id'])
model_grp = {'id': 'models', 'Extra CCs': []}
for model_c_file in sorted(models):
model = models[model_c_file]
extra_cc = {}
if 'bug kinds preprocessed C file' in model:
file, ext = os.path.splitext(
os.path.join(
'models',
os.path.basename(
model['bug kinds preprocessed C file'])))
base_name = core.utils.unique_file_name(
file, '{0}.json'.format(ext))
full_desc_file = '{0}{1}.json'.format(base_name, ext)
out_file = '{0}.c'.format(base_name)
self.logger.debug(
'Dump CC full description to file "{0}"'.format(
full_desc_file))
with open(full_desc_file, 'w', encoding='utf8') as fp:
json.dump(
{
'cwd':
empty_cc['cwd'],
'in': [
os.path.relpath(
model['bug kinds preprocessed C file'],
os.path.realpath(
clade.get_storage_path(
empty_cc['cwd'])))
],
'out': [os.path.realpath(out_file)],
'opts':
empty_cc['opts'] + [
'-DLDV_SETS_MODEL_' +
(model['sets model'] if 'sets model' in model
else self.conf['common sets model']).upper()
]
},
fp,
ensure_ascii=False,
sort_keys=True,
indent=4)
extra_cc['CC'] = os.path.relpath(
full_desc_file, self.conf['main working directory'])
if 'bug kinds' in model:
extra_cc['bug kinds'] = model['bug kinds']
if extra_cc:
model_grp['Extra CCs'].append(extra_cc)
self.abstract_task_desc['grps'].append(model_grp)
for dep in self.abstract_task_desc['deps'].values():
dep.append(model_grp['id'])
def add_models(self, generated_models):
self.logger.info(
'Add models to abstract verification task description')
models = []
if 'environment model' in self.abstract_task_desc:
models.append({
'model':
os.path.relpath(
os.path.join(self.conf['main working directory'],
self.abstract_task_desc['environment model']),
os.path.curdir),
'options': {},
'generated':
True
})
if 'extra C files' in self.abstract_task_desc:
self.abstract_task_desc['extra C files'] = []
for c_file in (extra_c_file["C file"] for extra_c_file in
self.abstract_task_desc['extra C files']
if "C file" in extra_c_file):
models.append(
os.path.relpath(
os.path.join(self.conf['main working directory'],
c_file), os.path.curdir))
def get_model_c_file(model):
# Model may be a C file or a dictionary with model file and option attributes.
if isinstance(model, dict):
return model['model']
else:
return model
# Get common and requirement specific models.
if 'exclude common models' in self.conf:
self.logger.info('Common models to be excluded:\n{0}'.format(
'\n'.join([
' {0}'.format(m)
for m in self.conf['exclude common models']
])))
common_models = [
m for m in self.conf['common models']
if m not in self.conf['exclude common models']
]
else:
common_models = self.conf['common models']
if common_models and 'models' in self.conf:
for common_model in common_models:
common_model_c_file = get_model_c_file(common_model)
for model in self.conf['models']:
if common_model_c_file == get_model_c_file(model):
raise KeyError(
'C file "{0}" is specified in both common and requirement specific models'
.format(common_model_c_file))
def add_model(model, model_c_file_realpath):
if isinstance(model, dict):
models.append({
'model': model_c_file_realpath,
'options': model['options']
})
else:
models.append(model_c_file_realpath)
if 'models' in self.conf:
# Find out actual C files.
for model in self.conf['models']:
model_c_file = get_model_c_file(model)
# Handle generated models which C files start with "$".
if model_c_file.startswith('$'):
is_generated_model_c_file_found = False
for generated_model_c_file in generated_models:
if generated_model_c_file.endswith(model_c_file[1:]):
if isinstance(model, dict):
# Specify model options for generated models that can not have model options themselves.
models.append({
'model': generated_model_c_file,
'options': model['options'],
'generated': True
})
else:
models.append({
'model': generated_model_c_file,
'options': {},
'generated': True
})
is_generated_model_c_file_found = True
if not is_generated_model_c_file_found:
raise KeyError(
'Model C file "{0}" was not generated'.format(
model_c_file[1:]))
# Handle non-generated models.
else:
model_c_file_realpath = klever.core.vtg.utils.find_file_or_dir(
self.logger, self.conf['main working directory'],
model_c_file)
self.logger.debug('Get model with C file "{0}"'.format(
model_c_file_realpath))
add_model(model, model_c_file_realpath)
# Like for models above.
for common_model in common_models:
common_model_c_file = get_model_c_file(common_model)
common_model_c_file_realpath = klever.core.vtg.utils.find_file_or_dir(
self.logger, self.conf['main working directory'],
common_model_c_file)
self.logger.debug('Get common model with C file "{0}"'.format(
common_model_c_file_realpath))
add_model(common_model, common_model_c_file_realpath)
self.logger.debug('Resulting models are: {0}'.format(models))
if not models:
self.logger.warning('No models are specified')
return
# CC extra full description files will be put to this directory as well as corresponding intermediate and final
# output files.
os.makedirs('models'.encode('utf-8'))
self.logger.info(
'Add aspects to abstract verification task description')
aspects = []
for model in models:
aspect = '{}.aspect'.format(
os.path.splitext(get_model_c_file(model))[0])
# Aspects are not mandatory. There may be pure C models, e.g. when one needs to provide some definitions
# without any weaving.
if not os.path.isfile(aspect):
continue
if not os.stat(aspect).st_size:
raise ValueError(
'Aspect "{0}" is empty and should be removed from the verification job'
.format(aspect))
self.logger.debug('Get aspect "{0}"'.format(aspect))
aspects.append(aspect)
# Sort aspects to apply them in the deterministic order.
aspects.sort()
# Always specify either specific model sets model or common one.
opts = [
'-DLDV_SETS_MODEL_' +
(model['options']['sets model']
if isinstance(model, dict) and 'sets model' in model['options']
else self.conf['common sets model']).upper()
]
if self.conf.get('memory safety'):
opts += ['-DLDV_MEMORY_SAFETY']
if 'specifications set' in self.conf:
opts += [
'-DLDV_SPECS_SET_{0}'.format(
self.conf['specifications set'].replace('.', '_'))
]
for grp in self.abstract_task_desc['grps']:
self.logger.info('Add aspects to C files of group "{0}"'.format(
grp['id']))
for extra_cc in grp['Extra CCs']:
if 'plugin aspects' not in extra_cc:
extra_cc['plugin aspects'] = []
extra_cc['plugin aspects'].append({
'plugin':
self.name,
'aspects': [
os.path.relpath(aspect,
self.conf['main working directory'])
for aspect in aspects
]
})
extra_cc['opts'] = opts
# Generate CC full description file per each model and add it to abstract task description.
# First of all obtain CC options to be used to compile models.
clade = Clade(self.conf['build base'])
if not clade.work_dir_ok():
raise RuntimeError('Build base is not OK')
meta = clade.get_meta()
if not meta['conf'].get('Compiler.preprocess_cmds', False):
# Model compiler input file represents input file which compiler options and CWD should be used for
# compiling models. This input file is relative to one of source paths.
compiler_cmds = None
for path in self.conf['working source trees']:
try:
compiler_cmds = list(
clade.get_compilation_cmds_by_file(
os.path.normpath(
os.path.join(
path,
self.conf['model compiler input file']))))
break
except KeyError:
pass
if not compiler_cmds:
raise RuntimeError(
"There is no compiler commands for {!r}".format(
self.conf['model compiler input file']))
elif len(compiler_cmds) > 1:
self.logger.warning(
"There are more than one compiler command for {!r}".format(
self.conf['model compiler input file']))
model_compiler_opts = clade.get_cmd_opts(compiler_cmds[0]['id'])
model_compiler_cwd = compiler_cmds[0]['cwd']
else:
# No specific compiler options are necessary for models.
model_compiler_opts = []
if len(self.conf['working source trees']) != 1:
raise NotImplementedError(
'There are several working source trees!')
model_compiler_cwd = self.conf['working source trees'][0]
model_grp = {'id': 'models', 'Extra CCs': []}
for model in sorted(models, key=get_model_c_file):
model_c_file = get_model_c_file(model)
file, ext = os.path.splitext(
os.path.join('models', os.path.basename(model_c_file)))
base_name = klever.core.utils.unique_file_name(
file, '{0}.json'.format(ext))
full_desc_file = '{0}{1}.json'.format(base_name, ext)
out_file = '{0}.c'.format(base_name)
self.logger.debug('Dump CC full description to file "{0}"'.format(
full_desc_file))
with open(full_desc_file, 'w', encoding='utf-8') as fp:
klever.core.utils.json_dump(
{
'cwd': model_compiler_cwd,
'in': [os.path.realpath(model_c_file)],
'out': [os.path.realpath(out_file)],
'opts': model_compiler_opts + opts
}, fp, self.conf['keep intermediate files'])
extra_cc = {
'CC':
os.path.relpath(full_desc_file,
self.conf['main working directory'])
}
if 'generated' in model:
extra_cc['generated'] = True
if isinstance(model, dict):
if model['options'].get('weave in model aspect'):
aspect = '{}.aspect'.format(
os.path.splitext(get_model_c_file(model))[0])
if not os.path.isfile(aspect):
raise FileNotFoundError(
'Aspect "{0}" to be weaved in model does not exist'
.format(aspect))
extra_cc['plugin aspects'] = [{
'plugin':
self.name,
'aspects': [
os.path.relpath(
aspect, self.conf['main working directory'])
]
}]
elif model['options'].get('weave in all aspects'):
extra_cc['plugin aspects'] = [{
'plugin':
self.name,
'aspects': [
os.path.relpath(
aspect, self.conf['main working directory'])
for aspect in aspects
]
}]
model_grp['Extra CCs'].append(extra_cc)
self.abstract_task_desc['grps'].append(model_grp)
for dep in self.abstract_task_desc['deps'].values():
dep.append(model_grp['id'])
