def clade_api(tmpdir_factory, cmds_file):
tmpdir = tmpdir_factory.mktemp("Clade")
c = Clade(tmpdir, cmds_file)
c.parse_list(["CrossRef", "Variables", "Macros", "Typedefs", "CDB"])
yield c
def build(self):
self._fetch_work_src_tree()
self._make_canonical_work_src_tree()
self._clean()
self._get_version()
if self.version:
self.logger.info('C program version is "{0}"'.format(self.version))
self._configure()
if self.configuration:
self.logger.info('C program configuration is "{0}"'.format(
self.configuration))
self._build()
if os.path.isdir(self.target_program_desc['build base']):
shutil.rmtree(self.target_program_desc['build base'])
if 'extra Clade options' in self.target_program_desc:
clade_conf = dict(self._CLADE_CONF)
clade_conf.update(self.target_program_desc['extra Clade options'])
else:
clade_conf = self._CLADE_CONF
clade = Clade(work_dir=self.target_program_desc['build base'],
cmds_file=os.path.join(self.work_src_tree, 'cmds.txt'),
conf=clade_conf,
preset=self._CLADE_PRESET)
clade.parse_list(
["CrossRef", "Callgraph", "Variables", "Typedefs", "Macros"])
self.logger.info(
'Save project attributes, working source trees and target program description to build base'
)
clade.add_meta_by_key('project attrs', [{
'name':
'project',
'value': [{
'name': 'name',
'value': type(self).__name__
}, {
'name': 'architecture',
'value': self.architecture
}, {
'name': 'version',
'value': self.version
}, {
'name': 'configuration',
'value': self.configuration
}]
}])
clade.add_meta_by_key('working source trees', self.work_src_trees)
clade.add_meta_by_key('target program description',
self.target_program_desc)
self.logger.info('Remove temporary directories')
for tmp_dir in self.tmp_dirs:
shutil.rmtree(tmp_dir)
def clade_api(tmpdir_factory):
tmpdir = tmpdir_factory.mktemp("Clade")
c = Clade(tmpdir)
c.intercept(command=test_project_make,
use_wrappers=True,
intercept_envs=True)
c.parse_list(["CrossRef", "Variables", "Macros", "Typedefs", "CDB"])
yield c
def test_tracer(tmpdir, cmds_file):
c = Clade(tmpdir, cmds_file, preset="klever_linux_kernel")
c.parse_list(c.conf["extensions"])
print(c.work_dir)
t = Tracer(c.work_dir)
from_func = t.find_functions(["main"])[0]
to_func = t.find_functions(["printf"])[0]
trace = t.trace(from_func, to_func)
assert len(trace) == 2
def __get_cross_refs(self, infile, opts, outfile, clade, cwd,
aspectator_search_dir):
# Get cross references and everything required for them.
# Limit parallel workers in Clade by 4 since at this stage there may be several parallel task generators and we
# prefer their parallelism over the Clade default one.
clade_extra = Clade(work_dir=os.path.realpath(outfile + ' clade'),
preset=self.conf['Clade']['preset'],
conf={'cpu_count': 4})
# TODO: this can be incorporated into instrumentation above but it will need some Clade changes.
# Emulate normal compilation (indeed just parsing thanks to "-fsyntax-only") to get additional
# dependencies (model source files) and information on them.
clade_extra.intercept([
klever.core.vtg.utils.get_cif_or_aspectator_exec(
self.conf, 'aspectator'),
'-I' + os.path.join(
os.path.dirname(self.conf['specifications base']), 'include')
] + klever.core.vtg.utils.prepare_cif_opts(opts, clade, True) +
[aspectator_search_dir, '-fsyntax-only', infile],
cwd=cwd)
clade_extra.parse_list(["CrossRef"])
if not clade_extra.work_dir_ok():
raise RuntimeError('Build base is not OK')
# Like in klever.core.job.Job#__upload_original_sources.
os.makedirs(outfile + ' additional sources')
for root, dirs, files in os.walk(clade_extra.storage_dir):
for file in files:
file = os.path.join(root, file)
storage_file = klever.core.utils.make_relative_path(
[clade_extra.storage_dir], file)
# Do not treat those source files that were already processed and uploaded as original sources.
if os.path.commonpath([
os.path.join(os.path.sep, storage_file),
clade.storage_dir
]) == clade.storage_dir:
continue
new_file = klever.core.utils.make_relative_path(
self.search_dirs, storage_file, absolutize=True)
# These source files do not belong neither to original sources nor to models, e.g. there are compiler
# headers.
if os.path.isabs(new_file):
continue
# We treat all remaining source files which paths do not start with "specifications" as generated
# models. This is not correct for all cases, e.g. when users put some files within $KLEVER_DATA_DIR.
if not new_file.startswith('specifications'):
new_file = os.path.join('generated models', new_file)
new_file = os.path.join(outfile + ' additional sources',
new_file)
os.makedirs(os.path.dirname(new_file), exist_ok=True)
shutil.copy(file, new_file)
cross_refs = CrossRefs(self.conf, self.logger, clade_extra,
os.path.join(os.path.sep, storage_file),
new_file, self.search_dirs)
cross_refs.get_cross_refs()
self.__merge_additional_srcs(outfile + ' additional sources')
if not self.conf['keep intermediate files']:
shutil.rmtree(outfile + ' clade')
def main(sys_args=sys.argv[1:]):
args = parse_args(sys_args)
conf = prepare_conf(args)
build_exit_code = 0
# Create Clade interface object
try:
c = Clade(work_dir=conf["work_dir"], cmds_file=conf["cmds_file"], conf=conf, preset=args.preset)
except RuntimeError as e:
raise SystemExit(e)
if os.path.isfile(conf["cmds_file"]) and args.intercept and not args.append:
c.logger.info("File with intercepted commands already exists: {!r}".format(conf["cmds_file"]))
sys.exit(-1)
elif os.path.isfile(conf["cmds_file"]) and not args.intercept and not args.append:
c.logger.info("Skipping build and reusing {!r} file".format(conf["cmds_file"]))
else:
if not args.command:
c.logger.error("Build command is missing")
sys.exit(-1)
c.logger.info("Starting build")
build_time_start = time.time()
build_exit_code = c.intercept(
conf["build_command"],
use_wrappers=conf["use_wrappers"],
append=args.append,
intercept_open=args.intercept_open,
intercept_envs=args.intercept_envs
)
build_delta = datetime.timedelta(seconds=(time.time() - build_time_start))
build_delta_str = str(build_delta).split(".")[0]
# Clade can still proceed further if exit code != 0
c.logger.error("Build finished in {} with exit code {}".format(build_delta_str, build_exit_code))
if args.intercept and os.path.exists(conf["cmds_file"]):
c.logger.info("Path to the file with intercepted commands: {!r}".format(conf["cmds_file"]))
sys.exit(build_exit_code)
if not os.path.exists(conf["cmds_file"]):
c.logger.error("Something is wrong: file with intercepted commands is empty")
sys.exit(-1)
try:
extensions = args.extension if args.extension else c.conf["extensions"]
c.logger.info("Executing extensions")
ext_time_start = time.time()
c.parse_list(extensions, args.force_exts)
ext_delta = datetime.timedelta(seconds=(time.time() - ext_time_start))
ext_delta_str = str(ext_delta).split(".")[0]
c.logger.info("Extensions finished in {}".format(ext_delta_str))
if build_exit_code != 0:
c.logger.error("Reminder that build finished with exit code {}".format(build_exit_code))
except RuntimeError as e:
if e.args:
raise SystemExit(e)
else:
raise SystemExit(-1)
sys.exit(0)
class Program:
_CLADE_CONF = dict()
_CLADE_PRESET = "base"
def __init__(self, logger, target_program_desc):
self.logger = logger
self.target_program_desc = target_program_desc
# Main working source tree where various build and auxiliary actions will be performed.
self.work_src_tree = self.target_program_desc['source code']
# Program attributes. We expect that architecture is always specified in the target program description while
# configuration and version can be either obtained during build somehow or remained unspecified.
self.architecture = self.target_program_desc['architecture']
self.configuration = None
self.version = self.target_program_desc.get('version')
# Working source trees are directories to be trimmed from file names.
self.work_src_trees = []
# Temporary directories that should be removed at the end of work.
self.tmp_dirs = []
# Path to the Clade cmds.txt file with intercepted commands
self.cmds_file = os.path.realpath(os.path.join(self.work_src_tree, 'cmds.txt'))
# Clade API object
clade_conf = dict(self._CLADE_CONF)
clade_conf.update(self.target_program_desc.get('extra Clade options', dict()))
self.clade = Clade(work_dir=self.target_program_desc['build base'],
cmds_file=self.cmds_file,
conf=clade_conf,
preset=self._CLADE_PRESET)
def _prepare_work_src_tree(self):
o = urllib.parse.urlparse(self.work_src_tree)
if o[0] in ('http', 'https', 'ftp'):
raise NotImplementedError('Source code is provided in unsupported form of a remote archive')
elif o[0] == 'git':
raise NotImplementedError('Source code is provided in unsupported form of a remote Git repository')
elif o[0]:
raise ValueError('Source code is provided in unsupported form "{0}"'.format(o[0]))
if os.path.isfile(self.work_src_tree):
raise NotImplementedError('Source code is provided in unsupported form of an archive')
# Local git repository
if os.path.isdir(os.path.join(self.work_src_tree, '.git')):
self.logger.debug("Source code is provided in form of a Git repository")
self.__prepare_git_work_src_tree()
self.work_src_trees.append(os.path.realpath(self.work_src_tree))
def __prepare_git_work_src_tree(self):
if 'git repository version' not in self.target_program_desc:
return
checkout = self.target_program_desc['git repository version']
self.logger.info(f'Checkout Git repository "{checkout}"')
# Repository lock file may remain from some previous crashed git command
git_index_lock = os.path.join(self.work_src_tree, '.git', 'index.lock')
if os.path.isfile(git_index_lock):
os.remove(git_index_lock)
# In case of dirty Git working directory checkout may fail so clean up it first.
execute_cmd(self.logger, 'git', 'clean', '-f', '-d', cwd=self.work_src_tree)
execute_cmd(self.logger, 'git', 'reset', '--hard', cwd=self.work_src_tree)
execute_cmd(self.logger, 'git', 'checkout', '-f', checkout, cwd=self.work_src_tree)
try:
# Use Git describe to properly identify program version
stdout = execute_cmd(self.logger, 'git', 'describe', cwd=self.work_src_tree, get_output=True)
self.version = stdout[0]
except subprocess.CalledProcessError:
# Use Git repository version from target program description if Git describe failed
self.version = checkout
def _run_clade(self):
if os.path.isdir(self.target_program_desc['build base']):
shutil.rmtree(self.target_program_desc['build base'])
self.clade.parse_list(self.clade.conf['extensions'])
self.logger.info('Save project attributes, working source trees and target program description to build base')
attrs = [
{
'name': 'name',
'value': type(self).__name__
},
{
'name': 'architecture',
'value': self.architecture
},
{
'name': 'version',
'value': self.version
}
]
if self.configuration:
attrs.append({
'name': 'configuration',
'value': self.configuration
})
self.clade.add_meta_by_key('project attrs', [{
'name': 'project',
'value': attrs
}])
self.clade.add_meta_by_key('working source trees', self.work_src_trees)
self.clade.add_meta_by_key('target program description', self.target_program_desc)
@staticmethod
def build_wrapper(build):
'''Wrapper for build() method'''
def wrapper(self, *args, **kwargs):
try:
return build(self, *args, **kwargs)
finally:
for tmp_dir in self.tmp_dirs:
self.logger.info(f'Remove temporary directory "{tmp_dir}"')
shutil.rmtree(tmp_dir)
if os.path.exists(self.cmds_file):
os.remove(self.cmds_file)
return wrapper
def build(self):
...
def main(sys_args=sys.argv[1:]):
args = parse_args(sys_args)
conf = prepare_conf(args)
# Create Clade interface object
try:
c = Clade(work_dir=conf["work_dir"],
cmds_file=conf["cmds_file"],
conf=conf,
preset=args.preset)
except RuntimeError as e:
raise SystemExit(e)
if os.path.isfile(
conf["cmds_file"]) and args.intercept and not args.append:
c.logger.info(
"File with intercepted commands already exists: {!r}".format(
conf["cmds_file"]))
sys.exit(-1)
elif os.path.isfile(conf["cmds_file"]) and not args.append:
c.logger.info("Skipping build and reusing {!r} file".format(
conf["cmds_file"]))
else:
if not args.command:
c.logger.error("Build command is missing")
sys.exit(-1)
c.logger.info("Starting build")
r = c.intercept(conf["command"],
use_wrappers=conf["use_wrappers"],
append=args.append)
if r:
# Clade can still proceed further
c.logger.error("Build failed with error code {}".format(r))
else:
c.logger.info("Build completed successfully")
if args.intercept:
if os.path.exists(conf["cmds_file"]):
c.logger.info(
"Path to the file with intercepted commands: {!r}".format(
conf["cmds_file"]))
sys.exit(r)
else:
c.logger.error(
"Something is wrong: file with intercepted commands is empty")
sys.exit(-1)
try:
extensions = args.extension if args.extension else c.conf["extensions"]
c.logger.info("Executing extensions")
c.parse_list(extensions, args.force_exts)
c.logger.info("Executing extensions finished")
except RuntimeError as e:
if e.args:
raise SystemExit(e)
else:
raise SystemExit(-1)
sys.exit(0)
