def envs_file():
# Disable multiprocessing
os.environ["CLADE_DEBUG"] = "1"
c = Clade(work_dir=test_project + '/clade')
c.intercept(command=test_project_make,
use_wrappers=True,
intercept_envs=True)
yield os.path.join(c.work_dir, "envs.txt")
def clade_api(tmpdir_factory):
tmpdir = tmpdir_factory.mktemp("Clade")
c = Clade(tmpdir)
c.intercept(command=test_project_make,
use_wrappers=True,
intercept_envs=True)
c.parse_list(["CrossRef", "Variables", "Macros", "Typedefs", "CDB"])
yield c
def _make(self,
*target,
opts=None,
env=None,
intercept_build_cmds=False,
get_output=False):
if opts is None:
opts = []
cmd = ['make', '-j', self.jobs] + opts + list(target)
if intercept_build_cmds:
clade = Clade(cmds_file=os.path.realpath(
os.path.join(self.work_src_tree, 'cmds.txt')))
r = clade.intercept(cmd, append=True, cwd=self.work_src_tree)
if r:
raise RuntimeError('Build failed')
return r
else:
return execute_cmd(self.logger,
*(cmd),
cwd=self.work_src_tree,
env=env,
get_output=get_output)
def main(args=sys.argv[1:]):
work_dir = tempfile.mkdtemp()
args = parse_args(args, work_dir)
conf = prepare_conf(args)
try:
c = Clade(work_dir, args.cmds, conf, args.preset)
except RuntimeError as e:
raise SystemExit(e)
if args.command and not os.path.isfile(args.cmds):
c.intercept(args.command, use_wrappers=args.wrappers)
c.parse("CDB")
shutil.rmtree(work_dir)
def test_intercept(tmpdir):
output = os.path.join(str(tmpdir), "cmds.txt")
c = Clade(cmds_file=output)
assert not c.intercept(command=test_project_make, use_wrappers=True)
assert os.path.isfile(output)
assert calculate_loc(output) > 1
def build(self, build_commands_file: str):
if not self.is_build:
return
os.chdir(self.source_dir)
# Remove Clade working directory and temporary files
tmp_path = os.path.join(self.source_dir, CLADE_WORK_DIR)
if os.path.exists(tmp_path):
shutil.rmtree(tmp_path)
tmp_path = os.path.join(self.source_dir, CLADE_BASE_FILE)
if os.path.exists(tmp_path):
os.remove(tmp_path)
if self.command_caller(self.make_clean_command):
self.logger.warning("Make clean failed")
if os.path.exists(self.env_path):
os.environ["PATH"] += os.pathsep + self.env_path
self.logger.debug("Using Clade tool to build sources with '{}'".format(
self.make_command))
try:
# noinspection PyUnresolvedReferences
from clade import Clade
c = Clade(CLADE_WORK_DIR, CLADE_BASE_FILE, conf=self.clade_conf)
c.intercept(str(self.make_command).split())
c.parse("SrcGraph")
cmds = c.compilation_cmds
for cmd in cmds:
identifier = cmd['id']
cmd['command'] = c.get_cmd_raw(identifier)[0]
cmd['opts'] = c.get_cmd_opts(identifier)
with open(build_commands_file, "w") as fd:
json.dump(cmds, fd, sort_keys=True, indent="\t")
except Exception:
error_msg = "Building has failed due to: {}".format(
traceback.format_exc())
if self.fail_if_failure:
sys.exit(error_msg)
else:
self.logger.warning(error_msg)
os.chdir(self.work_dir)
self.logger.info("Sources has been successfully built")
def test_windows(tmpdir):
work_dir = os.path.join(str(tmpdir), "clade")
output = os.path.join(str(tmpdir), "cmds.txt")
c = Clade(work_dir, cmds_file=output)
assert not c.intercept(command=test_build)
c.parse("SrcGraph")
assert c.pid_graph
assert c.cmds
assert c.cmd_graph
assert c.src_graph
def __get_cross_refs(self, infile, opts, outfile, clade, cwd,
aspectator_search_dir):
# Get cross references and everything required for them.
# Limit parallel workers in Clade by 4 since at this stage there may be several parallel task generators and we
# prefer their parallelism over the Clade default one.
clade_extra = Clade(work_dir=os.path.realpath(outfile + ' clade'),
preset=self.conf['Clade']['preset'],
conf={'cpu_count': 4})
# TODO: this can be incorporated into instrumentation above but it will need some Clade changes.
# Emulate normal compilation (indeed just parsing thanks to "-fsyntax-only") to get additional
# dependencies (model source files) and information on them.
clade_extra.intercept([
klever.core.vtg.utils.get_cif_or_aspectator_exec(
self.conf, 'aspectator'),
'-I' + os.path.join(
os.path.dirname(self.conf['specifications base']), 'include')
] + klever.core.vtg.utils.prepare_cif_opts(opts, clade, True) +
[aspectator_search_dir, '-fsyntax-only', infile],
cwd=cwd)
clade_extra.parse_list(["CrossRef"])
if not clade_extra.work_dir_ok():
raise RuntimeError('Build base is not OK')
# Like in klever.core.job.Job#__upload_original_sources.
os.makedirs(outfile + ' additional sources')
for root, dirs, files in os.walk(clade_extra.storage_dir):
for file in files:
file = os.path.join(root, file)
storage_file = klever.core.utils.make_relative_path(
[clade_extra.storage_dir], file)
# Do not treat those source files that were already processed and uploaded as original sources.
if os.path.commonpath([
os.path.join(os.path.sep, storage_file),
clade.storage_dir
]) == clade.storage_dir:
continue
new_file = klever.core.utils.make_relative_path(
self.search_dirs, storage_file, absolutize=True)
# These source files do not belong neither to original sources nor to models, e.g. there are compiler
# headers.
if os.path.isabs(new_file):
continue
# We treat all remaining source files which paths do not start with "specifications" as generated
# models. This is not correct for all cases, e.g. when users put some files within $KLEVER_DATA_DIR.
if not new_file.startswith('specifications'):
new_file = os.path.join('generated models', new_file)
new_file = os.path.join(outfile + ' additional sources',
new_file)
os.makedirs(os.path.dirname(new_file), exist_ok=True)
shutil.copy(file, new_file)
cross_refs = CrossRefs(self.conf, self.logger, clade_extra,
os.path.join(os.path.sep, storage_file),
new_file, self.search_dirs)
cross_refs.get_cross_refs()
self.__merge_additional_srcs(outfile + ' additional sources')
if not self.conf['keep intermediate files']:
shutil.rmtree(outfile + ' clade')
def main(sys_args=sys.argv[1:]):
args = parse_args(sys_args)
conf = prepare_conf(args)
build_exit_code = 0
# Create Clade interface object
try:
c = Clade(work_dir=conf["work_dir"], cmds_file=conf["cmds_file"], conf=conf, preset=args.preset)
except RuntimeError as e:
raise SystemExit(e)
if os.path.isfile(conf["cmds_file"]) and args.intercept and not args.append:
c.logger.info("File with intercepted commands already exists: {!r}".format(conf["cmds_file"]))
sys.exit(-1)
elif os.path.isfile(conf["cmds_file"]) and not args.intercept and not args.append:
c.logger.info("Skipping build and reusing {!r} file".format(conf["cmds_file"]))
else:
if not args.command:
c.logger.error("Build command is missing")
sys.exit(-1)
c.logger.info("Starting build")
build_time_start = time.time()
build_exit_code = c.intercept(
conf["build_command"],
use_wrappers=conf["use_wrappers"],
append=args.append,
intercept_open=args.intercept_open,
intercept_envs=args.intercept_envs
)
build_delta = datetime.timedelta(seconds=(time.time() - build_time_start))
build_delta_str = str(build_delta).split(".")[0]
# Clade can still proceed further if exit code != 0
c.logger.error("Build finished in {} with exit code {}".format(build_delta_str, build_exit_code))
if args.intercept and os.path.exists(conf["cmds_file"]):
c.logger.info("Path to the file with intercepted commands: {!r}".format(conf["cmds_file"]))
sys.exit(build_exit_code)
if not os.path.exists(conf["cmds_file"]):
c.logger.error("Something is wrong: file with intercepted commands is empty")
sys.exit(-1)
try:
extensions = args.extension if args.extension else c.conf["extensions"]
c.logger.info("Executing extensions")
ext_time_start = time.time()
c.parse_list(extensions, args.force_exts)
ext_delta = datetime.timedelta(seconds=(time.time() - ext_time_start))
ext_delta_str = str(ext_delta).split(".")[0]
c.logger.info("Extensions finished in {}".format(ext_delta_str))
if build_exit_code != 0:
c.logger.error("Reminder that build finished with exit code {}".format(build_exit_code))
except RuntimeError as e:
if e.args:
raise SystemExit(e)
else:
raise SystemExit(-1)
sys.exit(0)
def main(sys_args=sys.argv[1:]):
args = parse_args(sys_args)
conf = prepare_conf(args)
# Create Clade interface object
try:
c = Clade(work_dir=conf["work_dir"],
cmds_file=conf["cmds_file"],
conf=conf,
preset=args.preset)
except RuntimeError as e:
raise SystemExit(e)
if os.path.isfile(
conf["cmds_file"]) and args.intercept and not args.append:
c.logger.info(
"File with intercepted commands already exists: {!r}".format(
conf["cmds_file"]))
sys.exit(-1)
elif os.path.isfile(conf["cmds_file"]) and not args.append:
c.logger.info("Skipping build and reusing {!r} file".format(
conf["cmds_file"]))
else:
if not args.command:
c.logger.error("Build command is missing")
sys.exit(-1)
c.logger.info("Starting build")
r = c.intercept(conf["command"],
use_wrappers=conf["use_wrappers"],
append=args.append)
if r:
# Clade can still proceed further
c.logger.error("Build failed with error code {}".format(r))
else:
c.logger.info("Build completed successfully")
if args.intercept:
if os.path.exists(conf["cmds_file"]):
c.logger.info(
"Path to the file with intercepted commands: {!r}".format(
conf["cmds_file"]))
sys.exit(r)
else:
c.logger.error(
"Something is wrong: file with intercepted commands is empty")
sys.exit(-1)
try:
extensions = args.extension if args.extension else c.conf["extensions"]
c.logger.info("Executing extensions")
c.parse_list(extensions, args.force_exts)
c.logger.info("Executing extensions finished")
except RuntimeError as e:
if e.args:
raise SystemExit(e)
else:
raise SystemExit(-1)
sys.exit(0)
