def clade_api(tmpdir_factory, cmds_file): tmpdir = tmpdir_factory.mktemp("Clade") c = Clade(tmpdir, cmds_file) c.parse_list(["CrossRef", "Variables", "Macros", "Typedefs", "CDB"]) yield c
def build(self): self._fetch_work_src_tree() self._make_canonical_work_src_tree() self._clean() self._get_version() if self.version: self.logger.info('C program version is "{0}"'.format(self.version)) self._configure() if self.configuration: self.logger.info('C program configuration is "{0}"'.format( self.configuration)) self._build() if os.path.isdir(self.target_program_desc['build base']): shutil.rmtree(self.target_program_desc['build base']) if 'extra Clade options' in self.target_program_desc: clade_conf = dict(self._CLADE_CONF) clade_conf.update(self.target_program_desc['extra Clade options']) else: clade_conf = self._CLADE_CONF clade = Clade(work_dir=self.target_program_desc['build base'], cmds_file=os.path.join(self.work_src_tree, 'cmds.txt'), conf=clade_conf, preset=self._CLADE_PRESET) clade.parse_list( ["CrossRef", "Callgraph", "Variables", "Typedefs", "Macros"]) self.logger.info( 'Save project attributes, working source trees and target program description to build base' ) clade.add_meta_by_key('project attrs', [{ 'name': 'project', 'value': [{ 'name': 'name', 'value': type(self).__name__ }, { 'name': 'architecture', 'value': self.architecture }, { 'name': 'version', 'value': self.version }, { 'name': 'configuration', 'value': self.configuration }] }]) clade.add_meta_by_key('working source trees', self.work_src_trees) clade.add_meta_by_key('target program description', self.target_program_desc) self.logger.info('Remove temporary directories') for tmp_dir in self.tmp_dirs: shutil.rmtree(tmp_dir)
def clade_api(tmpdir_factory): tmpdir = tmpdir_factory.mktemp("Clade") c = Clade(tmpdir) c.intercept(command=test_project_make, use_wrappers=True, intercept_envs=True) c.parse_list(["CrossRef", "Variables", "Macros", "Typedefs", "CDB"]) yield c
def test_tracer(tmpdir, cmds_file): c = Clade(tmpdir, cmds_file, preset="klever_linux_kernel") c.parse_list(c.conf["extensions"]) print(c.work_dir) t = Tracer(c.work_dir) from_func = t.find_functions(["main"])[0] to_func = t.find_functions(["printf"])[0] trace = t.trace(from_func, to_func) assert len(trace) == 2
def __get_cross_refs(self, infile, opts, outfile, clade, cwd, aspectator_search_dir): # Get cross references and everything required for them. # Limit parallel workers in Clade by 4 since at this stage there may be several parallel task generators and we # prefer their parallelism over the Clade default one. clade_extra = Clade(work_dir=os.path.realpath(outfile + ' clade'), preset=self.conf['Clade']['preset'], conf={'cpu_count': 4}) # TODO: this can be incorporated into instrumentation above but it will need some Clade changes. # Emulate normal compilation (indeed just parsing thanks to "-fsyntax-only") to get additional # dependencies (model source files) and information on them. clade_extra.intercept([ klever.core.vtg.utils.get_cif_or_aspectator_exec( self.conf, 'aspectator'), '-I' + os.path.join( os.path.dirname(self.conf['specifications base']), 'include') ] + klever.core.vtg.utils.prepare_cif_opts(opts, clade, True) + [aspectator_search_dir, '-fsyntax-only', infile], cwd=cwd) clade_extra.parse_list(["CrossRef"]) if not clade_extra.work_dir_ok(): raise RuntimeError('Build base is not OK') # Like in klever.core.job.Job#__upload_original_sources. os.makedirs(outfile + ' additional sources') for root, dirs, files in os.walk(clade_extra.storage_dir): for file in files: file = os.path.join(root, file) storage_file = klever.core.utils.make_relative_path( [clade_extra.storage_dir], file) # Do not treat those source files that were already processed and uploaded as original sources. if os.path.commonpath([ os.path.join(os.path.sep, storage_file), clade.storage_dir ]) == clade.storage_dir: continue new_file = klever.core.utils.make_relative_path( self.search_dirs, storage_file, absolutize=True) # These source files do not belong neither to original sources nor to models, e.g. there are compiler # headers. if os.path.isabs(new_file): continue # We treat all remaining source files which paths do not start with "specifications" as generated # models. This is not correct for all cases, e.g. when users put some files within $KLEVER_DATA_DIR. if not new_file.startswith('specifications'): new_file = os.path.join('generated models', new_file) new_file = os.path.join(outfile + ' additional sources', new_file) os.makedirs(os.path.dirname(new_file), exist_ok=True) shutil.copy(file, new_file) cross_refs = CrossRefs(self.conf, self.logger, clade_extra, os.path.join(os.path.sep, storage_file), new_file, self.search_dirs) cross_refs.get_cross_refs() self.__merge_additional_srcs(outfile + ' additional sources') if not self.conf['keep intermediate files']: shutil.rmtree(outfile + ' clade')
def main(sys_args=sys.argv[1:]): args = parse_args(sys_args) conf = prepare_conf(args) build_exit_code = 0 # Create Clade interface object try: c = Clade(work_dir=conf["work_dir"], cmds_file=conf["cmds_file"], conf=conf, preset=args.preset) except RuntimeError as e: raise SystemExit(e) if os.path.isfile(conf["cmds_file"]) and args.intercept and not args.append: c.logger.info("File with intercepted commands already exists: {!r}".format(conf["cmds_file"])) sys.exit(-1) elif os.path.isfile(conf["cmds_file"]) and not args.intercept and not args.append: c.logger.info("Skipping build and reusing {!r} file".format(conf["cmds_file"])) else: if not args.command: c.logger.error("Build command is missing") sys.exit(-1) c.logger.info("Starting build") build_time_start = time.time() build_exit_code = c.intercept( conf["build_command"], use_wrappers=conf["use_wrappers"], append=args.append, intercept_open=args.intercept_open, intercept_envs=args.intercept_envs ) build_delta = datetime.timedelta(seconds=(time.time() - build_time_start)) build_delta_str = str(build_delta).split(".")[0] # Clade can still proceed further if exit code != 0 c.logger.error("Build finished in {} with exit code {}".format(build_delta_str, build_exit_code)) if args.intercept and os.path.exists(conf["cmds_file"]): c.logger.info("Path to the file with intercepted commands: {!r}".format(conf["cmds_file"])) sys.exit(build_exit_code) if not os.path.exists(conf["cmds_file"]): c.logger.error("Something is wrong: file with intercepted commands is empty") sys.exit(-1) try: extensions = args.extension if args.extension else c.conf["extensions"] c.logger.info("Executing extensions") ext_time_start = time.time() c.parse_list(extensions, args.force_exts) ext_delta = datetime.timedelta(seconds=(time.time() - ext_time_start)) ext_delta_str = str(ext_delta).split(".")[0] c.logger.info("Extensions finished in {}".format(ext_delta_str)) if build_exit_code != 0: c.logger.error("Reminder that build finished with exit code {}".format(build_exit_code)) except RuntimeError as e: if e.args: raise SystemExit(e) else: raise SystemExit(-1) sys.exit(0)
class Program: _CLADE_CONF = dict() _CLADE_PRESET = "base" def __init__(self, logger, target_program_desc): self.logger = logger self.target_program_desc = target_program_desc # Main working source tree where various build and auxiliary actions will be performed. self.work_src_tree = self.target_program_desc['source code'] # Program attributes. We expect that architecture is always specified in the target program description while # configuration and version can be either obtained during build somehow or remained unspecified. self.architecture = self.target_program_desc['architecture'] self.configuration = None self.version = self.target_program_desc.get('version') # Working source trees are directories to be trimmed from file names. self.work_src_trees = [] # Temporary directories that should be removed at the end of work. self.tmp_dirs = [] # Path to the Clade cmds.txt file with intercepted commands self.cmds_file = os.path.realpath(os.path.join(self.work_src_tree, 'cmds.txt')) # Clade API object clade_conf = dict(self._CLADE_CONF) clade_conf.update(self.target_program_desc.get('extra Clade options', dict())) self.clade = Clade(work_dir=self.target_program_desc['build base'], cmds_file=self.cmds_file, conf=clade_conf, preset=self._CLADE_PRESET) def _prepare_work_src_tree(self): o = urllib.parse.urlparse(self.work_src_tree) if o[0] in ('http', 'https', 'ftp'): raise NotImplementedError('Source code is provided in unsupported form of a remote archive') elif o[0] == 'git': raise NotImplementedError('Source code is provided in unsupported form of a remote Git repository') elif o[0]: raise ValueError('Source code is provided in unsupported form "{0}"'.format(o[0])) if os.path.isfile(self.work_src_tree): raise NotImplementedError('Source code is provided in unsupported form of an archive') # Local git repository if os.path.isdir(os.path.join(self.work_src_tree, '.git')): self.logger.debug("Source code is provided in form of a Git repository") self.__prepare_git_work_src_tree() self.work_src_trees.append(os.path.realpath(self.work_src_tree)) def __prepare_git_work_src_tree(self): if 'git repository version' not in self.target_program_desc: return checkout = self.target_program_desc['git repository version'] self.logger.info(f'Checkout Git repository "{checkout}"') # Repository lock file may remain from some previous crashed git command git_index_lock = os.path.join(self.work_src_tree, '.git', 'index.lock') if os.path.isfile(git_index_lock): os.remove(git_index_lock) # In case of dirty Git working directory checkout may fail so clean up it first. execute_cmd(self.logger, 'git', 'clean', '-f', '-d', cwd=self.work_src_tree) execute_cmd(self.logger, 'git', 'reset', '--hard', cwd=self.work_src_tree) execute_cmd(self.logger, 'git', 'checkout', '-f', checkout, cwd=self.work_src_tree) try: # Use Git describe to properly identify program version stdout = execute_cmd(self.logger, 'git', 'describe', cwd=self.work_src_tree, get_output=True) self.version = stdout[0] except subprocess.CalledProcessError: # Use Git repository version from target program description if Git describe failed self.version = checkout def _run_clade(self): if os.path.isdir(self.target_program_desc['build base']): shutil.rmtree(self.target_program_desc['build base']) self.clade.parse_list(self.clade.conf['extensions']) self.logger.info('Save project attributes, working source trees and target program description to build base') attrs = [ { 'name': 'name', 'value': type(self).__name__ }, { 'name': 'architecture', 'value': self.architecture }, { 'name': 'version', 'value': self.version } ] if self.configuration: attrs.append({ 'name': 'configuration', 'value': self.configuration }) self.clade.add_meta_by_key('project attrs', [{ 'name': 'project', 'value': attrs }]) self.clade.add_meta_by_key('working source trees', self.work_src_trees) self.clade.add_meta_by_key('target program description', self.target_program_desc) @staticmethod def build_wrapper(build): '''Wrapper for build() method''' def wrapper(self, *args, **kwargs): try: return build(self, *args, **kwargs) finally: for tmp_dir in self.tmp_dirs: self.logger.info(f'Remove temporary directory "{tmp_dir}"') shutil.rmtree(tmp_dir) if os.path.exists(self.cmds_file): os.remove(self.cmds_file) return wrapper def build(self): ...
def main(sys_args=sys.argv[1:]): args = parse_args(sys_args) conf = prepare_conf(args) # Create Clade interface object try: c = Clade(work_dir=conf["work_dir"], cmds_file=conf["cmds_file"], conf=conf, preset=args.preset) except RuntimeError as e: raise SystemExit(e) if os.path.isfile( conf["cmds_file"]) and args.intercept and not args.append: c.logger.info( "File with intercepted commands already exists: {!r}".format( conf["cmds_file"])) sys.exit(-1) elif os.path.isfile(conf["cmds_file"]) and not args.append: c.logger.info("Skipping build and reusing {!r} file".format( conf["cmds_file"])) else: if not args.command: c.logger.error("Build command is missing") sys.exit(-1) c.logger.info("Starting build") r = c.intercept(conf["command"], use_wrappers=conf["use_wrappers"], append=args.append) if r: # Clade can still proceed further c.logger.error("Build failed with error code {}".format(r)) else: c.logger.info("Build completed successfully") if args.intercept: if os.path.exists(conf["cmds_file"]): c.logger.info( "Path to the file with intercepted commands: {!r}".format( conf["cmds_file"])) sys.exit(r) else: c.logger.error( "Something is wrong: file with intercepted commands is empty") sys.exit(-1) try: extensions = args.extension if args.extension else c.conf["extensions"] c.logger.info("Executing extensions") c.parse_list(extensions, args.force_exts) c.logger.info("Executing extensions finished") except RuntimeError as e: if e.args: raise SystemExit(e) else: raise SystemExit(-1) sys.exit(0)