def test_files_to_add(tmpdir, cmds_file): c = Clade(tmpdir, cmds_file, conf={"Storage.files_to_add": [__file__]}) c.parse("Storage") storage_path = c.get_storage_path(__file__) assert storage_path assert os.path.exists(storage_path)
def test_storage(tmpdir): c = Clade(tmpdir) c.add_file_to_storage(__file__) c.add_file_to_storage("do_not_exist.c") assert os.path.exists(os.path.join(c.storage_dir, __file__)) assert c.get_storage_path(__file__) # Test possible race condition with unittest.mock.patch("shutil.copyfile") as copyfile_mock: copyfile_mock.side_effect = shutil.SameFileError c.add_file_to_storage(test_file)
def test_storage(tmpdir): c = Clade(tmpdir) returned_storage_path = c.add_file_to_storage(__file__) c.add_file_to_storage("do_not_exist.c") storage_path = c.get_storage_path(__file__) assert storage_path assert os.path.exists(storage_path) assert storage_path.startswith(c.storage_dir) assert returned_storage_path == storage_path # Test possible race condition with unittest.mock.patch("shutil.copyfile") as copyfile_mock: copyfile_mock.side_effect = shutil.SameFileError c.add_file_to_storage(test_file)
def weave(self): self.abstract_task_desc.setdefault('extra C files', dict()) clade = Clade(self.conf['build base']) if not clade.work_dir_ok(): raise RuntimeError('Build base is not OK') meta = clade.get_meta() # This is required to get compiler (Aspectator) specific stdarg.h since kernel C files are compiled # with "-nostdinc" option and system stdarg.h couldn't be used. aspectator_search_dir = '-isystem' + klever.core.utils.execute( self.logger, (klever.core.vtg.utils.get_cif_or_aspectator_exec( self.conf, 'aspectator'), '-print-file-name=include'), collect_all_stdout=True)[0] env = dict(os.environ) # Print stubs instead of inline Assembler since verifiers do not interpret it and even can fail. env['LDV_INLINE_ASM_STUB'] = '' for grp in self.abstract_task_desc['grps']: self.logger.info('Weave in C files of group "{0}"'.format( grp['id'])) for extra_cc in grp['Extra CCs']: # Each CC is either pair (compiler command identifier, compiler command type) or JSON file name # with compiler command description. if isinstance(extra_cc['CC'], list): cc = clade.get_cmd(*extra_cc['CC'], with_opts=True) if "in file" in extra_cc: # This is for CC commands with several input files infile = extra_cc["in file"] else: infile = cc["in"][0] infile = clade.get_storage_path(infile) if meta['conf'].get('Compiler.preprocess_cmds', False): infile = infile('.c')[0] + '.i' else: with open(os.path.join(self.conf['main working directory'], extra_cc['CC']), encoding='utf8') as fp: cc = json.load(fp) infile = cc["in"][0] # Distinguish source files having the same names. outfile_unique = '{0}.c'.format( klever.core.utils.unique_file_name( os.path.splitext(os.path.basename(infile))[0], '.c')) # This is used for storing/getting to/from cache where uniqueness is guaranteed by other means. outfile = '{0}.c'.format( os.path.splitext(os.path.basename(infile))[0]) self.logger.info('Weave in C file "{0}"'.format(infile)) # Produce aspect to be weaved in. if 'plugin aspects' in extra_cc: self.logger.info( 'Concatenate all aspects of all plugins together') # Resulting aspect. aspect = 'aspect' # Get all aspects. Place RSG aspects at beginning since they can instrument entities added by # aspects of other plugins while corresponding function declarations still need be at beginning # of file. aspects = [] for plugin_aspects in extra_cc['plugin aspects']: if plugin_aspects['plugin'] == 'RSG': aspects[0:0] = plugin_aspects['aspects'] else: aspects.extend(plugin_aspects['aspects']) # Concatenate aspects. with open(aspect, 'w', encoding='utf8') as fout, fileinput.input( [ os.path.join( self.conf['main working directory'], aspect) for aspect in aspects ], openhook=fileinput.hook_encoded( 'utf8')) as fin: for line in fin: fout.write(line) else: # Instrumentation is not required when there is no aspects. But we will still pass source files # through C-backend to make resulting code to look similarly and thus to avoid different issues # at merging source files and models together. aspect = None if aspect: self.logger.info( 'Aspect to be weaved in is "{0}"'.format(aspect)) else: self.logger.info( 'C file will be passed through C Back-end only') cwd = clade.get_storage_path(cc['cwd']) is_model = (grp['id'] == 'models') # Original sources should be woven in and we do not need to get cross references for them since this # was already done before. if not is_model: self.__weave(infile, cc['opts'], aspect, outfile_unique, clade, env, cwd, aspectator_search_dir, is_model) # For generated models we need to weave them in (actually, just pass through C Back-end) and to get # cross references always since most likely they all are different. elif 'generated' in extra_cc: self.__weave(infile, cc['opts'], aspect, outfile_unique, clade, env, cwd, aspectator_search_dir, is_model) if self.conf[ 'code coverage details'] != 'Original C source files': self.__get_cross_refs(infile, cc['opts'], outfile_unique, clade, cwd, aspectator_search_dir) # For non-generated models use results cache in addition. else: cache_dir = os.path.join( self.conf['cache directory'], klever.core.utils.get_file_name_checksum(infile)) with klever.core.utils.LockedOpen(cache_dir + '.tmp', 'w'): if os.path.exists(cache_dir): self.logger.info('Get woven in C file from cache') self.abstract_task_desc['extra C files'].append({ 'C file': os.path.relpath( os.path.join(cache_dir, os.path.basename(outfile)), self.conf['main working directory']) }) if self.conf[ 'code coverage details'] != 'Original C source files': self.logger.info( 'Get cross references from cache') self.__merge_additional_srcs( os.path.join(cache_dir, 'additional sources')) else: os.makedirs(cache_dir) self.__weave(infile, cc['opts'], aspect, outfile_unique, clade, env, cwd, aspectator_search_dir, is_model) self.logger.info('Store woven in C file to cache') shutil.copy(outfile_unique, os.path.join(cache_dir, outfile)) if self.conf[ 'code coverage details'] != 'Original C source files': self.__get_cross_refs(infile, cc['opts'], outfile_unique, clade, cwd, aspectator_search_dir) self.logger.info( 'Store cross references to cache') shutil.copytree( outfile_unique + ' additional sources', os.path.join(cache_dir, 'additional sources')) # For auxiliary files there is no cross references since it is rather hard to get them from Aspectator. But # there still highlighting. if self.conf['code coverage details'] == 'All source files': for aux_file in glob.glob('*.aux'): new_file = os.path.join( 'additional sources', 'generated models', os.path.relpath(aux_file, self.conf['main working directory'])) os.makedirs(os.path.dirname(new_file), exist_ok=True) shutil.copy(aux_file, new_file) cross_refs = CrossRefs(self.conf, self.logger, clade, aux_file, new_file, self.search_dirs) cross_refs.get_cross_refs() self.abstract_task_desc['additional sources'] = os.path.relpath('additional sources', self.conf['main working directory']) \ if os.path.isdir('additional sources') else None # Copy additional sources for total code coverage. if self.conf['code coverage details'] != 'Original C source files': with klever.core.utils.Cd('additional sources'): for root, dirs, files in os.walk(os.path.curdir): for file in files: # These files are handled below in addition to corresponding source files. if file.endswith('.json'): continue if self.conf['code coverage details'] == 'C source files including models' \ and not file.endswith('.c'): continue file = os.path.join(root, file) new_file = os.path.join( self.conf['additional sources directory'], file) os.makedirs(os.path.dirname(new_file), exist_ok=True) with klever.core.utils.LockedOpen( new_file + '.tmp', 'w'): if os.path.isfile(new_file): os.remove(new_file + '.tmp') continue shutil.copy(file, new_file) shutil.copy(file + '.idx.json', new_file + '.idx.json') os.remove(new_file + '.tmp') # These sections won't be refereed any more. del (self.abstract_task_desc['grps']) del (self.abstract_task_desc['deps'])
def request_arg_signs(self): self.logger.info('Request argument signatures') clade = Clade(work_dir=self.conf['build base']) for request_aspect in self.conf['request aspects']: request_aspect = core.vtg.utils.find_file_or_dir( self.logger, self.conf['main working directory'], request_aspect) self.logger.debug('Request aspect is "{0}"'.format(request_aspect)) # This is required to get compiler (Aspectator) specific stdarg.h since kernel C files are compiled with # "-nostdinc" option and system stdarg.h couldn't be used. aspectator_search_dir = '-isystem' + core.utils.execute( self.logger, ('aspectator', '-print-file-name=include'), collect_all_stdout=True)[0] for grp in self.abstract_task_desc['grps']: self.logger.info( 'Request argument signatures for C files of group "{0}"'. format(grp['id'])) for extra_cc in grp['Extra CCs']: self.logger.info( 'Request argument signatures for C file "{0}"'.format( extra_cc['in file'])) cc = clade.get_cmd(extra_cc['CC'], with_opts=True) env = dict(os.environ) env['LDV_ARG_SIGNS_FILE'] = os.path.realpath( os.path.splitext( os.path.splitext( os.path.basename(request_aspect))[0])[0]) # Add plugin aspects produced thus far (by EMG) since they can include additional headers for which # additional argument signatures should be extracted. Like in Weaver. if 'plugin aspects' in extra_cc: self.logger.info( 'Concatenate all aspects of all plugins together') # Resulting request aspect. aspect = '{0}.aspect'.format( core.utils.unique_file_name( os.path.splitext(os.path.basename( cc['out'][0]))[0], '.aspect')) # Get all aspects. Place original request aspect at beginning since it can instrument entities # added by aspects of other plugins while corresponding function declarations still need be at # beginning of file. aspects = [ os.path.relpath( request_aspect, self.conf['main working directory']) ] for plugin_aspects in extra_cc['plugin aspects']: aspects.extend(plugin_aspects['aspects']) # Concatenate aspects. with open( aspect, 'w', encoding='utf8' ) as fout, fileinput.input( [ os.path.join( self.conf['main working directory'], aspect) for aspect in aspects ], openhook=fileinput.hook_encoded( 'utf8')) as fin: for line in fin: fout.write(line) else: aspect = request_aspect core.utils.execute( self.logger, tuple([ 'cif', '--in', clade.get_storage_path(cc['in'][0]), '--aspect', os.path.realpath(aspect), '--stage', 'instrumentation', '--out', os.path.realpath('{0}.c'.format( core.utils.unique_file_name( os.path.splitext( os.path.basename(cc['out'][0]))[0], '.c.aux'))), '--debug', 'DEBUG' ] + ( ['--keep'] if self. conf['keep intermediate files'] else [] ) + ['--'] + core.vtg.utils.prepare_cif_opts( self.conf, cc['opts'], clade.storage_dir ) + [ # Besides header files specific for requirements will be # searched for. '-I' + os.path.realpath( os.path.dirname(self.conf['requirements DB'])), aspectator_search_dir ]), env, cwd=clade.get_storage_path(cc['cwd']), timeout=0.01, filter_func=core.vtg.utils.CIFErrorFilter())
def add_models(self, generated_models): self.logger.info( 'Add models to abstract verification task description') models = {} if 'environment model' in self.abstract_task_desc: rel_path = os.path.relpath( os.path.join(self.conf['main working directory'], self.abstract_task_desc['environment model']), os.path.curdir) models[rel_path] = {} # Get common and requirement specific models. if 'common models' in self.conf and 'models' in self.conf: for common_model_c_file in self.conf['common models']: if common_model_c_file in self.conf['models']: raise KeyError( 'C file "{0}" is specified in both common and requirement specific models' ) if 'models' in self.conf: for model_c_file in self.conf['models']: # Specify additional settings for generated models that have not any settings. if model_c_file.startswith('$'): is_generated_model_c_file_found = False for generated_model_c_file in generated_models: if generated_model_c_file.endswith(model_c_file[1:]): models[generated_model_c_file] = self.conf[ 'models'][model_c_file] is_generated_model_c_file_found = True if not is_generated_model_c_file_found: raise KeyError( 'Model C file "{0}" was not generated'.format( model_c_file[1:])) # Like common models processed below. for model_c_file in self.conf['models']: if not model_c_file.startswith('$'): model_c_file_realpath = core.vtg.utils.find_file_or_dir( self.logger, self.conf['main working directory'], model_c_file) self.logger.debug('Get model with C file "{0}"'.format( model_c_file_realpath)) models[model_c_file_realpath] = self.conf['models'][ model_c_file] if 'common models' in self.conf: for common_model_c_file in self.conf['common models']: common_model_c_file_realpath = core.vtg.utils.find_file_or_dir( self.logger, self.conf['main working directory'], common_model_c_file) self.logger.debug('Get common model with C file "{0}"'.format( common_model_c_file_realpath)) models[common_model_c_file_realpath] = self.conf[ 'common models'][common_model_c_file] self.logger.debug('Resulting models are: {0}'.format(models)) if not models: self.logger.warning('No models are specified') return # CC extra full description files will be put to this directory as well as corresponding intermediate and final # output files. os.makedirs('models'.encode('utf8')) self.logger.info( 'Add aspects to abstract verification task description') aspects = [] for model_c_file in models: aspect = '{}.aspect'.format(os.path.splitext(model_c_file)[0]) if not os.path.isfile(aspect): continue self.logger.debug('Get aspect "{0}"'.format(aspect)) aspects.append(aspect) # Sort aspects to apply them in the deterministic order. aspects.sort() for grp in self.abstract_task_desc['grps']: self.logger.info('Add aspects to C files of group "{0}"'.format( grp['id'])) for extra_cc in grp['Extra CCs']: if 'plugin aspects' not in extra_cc: extra_cc['plugin aspects'] = [] extra_cc['plugin aspects'].append({ 'plugin': self.name, 'aspects': [ os.path.relpath(aspect, self.conf['main working directory']) for aspect in aspects ] }) for model_c_file in models: model = models[model_c_file] if 'bug kinds' in model: self.logger.info( 'Preprocess bug kinds for model with C file "{0}"'.format( model_c_file)) # Collect all bug kinds specified in model to check that valid bug kinds are specified in requirement # model description. bug_kinds = set() lines = [] with open(model_c_file, encoding='utf8') as fp: for line in fp: # Bug kinds are specified in form of strings like in requirements DB as first actual # parameters of ldv_assert(). match = re.search(r'ldv_assert\("([^"]+)"', line) if match: bug_kind, = match.groups() bug_kinds.add(bug_kind) # Include bug kinds in names of ldv_assert(). lines.append( re.sub( r'ldv_assert\("([^"]+)", ?', r'ldv_assert_{0}('.format( re.sub(r'\W', '_', bug_kind)), line)) else: lines.append(line) for bug_kind in model['bug kinds']: if bug_kind not in bug_kinds: raise KeyError( 'Invalid bug kind "{0}" is specified in requirement model description' .format(bug_kind)) preprocessed_model_c_file = '{0}.bk.c'.format( core.utils.unique_file_name( os.path.join( 'models', os.path.splitext( os.path.basename(model_c_file))[0]), '.bk.c')) with open(preprocessed_model_c_file, 'w', encoding='utf8') as fp: # Create ldv_assert*() function declarations to avoid compilation warnings. These functions will # be defined later somehow by VTG. for bug_kind in sorted(bug_kinds): fp.write('extern void ldv_assert_{0}(int);\n'.format( re.sub(r'\W', '_', bug_kind))) # Specify original location to avoid references to *.bk.c files in error traces. fp.write('# 1 "{0}"\n'.format( os.path.abspath(model_c_file))) for line in lines: fp.write(line) model[ 'bug kinds preprocessed C file'] = preprocessed_model_c_file self.logger.debug( 'Preprocessed bug kinds for model with C file "{0}" was placed to "{1}"' .format(model_c_file, preprocessed_model_c_file)) else: model['bug kinds preprocessed C file'] = model_c_file # Generate CC full description file per each model and add it to abstract task description. # First of all obtain CC options to be used to compile models. clade = Clade(self.conf['build base']) # Relative path to source file which CC options to be used is specified in configuration. Clade needs absolute # path. The former is relative to one of source paths. for path in self.conf['source paths']: opts_file = os.path.join(path, self.conf['opts file']) try: empty_cc = list(clade.get_compilation_cmds_by_file(opts_file)) except KeyError: pass if not empty_cc: raise RuntimeError("There is not of cc commands for {!r}".format( self.conf['project']['opts file'])) elif len(empty_cc) > 1: self.logger.warning( "There are more than one cc command for {!r}".format( self.conf['project']['opts file'])) empty_cc = empty_cc.pop() empty_cc['opts'] = clade.get_cmd_opts(empty_cc['id']) model_grp = {'id': 'models', 'Extra CCs': []} for model_c_file in sorted(models): model = models[model_c_file] extra_cc = {} if 'bug kinds preprocessed C file' in model: file, ext = os.path.splitext( os.path.join( 'models', os.path.basename( model['bug kinds preprocessed C file']))) base_name = core.utils.unique_file_name( file, '{0}.json'.format(ext)) full_desc_file = '{0}{1}.json'.format(base_name, ext) out_file = '{0}.c'.format(base_name) self.logger.debug( 'Dump CC full description to file "{0}"'.format( full_desc_file)) with open(full_desc_file, 'w', encoding='utf8') as fp: json.dump( { 'cwd': empty_cc['cwd'], 'in': [ os.path.relpath( model['bug kinds preprocessed C file'], os.path.realpath( clade.get_storage_path( empty_cc['cwd']))) ], 'out': [os.path.realpath(out_file)], 'opts': empty_cc['opts'] + [ '-DLDV_SETS_MODEL_' + (model['sets model'] if 'sets model' in model else self.conf['common sets model']).upper() ] }, fp, ensure_ascii=False, sort_keys=True, indent=4) extra_cc['CC'] = os.path.relpath( full_desc_file, self.conf['main working directory']) if 'bug kinds' in model: extra_cc['bug kinds'] = model['bug kinds'] if extra_cc: model_grp['Extra CCs'].append(extra_cc) self.abstract_task_desc['grps'].append(model_grp) for dep in self.abstract_task_desc['deps'].values(): dep.append(model_grp['id'])
def test_storage(tmpdir, cmds_file): c = Clade(tmpdir, cmds_file) c.add_file_to_storage(__file__) assert os.path.exists(os.path.join(c.storage_dir, __file__)) assert c.get_storage_path(__file__)
class Job(klever.core.components.Component): CORE_COMPONENTS = ['PFG', 'VTG', 'VRP'] def __init__(self, conf, logger, parent_id, callbacks, mqs, vals, id=None, work_dir=None, attrs=None, separate_from_parent=True, include_child_resources=False, components_common_conf=None): super(Job, self).__init__(conf, logger, parent_id, callbacks, mqs, vals, id, work_dir, attrs, separate_from_parent, include_child_resources) self.common_components_conf = components_common_conf if work_dir: self.common_components_conf[ 'additional sources directory'] = os.path.join( os.path.realpath(work_dir), 'additional sources') self.clade = None self.components = [] self.component_processes = [] def decide_job_or_sub_job(self): self.logger.info('Decide job/sub-job "{0}"'.format(self.id)) # This is required to associate verification results with particular sub-jobs. # Skip leading "/" since this identifier is used in os.path.join() that returns absolute path otherwise. self.common_components_conf['sub-job identifier'] = self.id[1:] # Check and set build base here since many Core components need it. self.__set_build_base() self.clade = Clade(self.common_components_conf['build base']) if not self.clade.work_dir_ok(): raise RuntimeError('Build base is not OK') self.__retrieve_working_src_trees() self.__get_original_sources_basic_info() self.__upload_original_sources() # Create directory where files will be cached and remember absolute path to it for components. os.mkdir('cache') self.common_components_conf['cache directory'] = os.path.realpath( 'cache') if self.common_components_conf['keep intermediate files']: self.logger.debug( 'Create components configuration file "conf.json"') with open('conf.json', 'w', encoding='utf8') as fp: json.dump(self.common_components_conf, fp, ensure_ascii=False, sort_keys=True, indent=4) self.__get_job_or_sub_job_components() self.callbacks = klever.core.components.get_component_callbacks( self.logger, [type(self)] + self.components) self.launch_sub_job_components() self.clean_dir = True self.logger.info("All components finished") if self.conf.get('collect total code coverage', None): self.logger.debug('Waiting for a collecting coverage') while not self.vals['coverage_finished'].get( self.common_components_conf['sub-job identifier'], True): time.sleep(1) self.logger.debug("Coverage collected") main = decide_job_or_sub_job def __set_build_base(self): if 'build base' not in self.common_components_conf: raise KeyError( "Provide 'build base' configuration option to start verification" ) common_advice = 'please, fix "job.json" (attribute "build base")' common_advice += ' or/and deployment configuration file (attribute "Klever Build Bases")' # Try to find specified build base either in normal way or additionally in directory "build bases" that is # convenient to use when working with many build bases. try: build_base = klever.core.utils.find_file_or_dir( self.logger, os.path.curdir, self.common_components_conf['build base']) except FileNotFoundError: try: build_base = klever.core.utils.find_file_or_dir( self.logger, os.path.curdir, os.path.join('build bases', self.common_components_conf['build base'])) except FileNotFoundError: raise FileNotFoundError( 'Specified build base "{0}" does not exist, {1}'.format( self.common_components_conf['build base'], common_advice)) from None # Extract build base from archive. There should not be any intermediate directories in archives. if os.path.isfile(build_base) and (tarfile.is_tarfile(build_base) or zipfile.is_zipfile(build_base)): if tarfile.is_tarfile(build_base): self.logger.debug( 'Build base "{0}" is provided in form of TAR archive'. format(build_base)) with tarfile.open(build_base) as TarFile: TarFile.extractall('build base') else: self.logger.debug( 'Build base "{0}" is provided in form of ZIP archive'. format(build_base)) with zipfile.ZipFile(build_base) as zfp: zfp.extractall('build base') # Directory contains extracted build base. extracted_from = ' extracted from "{0}"'.format( os.path.realpath(build_base)) build_base = 'build base' else: extracted_from = '' # We need to specify absolute path to build base since it will be used in different Klever components. Besides, # this simplifies troubleshooting. build_base = os.path.realpath(build_base) # TODO: fix after https://github.com/17451k/clade/issues/108. if not os.path.isdir(build_base): raise FileExistsError( 'Build base "{0}" is not a directory, {1}'.format( build_base, extracted_from, common_advice)) if not os.path.isfile(os.path.join(build_base, 'meta.json')): raise FileExistsError( 'Directory "{0}"{1} is not a build base since it does not contain file "meta.json", {2}' .format(build_base, extracted_from, common_advice)) self.common_components_conf['build base'] = build_base self.logger.debug('Klever components will use build base "{0}"'.format( self.common_components_conf['build base'])) # Klever will try to cut off either working source trees (if specified) or at least build directory (otherwise) # from referred file names. Sometimes this is rather optional like for source files referred by error traces, but, # say, for program fragment identifiers this is strictly necessary, e.g. because of otherwise expert assessment will # not work as expected. def __retrieve_working_src_trees(self): clade_meta = self.clade.get_meta() self.common_components_conf['working source trees'] = clade_meta['working source trees'] \ if 'working source trees' in clade_meta else [clade_meta['build_dir']] def __refer_original_sources(self, src_id): klever.core.utils.report(self.logger, 'patch', { 'identifier': self.id, 'original_sources': src_id }, self.mqs['report files'], self.vals['report id'], self.conf['main working directory']) def __process_source_files(self): for file_name in self.clade.src_info: self.mqs['file names'].put(file_name) for i in range(self.workers_num): self.mqs['file names'].put(None) def __process_source_file(self): while True: file_name = self.mqs['file names'].get() if not file_name: return src_file_name = klever.core.utils.make_relative_path( self.common_components_conf['working source trees'], file_name) if src_file_name != file_name: src_file_name = os.path.join('source files', src_file_name) new_file_name = os.path.join('original sources', src_file_name.lstrip(os.path.sep)) os.makedirs(os.path.dirname(new_file_name), exist_ok=True) shutil.copy(self.clade.get_storage_path(file_name), new_file_name) cross_refs = CrossRefs( self.common_components_conf, self.logger, self.clade, file_name, new_file_name, self.common_components_conf['working source trees'], 'source files') cross_refs.get_cross_refs() def __get_original_sources_basic_info(self): self.logger.info( 'Get information on original sources for following visualization of uncovered source files' ) # For each source file we need to know the total number of lines and places where functions are defined. src_files_info = dict() for file_name, file_size in self.clade.src_info.items(): src_file_name = klever.core.utils.make_relative_path( self.common_components_conf['working source trees'], file_name) # Skip non-source files. if src_file_name == file_name: continue src_file_name = os.path.join('source files', src_file_name) src_files_info[src_file_name] = list() # Store source file size. src_files_info[src_file_name].append(file_size['loc']) # Store source file function definition lines. func_def_lines = list() funcs = self.clade.get_functions_by_file([file_name], False) if funcs: for func_name, func_info in list(funcs.values())[0].items(): func_def_lines.append(int(func_info['line'])) src_files_info[src_file_name].append(sorted(func_def_lines)) # Dump obtain information (huge data!) to load it when reporting total code coverage if everything will be okay. with open('original sources basic information.json', 'w') as fp: klever.core.utils.json_dump(src_files_info, fp, self.conf['keep intermediate files']) def __upload_original_sources(self): # Use Clade UUID to distinguish various original sources. It is pretty well since this UUID is uuid.uuid4(). src_id = self.clade.get_uuid() session = klever.core.session.Session(self.logger, self.conf['Klever Bridge'], self.conf['identifier']) if session.check_original_sources(src_id): self.logger.info('Original sources were uploaded already') self.__refer_original_sources(src_id) return self.logger.info( 'Cut off working source trees or build directory from original source file names and convert index data' ) os.makedirs('original sources') self.mqs['file names'] = multiprocessing.Queue() self.workers_num = klever.core.utils.get_parallel_threads_num( self.logger, self.conf) subcomponents = [('PSFS', self.__process_source_files)] for i in range(self.workers_num): subcomponents.append(('RSF', self.__process_source_file)) self.launch_subcomponents(False, *subcomponents) self.mqs['file names'].close() self.logger.info('Compress original sources') klever.core.utils.ArchiveFiles(['original sources' ]).make_archive('original sources.zip') self.logger.info('Upload original sources') try: session.upload_original_sources(src_id, 'original sources.zip') # Do not fail if there are already original sources. There may be complex data races because of checking and # uploading original sources archive are not atomic. except klever.core.session.BridgeError: if "original sources with this identifier already exists." not in list( session.error.values())[0]: raise self.__refer_original_sources(src_id) if not self.conf['keep intermediate files']: shutil.rmtree('original sources') os.remove('original sources.zip') def __get_job_or_sub_job_components(self): self.logger.info('Get components for sub-job "{0}"'.format(self.id)) self.components = [ getattr( importlib.import_module('.{0}'.format(component.lower()), 'klever.core'), component) for component in self.CORE_COMPONENTS ] self.logger.debug('Components to be launched: "{0}"'.format(', '.join( [component.__name__ for component in self.components]))) def launch_sub_job_components(self): """Has callbacks""" self.logger.info('Launch components for sub-job "{0}"'.format(self.id)) for component in self.components: p = component(self.common_components_conf, self.logger, self.id, self.callbacks, self.mqs, self.vals, separate_from_parent=True) self.component_processes.append(p) klever.core.components.launch_workers(self.logger, self.component_processes)
def add_models(self, generated_models): self.logger.info('Add models to abstract verification task description') models = [] if 'environment model' in self.abstract_task_desc: models.append({ 'model': os.path.relpath(os.path.join(self.conf['main working directory'], self.abstract_task_desc['environment model']), os.path.curdir), 'options': {}, 'generated': True }) if 'extra C files' in self.abstract_task_desc: self.abstract_task_desc['extra C files'] = [] for c_file in (extra_c_file["C file"] for extra_c_file in self.abstract_task_desc['extra C files'] if "C file" in extra_c_file): models.append(os.path.relpath(os.path.join(self.conf['main working directory'], c_file), os.path.curdir)) def get_model_c_file(model): # Model may be a C file or a dictionary with model file and option attributes. if isinstance(model, dict): return model['model'] else: return model # Get common and requirement specific models. if 'common models' in self.conf and 'models' in self.conf: for common_model_c_file in self.conf['common models']: for model in self.conf['models']: if common_model_c_file == get_model_c_file(model): raise KeyError('C file "{0}" is specified in both common and requirement specific models' .format(common_model_c_file)) if 'models' in self.conf: # Find out actual C files. for model in self.conf['models']: model_c_file = get_model_c_file(model) # Handle generated models which C files start with "$". if model_c_file.startswith('$'): is_generated_model_c_file_found = False for generated_model_c_file in generated_models: if generated_model_c_file.endswith(model_c_file[1:]): if isinstance(model, dict): # Specify model options for generated models that can not have model options themselves. models.append({ 'model': generated_model_c_file, 'options': model['options'], 'generated': True }) else: models.append({ 'model': generated_model_c_file, 'options': {}, 'generated': True }) is_generated_model_c_file_found = True if not is_generated_model_c_file_found: raise KeyError('Model C file "{0}" was not generated'.format(model_c_file[1:])) # Handle non-generated models. else: model_c_file_realpath = klever.core.vtg.utils.find_file_or_dir( self.logger, self.conf['main working directory'], model_c_file) self.logger.debug('Get model with C file "{0}"'.format(model_c_file_realpath)) if isinstance(model, dict): models.append({ 'model': model_c_file_realpath, 'options': model['options'] }) else: models.append(model_c_file_realpath) # Like for models above except for common models are always C files without any model settings. if 'common models' in self.conf: for common_model_c_file in self.conf['common models']: common_model_c_file_realpath = klever.core.vtg.utils.find_file_or_dir( self.logger, self.conf['main working directory'], common_model_c_file) self.logger.debug('Get common model with C file "{0}"'.format(common_model_c_file_realpath)) models.append(common_model_c_file_realpath) self.logger.debug('Resulting models are: {0}'.format(models)) if not models: self.logger.warning('No models are specified') return # CC extra full description files will be put to this directory as well as corresponding intermediate and final # output files. os.makedirs('models'.encode('utf8')) self.logger.info('Add aspects to abstract verification task description') aspects = [] for model in models: aspect = '{}.aspect'.format(os.path.splitext(get_model_c_file(model))[0]) if not os.path.isfile(aspect): continue self.logger.debug('Get aspect "{0}"'.format(aspect)) aspects.append(aspect) # Sort aspects to apply them in the deterministic order. aspects.sort() for grp in self.abstract_task_desc['grps']: self.logger.info('Add aspects to C files of group "{0}"'.format(grp['id'])) for extra_cc in grp['Extra CCs']: if 'plugin aspects' not in extra_cc: extra_cc['plugin aspects'] = [] extra_cc['plugin aspects'].append({ 'plugin': self.name, 'aspects': [os.path.relpath(aspect, self.conf['main working directory']) for aspect in aspects] }) # Generate CC full description file per each model and add it to abstract task description. # First of all obtain CC options to be used to compile models. clade = Clade(self.conf['build base']) if not clade.work_dir_ok(): raise RuntimeError('Build base is not OK') meta = clade.get_meta() if not meta['conf'].get('Compiler.preprocess_cmds', False): # Model compiler input file represents input file which compiler options and CWD should be used for # compiling models. This input file is relative to one of source paths. compiler_cmds = None for path in self.conf['working source trees']: try: compiler_cmds = list(clade.get_compilation_cmds_by_file( os.path.join(path, self.conf['model compiler input file']))) except KeyError: pass if not compiler_cmds: raise RuntimeError("There is no compiler commands for {!r}" .format(self.conf['model compiler input file'])) elif len(compiler_cmds) > 1: self.logger.warning("There are more than one compiler command for {!r}". format(self.conf['model compiler input file'])) model_compiler_opts = clade.get_cmd_opts(compiler_cmds[0]['id']) model_compiler_cwd = compiler_cmds[0]['cwd'] else: # No specific compiler options are necessary for models. model_compiler_opts = [] if len(self.conf['working source trees']) != 1: raise NotImplementedError('There are several working source trees!') model_compiler_cwd = self.conf['working source trees'][0] model_grp = {'id': 'models', 'Extra CCs': []} for model in sorted(models, key=get_model_c_file): model_c_file = get_model_c_file(model) file, ext = os.path.splitext(os.path.join('models', os.path.basename(model_c_file))) base_name = klever.core.utils.unique_file_name(file, '{0}.json'.format(ext)) full_desc_file = '{0}{1}.json'.format(base_name, ext) out_file = '{0}.c'.format(base_name) # Always specify either specific model sets model or common one. opts = ['-DLDV_SETS_MODEL_' + (model['options']['sets model'] if isinstance(model, dict) and 'sets model' in model['options'] else self.conf['common sets model']).upper()] self.logger.debug('Dump CC full description to file "{0}"'.format(full_desc_file)) with open(full_desc_file, 'w', encoding='utf8') as fp: klever.core.utils.json_dump({ 'cwd': model_compiler_cwd, 'in': [os.path.relpath(model_c_file, os.path.realpath(clade.get_storage_path(model_compiler_cwd)))], 'out': [os.path.realpath(out_file)], 'opts': model_compiler_opts + opts }, fp, self.conf['keep intermediate files']) extra_cc = {'CC': os.path.relpath(full_desc_file, self.conf['main working directory'])} if 'generated' in model: extra_cc['generated'] = True model_grp['Extra CCs'].append(extra_cc) self.abstract_task_desc['grps'].append(model_grp) for dep in self.abstract_task_desc['deps'].values(): dep.append(model_grp['id'])
def weave(self): self.abstract_task_desc['extra C files'] = [] clade = Clade(self.conf['build base']) # This is required to get compiler (Aspectator) specific stdarg.h since kernel C files are compiled # with "-nostdinc" option and system stdarg.h couldn't be used. aspectator_search_dir = '-isystem' + core.utils.execute( self.logger, ('aspectator', '-print-file-name=include'), collect_all_stdout=True)[0] env = dict(os.environ) # Print stubs instead of inline Assembler since verifiers do not interpret it and even can fail. env['LDV_INLINE_ASM_STUB'] = '' for grp in self.abstract_task_desc['grps']: self.logger.info('Weave in C files of group "{0}"'.format( grp['id'])) for extra_cc in grp['Extra CCs']: if 'CC' in extra_cc: if extra_cc['CC'].isdigit(): cc = clade.get_cmd(extra_cc['CC'], with_opts=True) else: with open(os.path.join( self.conf['main working directory'], extra_cc['CC']), encoding='utf8') as fp: cc = json.load(fp) # extra_cc is a cc command that is not from Clade # Thus paths in it need to be converted to be absolute # like in other Clade commands if "cwd" in cc and "in" in cc: cc["in"] = [ os.path.join(cc["cwd"], cc_in) for cc_in in cc["in"] ] if "cwd" in cc and "out" in cc: cc["out"] = [ os.path.join(cc["cwd"], cc_out) for cc_out in cc["out"] ] self.logger.info('Weave in C file "{0}"'.format( cc['in'][0])) cc['out'][0] = '{0}.c'.format( core.utils.unique_file_name( os.path.splitext(os.path.basename( cc['out'][0]))[0], '.abs-paths.i')) # Produce aspect to be weaved in. if 'plugin aspects' in extra_cc: self.logger.info( 'Concatenate all aspects of all plugins together') # Resulting aspect. aspect = 'aspect' # Get all aspects. Place RSG aspects at beginning since they can instrument entities added by # aspects of other plugins while corresponding function declarations still need be at beginning # of file. aspects = [] for plugin_aspects in extra_cc['plugin aspects']: if plugin_aspects['plugin'] == 'RSG': aspects[0:0] = plugin_aspects['aspects'] else: aspects.extend(plugin_aspects['aspects']) # Concatenate aspects. with open( aspect, 'w', encoding='utf8' ) as fout, fileinput.input( [ os.path.join( self.conf['main working directory'], aspect) for aspect in aspects ], openhook=fileinput.hook_encoded( 'utf8')) as fin: for line in fin: fout.write(line) else: # Simulate resulting aspect. aspect = '/dev/null' self.logger.debug( 'Aspect to be weaved in is "{0}"'.format(aspect)) core.utils.execute( self.logger, tuple([ 'cif', '--in', clade.get_storage_path(cc['in'][0]), '--aspect', os.path.realpath(aspect), # Besides header files specific for requirements specifications will be searched for. '--general-opts', '-I' + os.path.realpath( os.path.dirname(self.conf['requirements DB'])), '--aspect-preprocessing-opts', ' '.join(self.conf['aspect preprocessing options'] ) if 'aspect preprocessing options' in self.conf else '', '--out', os.path.realpath(cc['out'][0]), '--back-end', 'src', '--debug', 'DEBUG' ] + (['--keep'] if self. conf['keep intermediate files'] else []) + ['--'] + core.vtg.utils.prepare_cif_opts( self.conf, cc['opts'], clade.storage_dir) + [aspectator_search_dir]), env=env, cwd=clade.get_storage_path(cc['cwd']), timeout=0.01, filter_func=core.vtg.utils.CIFErrorFilter()) self.logger.debug('C file "{0}" was weaved in'.format( cc['in'][0])) # In addition preprocess output files since CIF outputs a bit unpreprocessed files. preprocessed_c_file = '{}.i'.format( os.path.splitext(cc['out'][0])[0]) core.utils.execute( self.logger, ('aspectator', '-E', '-x', 'c', cc['out'][0], '-o', preprocessed_c_file), timeout=0.01) if not self.conf['keep intermediate files']: os.remove(cc['out'][0]) self.logger.debug( 'Preprocessed weaved C file was put to "{0}"'.format( preprocessed_c_file)) abs_paths_c_file = '{0}.abs-paths.i'.format( os.path.splitext(cc['out'][0])[0]) with open(preprocessed_c_file, encoding='utf8') as fp_in, open( abs_paths_c_file, 'w', encoding='utf8') as fp_out: # Print preprocessor header as is. first_line = fp_in.readline() fp_out.write(first_line) for line in fp_in: fp_out.write(line) if line == first_line: break # Replace relative file paths with absolute ones for line directives in other lines. for line in fp_in: match = re.match(r'(# \d+ ")(.+)("\n)', line) if match: file = match.group(2) if not os.path.isabs(file): # All relative file paths are relative to CC working directory. file = os.path.abspath( os.path.join( os.path.realpath(clade.storage_dir) + cc['cwd'], file)) fp_out.write( match.group(1) + file + match.group(3)) else: fp_out.write(line) if not self.conf['keep intermediate files']: os.remove(preprocessed_c_file) self.logger.debug( 'Preprocessed weaved C file with absolute paths was put to "{0}"' .format(abs_paths_c_file)) extra_c_file = { 'C file': os.path.relpath(abs_paths_c_file, self.conf['main working directory']) } else: extra_c_file = {} if 'requirement id' in extra_cc: extra_c_file['requirement id'] = extra_cc['requirement id'] if 'bug kinds' in extra_cc: extra_c_file['bug kinds'] = extra_cc['bug kinds'] self.abstract_task_desc['extra C files'].append(extra_c_file) # These sections won't be reffered any more. del (self.abstract_task_desc['grps']) del (self.abstract_task_desc['deps'])
class Job(klever.core.components.Component): CORE_COMPONENTS = [ 'PFG', 'VTG', 'VRP' ] def __init__(self, conf, logger, parent_id, callbacks, mqs, vals, id=None, work_dir=None, attrs=None, separate_from_parent=True, include_child_resources=False, components_common_conf=None): super(Job, self).__init__(conf, logger, parent_id, callbacks, mqs, vals, id, work_dir, attrs, separate_from_parent, include_child_resources) self.common_components_conf = components_common_conf if work_dir: self.common_components_conf['additional sources directory'] = os.path.join(os.path.realpath(work_dir), 'additional sources') self.clade = None self.components = [] self.component_processes = [] def decide_job_or_sub_job(self): self.logger.info('Decide job/sub-job "{0}"'.format(self.id)) # This is required to associate verification results with particular sub-jobs. # Skip leading "/" since this identifier is used in os.path.join() that returns absolute path otherwise. self.common_components_conf['sub-job identifier'] = self.id[1:] self.logger.info('Get specifications set') if 'specifications set' in self.common_components_conf: spec_set = self.common_components_conf['specifications set'] else: raise KeyError('Specify attribute "specifications set" within job.json') self.logger.debug('Specifications set is "{0}"'.format(spec_set)) # Check that specifications set is supported. with open(self.common_components_conf['specifications base'], encoding='utf-8') as fp: req_spec_base = json.load(fp) spec_set = self.common_components_conf['specifications set'] if spec_set not in req_spec_base['specification sets']: raise ValueError("Klever does not support specifications set {!r} yet, available options are: {}" .format(spec_set, ', '.join(req_spec_base['specification sets']))) # Check and set build base here since many Core components need it. self.__set_build_base() self.clade = Clade(self.common_components_conf['build base']) if not self.clade.work_dir_ok(): raise RuntimeError(f'Build base "{self.common_components_conf["build base"]}" is not OK') self.__retrieve_working_src_trees() self.__get_original_sources_basic_info() self.__upload_original_sources() # Create directory where files will be cached and remember absolute path to it for components. os.mkdir('cache') self.common_components_conf['cache directory'] = os.path.realpath('cache') if self.common_components_conf['keep intermediate files']: self.logger.debug('Create components configuration file "conf.json"') with open('conf.json', 'w', encoding='utf-8') as fp: json.dump(self.common_components_conf, fp, ensure_ascii=False, sort_keys=True, indent=4) self.__get_job_or_sub_job_components() self.callbacks = klever.core.components.get_component_callbacks(self.logger, [type(self)] + self.components) self.launch_sub_job_components() self.clean_dir = True self.logger.info("All components finished") if self.conf.get('collect total code coverage', None): self.logger.debug('Waiting for a collecting coverage') while not self.vals['coverage_finished'].get(self.common_components_conf['sub-job identifier'], True): time.sleep(1) self.logger.debug("Coverage collected") main = decide_job_or_sub_job def __set_build_base(self): if 'build base' not in self.common_components_conf: raise KeyError("Provide 'build base' configuration option to start verification") common_advice = 'please, fix "job.json" (attribute "build base")' common_advice += ' or/and deployment configuration file (attribute "Klever Build Bases")' # Try to find specified build base either in normal way or additionally in directory "build bases" that is # convenient to use when working with many build bases. try: build_base = klever.core.utils.find_file_or_dir(self.logger, self.common_components_conf['main working directory'], self.common_components_conf['build base']) except FileNotFoundError as e: self.logger.warning('Failed to find build base:\n{}'.format(traceback.format_exc().rstrip())) try: build_base = klever.core.utils.find_file_or_dir( self.logger, self.common_components_conf['main working directory'], os.path.join('build bases', self.common_components_conf['build base'])) except FileNotFoundError as e: self.logger.warning('Failed to find build base:\n{}'.format(traceback.format_exc().rstrip())) raise FileNotFoundError( 'Specified build base "{0}" does not exist, {1}'.format(self.common_components_conf['build base'], common_advice)) from None # Extract build base from archive. There should not be any intermediate directories in archives. if os.path.isfile(build_base) and (tarfile.is_tarfile(build_base) or zipfile.is_zipfile(build_base)): if tarfile.is_tarfile(build_base): self.logger.debug('Build base "{0}" is provided in form of TAR archive'.format(build_base)) with tarfile.open(build_base) as TarFile: TarFile.extractall('build base') else: self.logger.debug('Build base "{0}" is provided in form of ZIP archive'.format(build_base)) with zipfile.ZipFile(build_base) as zfp: zfp.extractall('build base') # Directory contains extracted build base. extracted_from = ' extracted from "{0}"'.format(os.path.realpath(build_base)) build_base = 'build base' else: extracted_from = '' # We need to specify absolute path to build base since it will be used in different Klever components. Besides, # this simplifies troubleshooting. build_base = os.path.realpath(build_base) # TODO: fix after https://github.com/17451k/clade/issues/108. if not os.path.isdir(build_base): raise FileExistsError('Build base "{0}" is not a directory, {1}' .format(build_base, extracted_from, common_advice)) if not os.path.isfile(os.path.join(build_base, 'meta.json')): raise FileExistsError( 'Directory "{0}"{1} is not a build base since it does not contain file "meta.json", {2}' .format(build_base, extracted_from, common_advice)) self.common_components_conf['build base'] = build_base self.logger.debug('Klever components will use build base "{0}"' .format(self.common_components_conf['build base'])) # Klever will try to cut off either working source trees (if specified) or maximum common paths of CC/CL input files # and LD/Link output files (otherwise) from referred file names. Sometimes this is rather optional like for source # files referred by error traces, but, say, for program fragment identifiers this is strictly necessary, e.g. # because of otherwise expert assessment will not work as expected. def __retrieve_working_src_trees(self): clade_meta = self.clade.get_meta() # Best of all if users specify working source trees in build bases manually themselves. It is a most accurate # approach. if 'working source trees' in clade_meta: work_src_trees = clade_meta['working source trees'] # Otherwise try to find out them automatically as described above. else: in_files = [] for cmd in self.clade.get_all_cmds_by_type("CC") + self.clade.get_all_cmds_by_type("CL"): if cmd['in']: for in_file in cmd['in']: # Sometimes some auxiliary stuff is built in addition to normal C source files that are most # likely located in a place we would like to get. if not in_file.startswith('/tmp') and in_file != '/dev/null': in_files.append(os.path.join(cmd['cwd'], in_file)) in_files_prefix = os.path.dirname(os.path.commonprefix(in_files)) self.logger.info('Common prefix of CC/CL input files is "{0}"'.format(in_files_prefix)) out_files = [] for cmd in self.clade.get_all_cmds_by_type("LD") + self.clade.get_all_cmds_by_type("Link"): if cmd['out']: for out_file in cmd['out']: # Like above. if not out_file.startswith('/tmp') and out_file != '/dev/null': out_files.append(os.path.join(cmd['cwd'], out_file)) out_files_prefix = os.path.dirname(os.path.commonprefix(out_files)) self.logger.info('Common prefix of LD/Link output files is "{0}"'.format(out_files_prefix)) # Meaningful paths look like "/dir...". meaningful_paths = [] for path in (in_files_prefix, out_files_prefix): if path and path != os.path.sep and path not in meaningful_paths: meaningful_paths.append(path) if meaningful_paths: work_src_trees = meaningful_paths # At least consider build directory as working source tree if the automatic procedure fails. else: self.logger.warning( 'Consider build directory "{0}" as working source tree.' 'This may be dangerous and we recommend to specify appropriate working source trees manually!' .format(clade_meta['build_dir'])) work_src_trees = [clade_meta['build_dir']] # Consider minimal path if it is common prefix for other ones. For instance, if we have "/dir1/dir2" and "/dir1" # then "/dir1" will become the only working source tree. if len(work_src_trees) > 1: min_work_src_tree = min(work_src_trees) if os.path.commonprefix(work_src_trees) == min_work_src_tree: work_src_trees = [min_work_src_tree] self.logger.info( 'Working source trees to be used are as follows:\n{0}' .format('\n'.join([' {0}'.format(t) for t in work_src_trees]))) self.common_components_conf['working source trees'] = work_src_trees def __refer_original_sources(self, src_id): klever.core.utils.report( self.logger, 'patch', { 'identifier': self.id, 'original_sources': src_id }, self.mqs['report files'], self.vals['report id'], self.conf['main working directory'] ) def __process_source_files(self): for file_name in self.clade.src_info: self.mqs['file names'].put(file_name) for i in range(self.workers_num): self.mqs['file names'].put(None) def __process_source_file(self): while True: file_name = self.mqs['file names'].get() if not file_name: return src_file_name = klever.core.utils.make_relative_path(self.common_components_conf['working source trees'], file_name) if src_file_name != file_name: src_file_name = os.path.join('source files', src_file_name) new_file_name = os.path.join('original sources', src_file_name.lstrip(os.path.sep)) os.makedirs(os.path.dirname(new_file_name), exist_ok=True) shutil.copy(self.clade.get_storage_path(file_name), new_file_name) cross_refs = CrossRefs(self.common_components_conf, self.logger, self.clade, file_name, new_file_name, self.common_components_conf['working source trees'], 'source files') cross_refs.get_cross_refs() def __get_original_sources_basic_info(self): self.logger.info('Get information on original sources for following visualization of uncovered source files') # For each source file we need to know the total number of lines and places where functions are defined. src_files_info = dict() for file_name, file_size in self.clade.src_info.items(): src_file_name = klever.core.utils.make_relative_path(self.common_components_conf['working source trees'], file_name) # Skip non-source files. if src_file_name == file_name: continue src_file_name = os.path.join('source files', src_file_name) src_files_info[src_file_name] = list() # Store source file size. src_files_info[src_file_name].append(file_size['loc']) # Store source file function definition lines. func_def_lines = list() funcs = self.clade.get_functions_by_file([file_name], False) if funcs: for func_name, func_info in list(funcs.values())[0].items(): func_def_lines.append(int(func_info['line'])) src_files_info[src_file_name].append(sorted(func_def_lines)) # Dump obtain information (huge data!) to load it when reporting total code coverage if everything will be okay. with open('original sources basic information.json', 'w') as fp: klever.core.utils.json_dump(src_files_info, fp, self.conf['keep intermediate files']) def __upload_original_sources(self): # Use Clade UUID to distinguish various original sources. It is pretty well since this UUID is uuid.uuid4(). src_id = self.clade.get_uuid() # In addition, take into account a meta content as we like to change it manually often. In this case it may be # necessary to re-index the build base. It is not clear if this is the case actually, so, do this in case of # any changes in meta. src_id += '-' + klever.core.utils.get_file_name_checksum(json.dumps(self.clade.get_meta()))[:12] session = klever.core.session.Session(self.logger, self.conf['Klever Bridge'], self.conf['identifier']) if session.check_original_sources(src_id): self.logger.info('Original sources were uploaded already') self.__refer_original_sources(src_id) return self.logger.info( 'Cut off working source trees or build directory from original source file names and convert index data') os.makedirs('original sources') self.mqs['file names'] = multiprocessing.Queue() self.workers_num = klever.core.utils.get_parallel_threads_num(self.logger, self.conf) subcomponents = [('PSFS', self.__process_source_files)] for i in range(self.workers_num): subcomponents.append(('PSF', self.__process_source_file)) self.launch_subcomponents(False, *subcomponents) self.mqs['file names'].close() self.logger.info('Compress original sources') klever.core.utils.ArchiveFiles(['original sources']).make_archive('original sources.zip') self.logger.info('Upload original sources') try: session.upload_original_sources(src_id, 'original sources.zip') # Do not fail if there are already original sources. There may be complex data races because of checking and # uploading original sources archive are not atomic. except klever.core.session.BridgeError: if "original sources with this identifier already exists." not in list(session.error.values())[0]: raise self.__refer_original_sources(src_id) if not self.conf['keep intermediate files']: shutil.rmtree('original sources') os.remove('original sources.zip') def __get_job_or_sub_job_components(self): self.logger.info('Get components for sub-job "{0}"'.format(self.id)) self.components = [getattr(importlib.import_module('.{0}'.format(component.lower()), 'klever.core'), component) for component in self.CORE_COMPONENTS] self.logger.debug('Components to be launched: "{0}"'.format( ', '.join([component.__name__ for component in self.components]))) def launch_sub_job_components(self): """Has callbacks""" self.logger.info('Launch components for sub-job "{0}"'.format(self.id)) for component in self.components: p = component(self.common_components_conf, self.logger, self.id, self.callbacks, self.mqs, self.vals, separate_from_parent=True) self.component_processes.append(p) klever.core.components.launch_workers(self.logger, self.component_processes)
def request_arg_signs(self): self.logger.info('Request argument signatures') clade = Clade(work_dir=self.conf['build base']) if not clade.work_dir_ok(): raise RuntimeError('Build base is not OK') meta = clade.get_meta() for request_aspect in self.conf['request aspects']: request_aspect = klever.core.vtg.utils.find_file_or_dir(self.logger, self.conf['main working directory'], request_aspect) self.logger.debug('Request aspect is "{0}"'.format(request_aspect)) for grp in self.abstract_task_desc['grps']: self.logger.info('Request argument signatures for C files of group "{0}"'.format(grp['id'])) for extra_cc in grp['Extra CCs']: infile = extra_cc['in file'] self.logger.info('Request argument signatures for C file "{0}"'.format(infile)) cc = clade.get_cmd(*extra_cc['CC'], with_opts=True) env = dict(os.environ) env['LDV_ARG_SIGNS_FILE'] = os.path.realpath( os.path.splitext(os.path.splitext(os.path.basename(request_aspect))[0])[0]) self.logger.debug('Argument signature file is "{0}"' .format(os.path.relpath(env['LDV_ARG_SIGNS_FILE']))) # Add plugin aspects produced thus far (by EMG) since they can include additional headers for which # additional argument signatures should be extracted. Like in Weaver. if 'plugin aspects' in extra_cc: self.logger.info('Concatenate all aspects of all plugins together') # Resulting request aspect. aspect = '{0}.aspect'.format( klever.core.utils.unique_file_name(os.path.splitext(os.path.basename(infile))[0], '.aspect')) # Get all aspects. Place original request aspect at beginning since it can instrument entities # added by aspects of other plugins while corresponding function declarations still need be at # beginning of file. aspects = [os.path.relpath(request_aspect, self.conf['main working directory'])] for plugin_aspects in extra_cc['plugin aspects']: aspects.extend(plugin_aspects['aspects']) # Concatenate aspects. with open(aspect, 'w', encoding='utf-8') as fout, fileinput.input( [os.path.join(self.conf['main working directory'], aspect) for aspect in aspects], openhook=fileinput.hook_encoded('utf-8')) as fin: for line in fin: fout.write(line) else: aspect = request_aspect storage_path = clade.get_storage_path(infile) if meta['conf'].get('Compiler.preprocess_cmds', False) and \ 'klever-core-work-dir' not in storage_path: storage_path = storage_path.split('.c')[0] + '.i' opts = cc['opts'] # Like in Weaver. opts.append(klever.core.vtg.utils.define_arch_dependent_macro(self.conf)) klever.core.utils.execute( self.logger, tuple( [ klever.core.vtg.utils.get_cif_or_aspectator_exec(self.conf, 'cif'), '--in', storage_path, '--aspect', os.path.realpath(aspect), '--stage', 'instrumentation', '--out', os.path.realpath('{0}.c'.format(klever.core.utils.unique_file_name( os.path.splitext(os.path.basename(infile))[0], '.c.aux'))), '--debug', 'DEBUG' ] + (['--keep'] if self.conf['keep intermediate files'] else []) + ['--'] + klever.core.vtg.utils.prepare_cif_opts(opts, clade) + # Like in Weaver. ['-I' + os.path.join(os.path.dirname(self.conf['specifications base']), 'include')] ), env, cwd=clade.get_storage_path(cc['cwd']), timeout=0.01, filter_func=klever.core.vtg.utils.CIFErrorFilter())