def add_models(self, generated_models): self.logger.info( 'Add models to abstract verification task description') models = {} if 'environment model' in self.abstract_task_desc: rel_path = os.path.relpath( os.path.join(self.conf['main working directory'], self.abstract_task_desc['environment model']), os.path.curdir) models[rel_path] = {} # Get common and requirement specific models. if 'common models' in self.conf and 'models' in self.conf: for common_model_c_file in self.conf['common models']: if common_model_c_file in self.conf['models']: raise KeyError( 'C file "{0}" is specified in both common and requirement specific models' ) if 'models' in self.conf: for model_c_file in self.conf['models']: # Specify additional settings for generated models that have not any settings. if model_c_file.startswith('$'): is_generated_model_c_file_found = False for generated_model_c_file in generated_models: if generated_model_c_file.endswith(model_c_file[1:]): models[generated_model_c_file] = self.conf[ 'models'][model_c_file] is_generated_model_c_file_found = True if not is_generated_model_c_file_found: raise KeyError( 'Model C file "{0}" was not generated'.format( model_c_file[1:])) # Like common models processed below. for model_c_file in self.conf['models']: if not model_c_file.startswith('$'): model_c_file_realpath = core.vtg.utils.find_file_or_dir( self.logger, self.conf['main working directory'], model_c_file) self.logger.debug('Get model with C file "{0}"'.format( model_c_file_realpath)) models[model_c_file_realpath] = self.conf['models'][ model_c_file] if 'common models' in self.conf: for common_model_c_file in self.conf['common models']: common_model_c_file_realpath = core.vtg.utils.find_file_or_dir( self.logger, self.conf['main working directory'], common_model_c_file) self.logger.debug('Get common model with C file "{0}"'.format( common_model_c_file_realpath)) models[common_model_c_file_realpath] = self.conf[ 'common models'][common_model_c_file] self.logger.debug('Resulting models are: {0}'.format(models)) if not models: self.logger.warning('No models are specified') return # CC extra full description files will be put to this directory as well as corresponding intermediate and final # output files. os.makedirs('models'.encode('utf8')) self.logger.info( 'Add aspects to abstract verification task description') aspects = [] for model_c_file in models: aspect = '{}.aspect'.format(os.path.splitext(model_c_file)[0]) if not os.path.isfile(aspect): continue self.logger.debug('Get aspect "{0}"'.format(aspect)) aspects.append(aspect) # Sort aspects to apply them in the deterministic order. aspects.sort() for grp in self.abstract_task_desc['grps']: self.logger.info('Add aspects to C files of group "{0}"'.format( grp['id'])) for extra_cc in grp['Extra CCs']: if 'plugin aspects' not in extra_cc: extra_cc['plugin aspects'] = [] extra_cc['plugin aspects'].append({ 'plugin': self.name, 'aspects': [ os.path.relpath(aspect, self.conf['main working directory']) for aspect in aspects ] }) for model_c_file in models: model = models[model_c_file] if 'bug kinds' in model: self.logger.info( 'Preprocess bug kinds for model with C file "{0}"'.format( model_c_file)) # Collect all bug kinds specified in model to check that valid bug kinds are specified in requirement # model description. bug_kinds = set() lines = [] with open(model_c_file, encoding='utf8') as fp: for line in fp: # Bug kinds are specified in form of strings like in requirements DB as first actual # parameters of ldv_assert(). match = re.search(r'ldv_assert\("([^"]+)"', line) if match: bug_kind, = match.groups() bug_kinds.add(bug_kind) # Include bug kinds in names of ldv_assert(). lines.append( re.sub( r'ldv_assert\("([^"]+)", ?', r'ldv_assert_{0}('.format( re.sub(r'\W', '_', bug_kind)), line)) else: lines.append(line) for bug_kind in model['bug kinds']: if bug_kind not in bug_kinds: raise KeyError( 'Invalid bug kind "{0}" is specified in requirement model description' .format(bug_kind)) preprocessed_model_c_file = '{0}.bk.c'.format( core.utils.unique_file_name( os.path.join( 'models', os.path.splitext( os.path.basename(model_c_file))[0]), '.bk.c')) with open(preprocessed_model_c_file, 'w', encoding='utf8') as fp: # Create ldv_assert*() function declarations to avoid compilation warnings. These functions will # be defined later somehow by VTG. for bug_kind in sorted(bug_kinds): fp.write('extern void ldv_assert_{0}(int);\n'.format( re.sub(r'\W', '_', bug_kind))) # Specify original location to avoid references to *.bk.c files in error traces. fp.write('# 1 "{0}"\n'.format( os.path.abspath(model_c_file))) for line in lines: fp.write(line) model[ 'bug kinds preprocessed C file'] = preprocessed_model_c_file self.logger.debug( 'Preprocessed bug kinds for model with C file "{0}" was placed to "{1}"' .format(model_c_file, preprocessed_model_c_file)) else: model['bug kinds preprocessed C file'] = model_c_file # Generate CC full description file per each model and add it to abstract task description. # First of all obtain CC options to be used to compile models. clade = Clade(self.conf['build base']) # Relative path to source file which CC options to be used is specified in configuration. Clade needs absolute # path. The former is relative to one of source paths. for path in self.conf['source paths']: opts_file = os.path.join(path, self.conf['opts file']) try: empty_cc = list(clade.get_compilation_cmds_by_file(opts_file)) except KeyError: pass if not empty_cc: raise RuntimeError("There is not of cc commands for {!r}".format( self.conf['project']['opts file'])) elif len(empty_cc) > 1: self.logger.warning( "There are more than one cc command for {!r}".format( self.conf['project']['opts file'])) empty_cc = empty_cc.pop() empty_cc['opts'] = clade.get_cmd_opts(empty_cc['id']) model_grp = {'id': 'models', 'Extra CCs': []} for model_c_file in sorted(models): model = models[model_c_file] extra_cc = {} if 'bug kinds preprocessed C file' in model: file, ext = os.path.splitext( os.path.join( 'models', os.path.basename( model['bug kinds preprocessed C file']))) base_name = core.utils.unique_file_name( file, '{0}.json'.format(ext)) full_desc_file = '{0}{1}.json'.format(base_name, ext) out_file = '{0}.c'.format(base_name) self.logger.debug( 'Dump CC full description to file "{0}"'.format( full_desc_file)) with open(full_desc_file, 'w', encoding='utf8') as fp: json.dump( { 'cwd': empty_cc['cwd'], 'in': [ os.path.relpath( model['bug kinds preprocessed C file'], os.path.realpath( clade.get_storage_path( empty_cc['cwd']))) ], 'out': [os.path.realpath(out_file)], 'opts': empty_cc['opts'] + [ '-DLDV_SETS_MODEL_' + (model['sets model'] if 'sets model' in model else self.conf['common sets model']).upper() ] }, fp, ensure_ascii=False, sort_keys=True, indent=4) extra_cc['CC'] = os.path.relpath( full_desc_file, self.conf['main working directory']) if 'bug kinds' in model: extra_cc['bug kinds'] = model['bug kinds'] if extra_cc: model_grp['Extra CCs'].append(extra_cc) self.abstract_task_desc['grps'].append(model_grp) for dep in self.abstract_task_desc['deps'].values(): dep.append(model_grp['id'])
def add_models(self, generated_models): self.logger.info( 'Add models to abstract verification task description') models = [] if 'environment model' in self.abstract_task_desc: models.append({ 'model': os.path.relpath( os.path.join(self.conf['main working directory'], self.abstract_task_desc['environment model']), os.path.curdir), 'options': {}, 'generated': True }) if 'extra C files' in self.abstract_task_desc: self.abstract_task_desc['extra C files'] = [] for c_file in (extra_c_file["C file"] for extra_c_file in self.abstract_task_desc['extra C files'] if "C file" in extra_c_file): models.append( os.path.relpath( os.path.join(self.conf['main working directory'], c_file), os.path.curdir)) def get_model_c_file(model): # Model may be a C file or a dictionary with model file and option attributes. if isinstance(model, dict): return model['model'] else: return model # Get common and requirement specific models. if 'exclude common models' in self.conf: self.logger.info('Common models to be excluded:\n{0}'.format( '\n'.join([ ' {0}'.format(m) for m in self.conf['exclude common models'] ]))) common_models = [ m for m in self.conf['common models'] if m not in self.conf['exclude common models'] ] else: common_models = self.conf['common models'] if common_models and 'models' in self.conf: for common_model in common_models: common_model_c_file = get_model_c_file(common_model) for model in self.conf['models']: if common_model_c_file == get_model_c_file(model): raise KeyError( 'C file "{0}" is specified in both common and requirement specific models' .format(common_model_c_file)) def add_model(model, model_c_file_realpath): if isinstance(model, dict): models.append({ 'model': model_c_file_realpath, 'options': model['options'] }) else: models.append(model_c_file_realpath) if 'models' in self.conf: # Find out actual C files. for model in self.conf['models']: model_c_file = get_model_c_file(model) # Handle generated models which C files start with "$". if model_c_file.startswith('$'): is_generated_model_c_file_found = False for generated_model_c_file in generated_models: if generated_model_c_file.endswith(model_c_file[1:]): if isinstance(model, dict): # Specify model options for generated models that can not have model options themselves. models.append({ 'model': generated_model_c_file, 'options': model['options'], 'generated': True }) else: models.append({ 'model': generated_model_c_file, 'options': {}, 'generated': True }) is_generated_model_c_file_found = True if not is_generated_model_c_file_found: raise KeyError( 'Model C file "{0}" was not generated'.format( model_c_file[1:])) # Handle non-generated models. else: model_c_file_realpath = klever.core.vtg.utils.find_file_or_dir( self.logger, self.conf['main working directory'], model_c_file) self.logger.debug('Get model with C file "{0}"'.format( model_c_file_realpath)) add_model(model, model_c_file_realpath) # Like for models above. for common_model in common_models: common_model_c_file = get_model_c_file(common_model) common_model_c_file_realpath = klever.core.vtg.utils.find_file_or_dir( self.logger, self.conf['main working directory'], common_model_c_file) self.logger.debug('Get common model with C file "{0}"'.format( common_model_c_file_realpath)) add_model(common_model, common_model_c_file_realpath) self.logger.debug('Resulting models are: {0}'.format(models)) if not models: self.logger.warning('No models are specified') return # CC extra full description files will be put to this directory as well as corresponding intermediate and final # output files. os.makedirs('models'.encode('utf-8')) self.logger.info( 'Add aspects to abstract verification task description') aspects = [] for model in models: aspect = '{}.aspect'.format( os.path.splitext(get_model_c_file(model))[0]) # Aspects are not mandatory. There may be pure C models, e.g. when one needs to provide some definitions # without any weaving. if not os.path.isfile(aspect): continue if not os.stat(aspect).st_size: raise ValueError( 'Aspect "{0}" is empty and should be removed from the verification job' .format(aspect)) self.logger.debug('Get aspect "{0}"'.format(aspect)) aspects.append(aspect) # Sort aspects to apply them in the deterministic order. aspects.sort() # Always specify either specific model sets model or common one. opts = [ '-DLDV_SETS_MODEL_' + (model['options']['sets model'] if isinstance(model, dict) and 'sets model' in model['options'] else self.conf['common sets model']).upper() ] if self.conf.get('memory safety'): opts += ['-DLDV_MEMORY_SAFETY'] if 'specifications set' in self.conf: opts += [ '-DLDV_SPECS_SET_{0}'.format( self.conf['specifications set'].replace('.', '_')) ] for grp in self.abstract_task_desc['grps']: self.logger.info('Add aspects to C files of group "{0}"'.format( grp['id'])) for extra_cc in grp['Extra CCs']: if 'plugin aspects' not in extra_cc: extra_cc['plugin aspects'] = [] extra_cc['plugin aspects'].append({ 'plugin': self.name, 'aspects': [ os.path.relpath(aspect, self.conf['main working directory']) for aspect in aspects ] }) extra_cc['opts'] = opts # Generate CC full description file per each model and add it to abstract task description. # First of all obtain CC options to be used to compile models. clade = Clade(self.conf['build base']) if not clade.work_dir_ok(): raise RuntimeError('Build base is not OK') meta = clade.get_meta() if not meta['conf'].get('Compiler.preprocess_cmds', False): # Model compiler input file represents input file which compiler options and CWD should be used for # compiling models. This input file is relative to one of source paths. compiler_cmds = None for path in self.conf['working source trees']: try: compiler_cmds = list( clade.get_compilation_cmds_by_file( os.path.normpath( os.path.join( path, self.conf['model compiler input file'])))) break except KeyError: pass if not compiler_cmds: raise RuntimeError( "There is no compiler commands for {!r}".format( self.conf['model compiler input file'])) elif len(compiler_cmds) > 1: self.logger.warning( "There are more than one compiler command for {!r}".format( self.conf['model compiler input file'])) model_compiler_opts = clade.get_cmd_opts(compiler_cmds[0]['id']) model_compiler_cwd = compiler_cmds[0]['cwd'] else: # No specific compiler options are necessary for models. model_compiler_opts = [] if len(self.conf['working source trees']) != 1: raise NotImplementedError( 'There are several working source trees!') model_compiler_cwd = self.conf['working source trees'][0] model_grp = {'id': 'models', 'Extra CCs': []} for model in sorted(models, key=get_model_c_file): model_c_file = get_model_c_file(model) file, ext = os.path.splitext( os.path.join('models', os.path.basename(model_c_file))) base_name = klever.core.utils.unique_file_name( file, '{0}.json'.format(ext)) full_desc_file = '{0}{1}.json'.format(base_name, ext) out_file = '{0}.c'.format(base_name) self.logger.debug('Dump CC full description to file "{0}"'.format( full_desc_file)) with open(full_desc_file, 'w', encoding='utf-8') as fp: klever.core.utils.json_dump( { 'cwd': model_compiler_cwd, 'in': [os.path.realpath(model_c_file)], 'out': [os.path.realpath(out_file)], 'opts': model_compiler_opts + opts }, fp, self.conf['keep intermediate files']) extra_cc = { 'CC': os.path.relpath(full_desc_file, self.conf['main working directory']) } if 'generated' in model: extra_cc['generated'] = True if isinstance(model, dict): if model['options'].get('weave in model aspect'): aspect = '{}.aspect'.format( os.path.splitext(get_model_c_file(model))[0]) if not os.path.isfile(aspect): raise FileNotFoundError( 'Aspect "{0}" to be weaved in model does not exist' .format(aspect)) extra_cc['plugin aspects'] = [{ 'plugin': self.name, 'aspects': [ os.path.relpath( aspect, self.conf['main working directory']) ] }] elif model['options'].get('weave in all aspects'): extra_cc['plugin aspects'] = [{ 'plugin': self.name, 'aspects': [ os.path.relpath( aspect, self.conf['main working directory']) for aspect in aspects ] }] model_grp['Extra CCs'].append(extra_cc) self.abstract_task_desc['grps'].append(model_grp) for dep in self.abstract_task_desc['deps'].values(): dep.append(model_grp['id'])