def test_macros(tmpdir, cmds_file): c = Clade(tmpdir, cmds_file) c.parse("Macros") definitions_are_ok(c.get_macros_definitions()) expansions_are_ok(c.get_macros_expansions())
class Source: """ Representation of a collection with the data collected by a source code analysis. The collection contains information about functions, variable initializations, a functions call graph, macros. """ def __init__(self, logger, conf, abstract_task): """ Setup initial attributes and get logger object. :param logger: logging object. :param conf: Source code analysis configuration. :param abstract_task: Abstract verification task dictionary (given by VTG). :param conf: Configuration properties dictionary. """ self.logger = logger self._conf = conf self._source_functions = dict() self._source_vars = dict() self._macros = dict() self.__function_calls_cache = dict() # Initialize Clade cient to make requests self._clade = Clade(self._conf['build base']) # Ask for dependencies for each CC cfiles, files_map = self._collect_file_dependencies(abstract_task) # Read file with source analysis self._import_code_analysis(cfiles, files_map) @property def source_functions(self): """ Return a list of function names. :return: function names list. """ return list(self._source_functions.keys()) def get_source_function(self, name, path=None, declaration=None): """ Provides the function by a given name from the collection. :param name: Function name. :param path: File where the function should be declared or defined. :param declaration: Declaration object representing the function of interest. :return: Function object or None. """ name = self.refined_name(name) if name and name in self._source_functions: if path and path in self._source_functions[name]: return self._source_functions[name][path] else: functions = self.get_source_functions(name, declaration=declaration) if len(functions) == 1: return functions[0] elif len(functions) > 1: raise ValueError( "There are several definitions of function {!r} in provided code you must specify " "scope".format(name)) return None def get_source_functions(self, name, declaration=None): """ Provides all functions found by a given name from the collection. :param name: Function name. :param declaration: Declaration object representing the function of interest. :return: List with Function objects. """ name = self.refined_name(name) result = [] if name and name in self._source_functions: for func in self._source_functions[name].values(): if func not in result and ( not declaration or (declaration and declaration.compare(func.declaration))): result.append(func) return result def set_source_function(self, new_obj, path): """ Replace an Function object in the collection. :param new_obj: Function object. :param path: File where the function should be declared or defined. :return: None. """ if new_obj.name not in self._source_functions: self._source_functions[new_obj.name] = dict() self._source_functions[new_obj.name][path] = new_obj def remove_source_function(self, name): """ Delete the function from the collection. :param name: Function name. :return: None. """ del self._source_functions[name] @property def source_variables(self): """ Return list of global variables. :return: Variable names list. """ return list(self._source_vars.keys()) def get_source_variable(self, name, path=None): """ Provides a gloabal variable by a given name and scope file from the collection. :param name: Variable name. :param path: File with the variable declaration or initialization. :return: Variable object or None. """ name = self.refined_name(name) if name and name in self._source_vars: if path and path in self._source_vars[name]: return self._source_vars[name][path] else: variables = self.get_source_variables(name) if len(variables) == 1: return variables[0] return None def get_source_variables(self, name): """ Provides all global variables by a given name from the collection. :param name: Variable name. :return: List with Variable objects. """ name = self.refined_name(name) result = [] if name and name in self._source_vars: for var in self._source_vars[name].values(): if var not in result: result.append(var) return result def set_source_variable(self, new_obj, path): """ Replace an object in global variables collection. :param new_obj: Variable object. :param path: File with the variable declaration or initialization. :return: None. """ if new_obj.name not in self._source_vars: self._source_vars[new_obj.name] = dict() self._source_vars[new_obj.name][path] = new_obj def remove_source_variable(self, name): """ Delete the global variable from the collection. :param name: Variable name. :return: None. """ del self._source_vars[name] def get_macro(self, name): """ Provides a macro by a given name from the collection. :param name: Macro name. :return: Macro object or None. """ if name in self._macros: return self._macros[name] else: return None def set_macro(self, new_obj): """ Set or replace an object in macros collection. :param new_obj: Macro object. :return: None. """ self._macros[new_obj.name] = new_obj def remove_macro(self, name): """ Delete the macro from the collection. :param name: Macro name. :return: None. """ del self._macros[name] @staticmethod def refined_name(call): """ Resolve function name from simple expressions which contains explicit function name like '& myfunc', '(myfunc)', '(& myfunc)' or 'myfunc'. :param call: An expression string. :return: Extracted function name string. """ name_re = re.compile("\(?\s*&?\s*(\w+)\s*\)?$") if name_re.fullmatch(call): return name_re.fullmatch(call).group(1) else: return None def _import_code_analysis(self, cfiles, dependencies): """ Read global variables, functions and macros to fill up the collection. :param source_analysis: Dictionary with the content of source analysis. :param files_map: Dictionary to resolve main file by an included file. :return: None. """ # Import typedefs if there are provided self.logger.info("Extract complete types definitions") typedef = self._clade.get_typedefs( set(dependencies.keys()).union(cfiles)) if typedef: import_typedefs(typedef, dependencies) variables = self._clade.get_variables(cfiles) if variables: self.logger.info("Import global variables initializations") for path, vals in variables.items(): for variable in vals: variable_name = extract_name(variable['declaration']) if not variable_name: raise ValueError('Global variable without a name') var = Variable(variable_name, variable['declaration']) # Here we know, that if we met a variable in an another file then it is an another variable because # a program should contain a single global variable initialization self.set_source_variable(var, path) var.declaration_files.add(path) var.initialization_file = path var.static = is_static(variable['declaration']) if 'value' in variable: var.value = variable['value'] # Variables which are used in variables initalizations self.logger.info("Import source functions") vfunctions = self._clade.get_used_in_vars_functions() # Get functions defined in dependencies and in the main functions and have calls cg = self._clade.get_callgraph(set(dependencies.keys())) # Function scope definitions # todo: maybe this should be fixed in Clade # As we will not get definitions for library functions if there are in compiled parts we should add all scopes # that are given for all function called from outside of the code we analyze for scope in (s for s in cfiles if s in cg): for func in (f for f in cg[scope] if cg[scope][f].get('calls')): for dep in cg[scope][func].get('calls'): dependencies.setdefault(dep, set()) dependencies[dep].add(scope) fs = self._clade.get_functions_by_file( set(dependencies.keys()).union(cfiles)) # Add called functions for scope in cg: for func in cg[scope]: desc = cg[scope][func] if scope in cfiles: # Definition of the function is in the code of interest self._add_function(func, scope, fs, dependencies, cfiles) # Add called functions for def_scope, cf_desc in desc.get('calls', dict()).items(): if def_scope not in cfiles: for called_func in ( f for f in cf_desc if def_scope in fs and f in fs[def_scope]): self._add_function(called_func, def_scope, fs, dependencies, cfiles) elif ('called_in' in desc and set(desc['called_in'].keys()).intersection(cfiles) ) or func in vfunctions: if scope in fs and func in fs[scope]: # Function is called in the target code but defined in dependencies self._add_function(func, scope, fs, dependencies, cfiles) elif scope != 'unknown': self.logger.warning( "There is no information on declarations of function {!r} from {!r} scope" .format(func, scope)) # Add functions missed in the call graph for scope in (s for s in fs if s in cfiles): for func in fs[scope]: func_intf = self.get_source_function(func, scope) if not func_intf: self._add_function(func, scope, fs, dependencies, cfiles) for func in self.source_functions: for obj in self.get_source_functions(func): scopes = set(obj.declaration_files).union(set( obj.header_files)) if not obj.definition_file: # It is likely be this way scopes.add('unknown') for scope in (s for s in scopes if cg.get(s, dict()).get(func)): for cscope, desc in ((s, d) for s, d in cg[scope][func].get( 'called_in', {}).items() if s in cfiles): for caller in desc: for line in desc[caller]: params = desc[caller][line].get('args') caller_intf = self.get_source_function( caller, cscope) obj.add_call(caller, cscope) if params: # Here can be functions which are not defined or visible for _, passed_func in list(params): passed_obj = self.get_source_function( passed_func, cscope) if not passed_obj: passed_scope = self._search_function( passed_func, cscope, fs) if passed_scope: self._add_function( passed_func, passed_scope, fs, dependencies, cfiles) else: self.logger.warning( "Cannot find function {!r} from scope {!r}" .format( passed_func, cscope)) # Ignore this call since model will not be correct without signature params = None break caller_intf.call_in_function(obj, params) macros_file = get_conf_property(self._conf['source analysis'], 'macros white list') if macros_file: macros_file = find_file_or_dir( self.logger, self._conf['main working directory'], macros_file) with open(macros_file, 'r', encoding='utf8') as fp: white_list = ujson.load(fp) if white_list: macros = self._clade.get_macros_expansions(cfiles, white_list) for path, macros in macros.items(): for macro, desc in macros.items(): obj = self.get_macro(macro) if not obj: obj = Macro(macro) for call in desc.get('args', []): obj.add_parameters(path, call) self.set_macro(obj) def _search_function(self, func_name, some_scope, fs): # Be aware of this funciton - it is costly if some_scope in fs and func_name in fs[some_scope]: return some_scope elif 'unknown' in fs and func_name in fs['unknown']: return 'unknown' else: for s in (s for s in fs if func_name in fs[s]): return s return None def _add_function(self, func, scope, fs, deps, cfiles): fs_desc = fs[scope][func] if scope == 'unknown': key = list(fs_desc['declarations'].keys())[0] signature = fs_desc['declarations'][key]['signature'] func_intf = Function(func, signature) # Do not set definition file since it is out of scope of the target program fragment else: signature = fs_desc.get('signature') func_intf = Function(func, signature) func_intf.definition_file = scope # Set static if fs_desc.get('type') == "static": func_intf.static = True else: func_intf.static = False # Add declarations files = {func_intf.definition_file } if func_intf.definition_file else set() if fs_desc['declarations']: files.update({ f for f in fs_desc['declarations'] if f != 'unknown' and f in deps }) for file in files: if file not in cfiles and file not in func_intf.header_files: func_intf.header_files.append(file) for cfile in deps[file]: self.set_source_function(func_intf, cfile) func_intf.declaration_files.add(cfile) def _collect_file_dependencies(self, abstract_task): """ Collect for each included header file or c file its "main" file to which it was included. This is required since we cannot write aspects and instrument files which have no CC command so me build this map. :param abstract_task: Abstract task dictionary. :return: Collection dictionary {included file: {files that include this one}}. """ collection = dict() c_files = set() def _collect_cc_deps(cfile, deps): # Collect for each file CC entry to which it is included for file in deps: if file not in collection: collection[file] = set() collection[file].add(cfile) # Read each CC description and import map of files to in files for group in abstract_task['grps']: for desc in group['Extra CCs']: cc_desc = self._clade.get_cmd(desc['CC']) c_file = cc_desc['in'][0] # Now read deps _collect_cc_deps(c_file, self._clade.get_cmd_deps(desc['CC'])) c_files.add(c_file) return c_files, collection
class Diff: def __init__(self, work_dir1, work_dir2, log_level="INFO"): self.work_dir1 = work_dir1 self.work_dir2 = work_dir2 self.cl1 = Clade(work_dir1) self.cl2 = Clade(work_dir2) logger.setLevel(log_level) def compare(self): self.compare_extension_lists() self.compare_pid_graphs() self.compare_cmds() self.compare_storages() self.compare_cmd_graphs() self.compare_src_graphs() self.compare_functions() self.compare_macros() self.compare_callgraphs() pass def compare_extension_lists(self): a = self.__get_extension_list(self.work_dir1) b = self.__get_extension_list(self.work_dir2) if a == b: logger.info("Sets of extensions are the same: {!r}".format( ", ".join(a))) else: logger.error("Sets of extensions are different") logger.error("First set: {!r}".format(", ".join(a))) logger.error("Second set: {!r}".format(", ".join(b))) def compare_pid_graphs(self): if not self.__ext_work_dirs_exist("PidGraph"): return keys_are_same = True direct_parents_are_same = True indirect_parents_are_same = True pid_by_id1 = self.cl1.pid_by_id pid_by_id2 = self.cl2.pid_by_id keys1 = pid_by_id1.keys() keys2 = pid_by_id2.keys() if keys1 == keys2: logger.debug("IDs of intercepted commands are the same") common_ids = keys1 else: common_ids = set(keys1) & set(keys2) keys_are_same = False removed = [x for x in keys1 if x not in common_ids] added = [x for x in keys2 if x not in common_ids] if removed: logger.error( "{!r} ids were removed from the pid graph".format(removed)) if added: logger.error( "{!r} ids were added to the pid graph".format(added)) for key in common_ids: if pid_by_id1[key] != pid_by_id2[key]: logger.error( "ID {!r} has different direct parents: {!r} vs {!r}". format(key, pid_by_id1[key], pid_by_id2[key])) direct_parents_are_same = False if direct_parents_are_same and keys_are_same: logger.debug("All ids have the same direct parents") elif direct_parents_are_same and not keys_are_same: logger.debug("All common ids have the same direct parents") pid_graph1 = self.cl1.pid_graph pid_graph2 = self.cl2.pid_graph for key in common_ids: if pid_graph1[key] != pid_graph2[key]: logger.error( "ID {!r} has different indirect parents: {!r} vs {!r}". format(key, pid_graph1[key], pid_graph2[key])) indirect_parents_are_same = False if indirect_parents_are_same and keys_are_same: logger.debug("All ids have the same indirect parents") elif indirect_parents_are_same and not keys_are_same: logger.debug("All common ids have the same indirect parents") if (keys_are_same and direct_parents_are_same and indirect_parents_are_same): logger.info("Pid graphs are the same") else: logger.info("Pid graphs are different") def compare_cmds(self): if not self.__ext_work_dirs_exist("CmdGraph"): return keys_are_same = True inputs_are_same = True outputs_are_same = True deps_are_same = True cmds1 = self.cl1.cmds cmds2 = self.cl2.cmds cmd_ids1 = [x["id"] for x in cmds1] cmd_ids2 = [x["id"] for x in cmds2] if cmd_ids1 == cmd_ids2: logger.debug("Sets of parsed commands are the same") common_ids = cmd_ids1 else: common_ids = set(cmd_ids1) & set(cmd_ids2) keys_are_same = False removed = [x for x in cmd_ids1 if x not in common_ids] added = [x for x in cmd_ids2 if x not in common_ids] for cmd_id in removed: logger.error("{!r} command with ID={!r} was removed".format( self.cl1.get_cmd_type(cmd_id), cmd_id)) for cmd_id in added: logger.error("{!r} command with ID={!r} was added".format( self.cl2.get_cmd_type(cmd_id), cmd_id)) for cmd_id in common_ids: cmd1 = self.cl1.get_cmd(cmd_id) cmd2 = self.cl2.get_cmd(cmd_id) if cmd1["in"] != cmd2["in"]: logger.error( "{!r} command with ID={!r} changed its input files: {!r} vs {!r}" .format( self.cl1.get_cmd_type(cmd_id), cmd_id, cmd1["in"], cmd2["in"], )) inputs_are_same = False if cmd1["out"] != cmd2["out"]: logger.error( "{!r} command with ID={!r} changed its output files: {!r} vs {!r}" .format( self.cl1.get_cmd_type(cmd_id), cmd_id, cmd1["out"], cmd2["out"], )) outputs_are_same = False common_compilation_cmds_ids = [ x["id"] for x in self.cl2.compilation_cmds if x["id"] in common_ids ] for cmd_id in common_compilation_cmds_ids: cmd_deps1 = set(self.cl1.get_cmd(cmd_id, with_deps=True)["deps"]) cmd_deps2 = set(self.cl2.get_cmd(cmd_id, with_deps=True)["deps"]) if cmd_deps1 != cmd_deps2: removed = cmd_deps1 - cmd_deps2 added = cmd_deps2 - cmd_deps1 for dep in removed: logger.error( "{!r} command with ID={!r} no longer has dependency: {!r}" .format(self.cl1.get_cmd_type(cmd_id), cmd_id, dep)) for dep in added: logger.error( "{!r} command with ID={!r} has new dependency: {!r}". format(self.cl1.get_cmd_type(cmd_id), cmd_id, dep)) deps_are_same = False if (keys_are_same and inputs_are_same and outputs_are_same and deps_are_same): logger.info("Parsed commands are the same") else: logger.info("Parsed commands are different") def compare_storages(self): if not self.__ext_work_dirs_exist("Storage"): return storage_files1 = set() storage_files2 = set() for root, _, filenames in os.walk(self.cl1.storage_dir): for filename in filenames: storage_files1.add( os.path.relpath( os.path.join(root, filename), start=self.cl1.storage_dir, )) for root, _, filenames in os.walk(self.cl2.storage_dir): for filename in filenames: storage_files2.add( os.path.relpath( os.path.join(root, filename), start=self.cl2.storage_dir, )) if storage_files1 == storage_files2: logger.info("Files in the Storage are the same") else: removed = storage_files1 - storage_files2 added = storage_files2 - storage_files1 for file in removed: logger.error( "{!r} file was removed from the Storage".format(file)) for file in added: logger.error("{!r} file was added to the Storage".format(file)) def compare_cmd_graphs(self): if not self.__ext_work_dirs_exist("CmdGraph"): return cmd_graph1 = self.cl1.cmd_graph cmd_graph2 = self.cl2.cmd_graph used_by_are_same = True using_are_same = True cmds_ids1 = set(cmd_graph1.keys()) cmds_ids2 = set(cmd_graph2.keys()) if cmds_ids1 == cmds_ids2: common_ids = cmds_ids1 else: common_ids = cmds_ids1 & cmds_ids2 for cmd_id in common_ids: used_by1 = set(cmd_graph1[cmd_id]["used_by"]) used_by2 = set(cmd_graph2[cmd_id]["used_by"]) if used_by1 != used_by2: removed = used_by1 - used_by2 added = used_by2 - used_by1 if removed: logger.error( "{!r} command with ID={!r} is no longer used by: {!r}". format( cmd_graph1[cmd_id]["type"], cmd_id, ", ".join(removed), )) if added: logger.error( "{!r} command with ID={!r} is now used by: {!r}". format( cmd_graph1[cmd_id]["type"], cmd_id, ", ".join(added), )) used_by_are_same = False using1 = set(cmd_graph1[cmd_id]["using"]) using2 = set(cmd_graph2[cmd_id]["using"]) if using1 != using2: removed = using1 - using2 added = using2 - using1 if removed: logger.error( "{!r} command with ID={!r} is no longer using: {!r}". format( cmd_graph1[cmd_id]["type"], cmd_id, ", ".join(removed), )) if added: logger.error( "{!r} command with ID={!r} is now using: {!r}".format( cmd_graph1[cmd_id]["type"], cmd_id, ", ".join(added), )) using_are_same = False if used_by_are_same and using_are_same: logger.info("Cmd graphs are the same") else: logger.info("Cmd graphs are different") def compare_src_graphs(self): if not self.__ext_work_dirs_exist("SrcGraph"): return src_graph1 = self.cl1.src_graph src_graph2 = self.cl2.src_graph keys_are_same = True used_by_are_same = True compiled_in_are_same = True files1 = set(src_graph1.keys()) files2 = set(src_graph2.keys()) if files1 == files2: logger.info("Files in the source graph are the same") common_files = files1 else: common_files = files1 & files2 removed = files1 - files2 added = files2 - files1 for file in removed: logger.error( "{!r} file was removed from the source graph".format(file)) for file in added: logger.error( "{!r} file was added to the source graph".format(file)) keys_are_same = False for file in common_files: used_by1 = set(src_graph1[file]["used_by"]) used_by2 = set(src_graph2[file]["used_by"]) if used_by1 != used_by2: removed = used_by1 - used_by2 added = used_by2 - used_by1 if removed: logger.error( "{!r} file is no longer used by {!r} commands".format( file, ", ".join(removed))) if added: logger.error( "{!r} file is now used by {!r} commands".format( file, ", ".join(added))) used_by_are_same = False compiled_in1 = set(src_graph1[file]["compiled_in"]) compiled_in2 = set(src_graph1[file]["compiled_in"]) if compiled_in1 != compiled_in2: removed = compiled_in1 - compiled_in2 added = compiled_in2 - compiled_in1 if removed: logger.error( "{!r} file is no longer compiled in {!r} commands". format(file, ", ".join(removed))) if added: logger.error( "{!r} file is now compiled in {!r} commands".format( file, ", ".join(added))) compiled_in_are_same = False if keys_are_same and used_by_are_same and compiled_in_are_same: logger.info("Source graphs are the same") else: logger.info("Source graphs are different") def compare_functions(self): if not self.__ext_work_dirs_exist("Functions"): return keys_are_same = True types_are_same = True lines_are_same = True signatures_are_same = True decls_are_same = True f1 = self.cl1.functions_by_file f2 = self.cl2.functions_by_file keys1 = set() keys2 = set() for file in f1: for func in f1[file]: keys1.add((file, func)) for file in f2: for func in f2[file]: keys2.add((file, func)) if keys1 == keys2: common_keys = keys2 else: common_keys = keys1 & keys2 removed = keys1 - keys2 added = keys2 - keys1 for file, func in removed: logger.error("{!r} function from {!r} file was removed".format( func, file)) for file, func in added: logger.error("{!r} function from {!r} file was added".format( func, file)) keys_are_same = False for file, func in common_keys: type1 = f1[file][func]["type"] type2 = f2[file][func]["type"] if type1 != type2: logger.error( "{!r} function from {!r} file changed its type from {!r} to {!r}" .format(func, file, type1, type2)) types_are_same = False line1 = f1[file][func]["line"] line2 = f2[file][func]["line"] if line1 != line2: logger.error( "{!r} function from {!r} file changed its definition line from {!r} to {!r}" .format(func, file, line1, line2)) lines_are_same = False signature1 = f1[file][func]["signature"] signature2 = f2[file][func]["signature"] if signature1 != signature2: logger.error( "{!r} function from {!r} file changed its signature from {!r} to {!r}" .format(func, file, signature1, signature2)) signatures_are_same = False if f1[file][func]["declarations"]: decl_files1 = set(f1[file][func]["declarations"].keys()) else: decl_files1 = set() if f2[file][func]["declarations"]: decl_files2 = set(f2[file][func]["declarations"].keys()) else: decl_files2 = set() if decl_files1 == decl_files2: common_decl_files = decl_files2 else: common_decl_files = decl_files1 & decl_files2 removed = decl_files1 - decl_files2 added = decl_files2 - decl_files1 for decl_file in removed: logger.error( "{!r} function from {!r} file no longer has declaration in {!r}" .format(func, file, decl_file)) for decl_file in added: logger.error( "{!r} function from {!r} file has new declaration in {!r}" .format(func, file, decl_file)) decls_are_same = False for decl_file in common_decl_files: decl1 = f1[file][func]["declarations"][decl_file] decl2 = f2[file][func]["declarations"][decl_file] line1 = decl1["line"] line2 = decl2["line"] if line1 != line2: logger.error( "{!r} function from {!r} file changed its declaration line from {!r} to {!r}" .format(func, file, line1, line2)) decls_are_same = False signature1 = decl1["signature"] signature2 = decl2["signature"] if signature1 != signature2: logger.error( "{!r} function from {!r} file changed its declaration signature from {!r} to {!r}" .format(func, file, signature1, signature2)) decls_are_same = False type1 = decl1["type"] type2 = decl2["type"] if type1 != type2: logger.error( "{!r} function from {!r} file changed its declaration type from {!r} to {!r}" .format(func, file, type1, type2)) decls_are_same = False if (keys_are_same and types_are_same and lines_are_same and signatures_are_same and decls_are_same): logger.info("Functions are the same") else: logger.info("Functions are different") def compare_macros(self): if not self.__ext_work_dirs_exist("Macros"): return exp_files_are_same = True exp_names_are_same = True exp_args_are_same = True exp1 = self.cl1.get_macros_expansions() exp2 = self.cl2.get_macros_expansions() exp_files1 = set(exp1) exp_files2 = set(exp2) if exp_files1 == exp_files2: common_exp_files = exp_files2 else: common_exp_files = exp_files1 & exp_files2 removed = exp_files1 - exp_files2 added = exp_files2 - exp_files1 for exp_file in removed: logger.error( "{!r} file was removed from macros expansions".format( exp_file)) for exp_file in added: logger.error("{!r} file was added to macros expansions".format( exp_file)) exp_files_are_same = False for exp_file in common_exp_files: exp_names1 = set(exp1[exp_file]) exp_names2 = set(exp2[exp_file]) if exp_names1 == exp_names2: common_exp_names = exp_names2 else: common_exp_names = exp_names1 & exp_names2 removed = exp_names1 - exp_names2 added = exp_names2 - exp_names1 for exp_name in removed: logger.error( "Expansion of macro {!r} from {!r} file was removed". format(exp_name, exp_file)) for exp_name in added: logger.error( "Expansion of macro {!r} from {!r} file was added". format(exp_name, exp_file)) exp_names_are_same = False for exp_name in common_exp_names: args1 = set([ x for sublist in exp1[exp_file][exp_name]["args"] for x in sublist ]) args2 = set([ x for sublist in exp2[exp_file][exp_name]["args"] for x in sublist ]) if args1 != args2: removed = args1 - args2 added = args2 - args1 for args in removed: logger.error( "Macro {!r} from {!r} file no longer has these expansion args: {!r}" .format(exp_name, exp_file, args)) for args in added: logger.error( "Macro {!r} from {!r} file now has new expansion args: {!r}" .format(exp_name, exp_file, args)) exp_args_are_same = False if exp_files_are_same and exp_names_are_same and exp_args_are_same: logger.info("Macros expansions are the same") else: logger.info("Macros expansions are different") def1 = self.cl1.get_macros_definitions() def2 = self.cl2.get_macros_definitions() def_files_are_same = True def_names_are_same = True def_lines_are_same = True def_files1 = set(def1) def_files2 = set(def2) if def_files1 == def_files2: common_def_files = def_files2 else: common_def_files = def_files1 & def_files2 removed = def_files1 - def_files2 added = def_files2 - def_files1 for def_file in removed: logger.error( "{!r} file was removed from macros definitions".format( def_file)) for def_file in added: logger.error( "{!r} file was added to macros definitions".format( def_file)) def_files_are_same = False for def_file in common_def_files: def_names1 = set(def1[def_file]) def_names2 = set(def1[def_file]) if def_names1 == def_names2: common_def_names = def_names2 else: common_def_names = def_names1 & def_names2 removed = def_names1 - def_names2 added = def_names2 - def_names1 for def_name in removed: logger.error( "Definition of macro {!r} from {!r} file was removed". format(def_name, def_file)) for def_name in added: logger.error( "Definition of macro {!r} from {!r} file was added". format(def_name, def_file)) def_names_are_same = False for def_name in common_def_names: lines1 = def1[def_file][def_name] lines2 = def1[def_file][def_name] if lines1 != lines2: removed = lines1 - lines2 added = lines2 - lines1 for line in removed: logger.error( "Macro {!r} from {!r} file no longer has definition on line {!r}" .format(def_name, def_file, line)) for line in added: logger.error( "Macro {!r} from {!r} file now has definition on line {!r}" .format(def_name, def_file, line)) def_lines_are_same = False if def_files_are_same and def_names_are_same and def_lines_are_same: logger.info("Macros definitions are the same") else: logger.info("Macros definitions are different") def compare_callgraphs(self): if not self.__ext_work_dirs_exist("Callgraph"): return files_are_same = True funcs_are_same = True called_in_files_are_same = True called_in_funcs_are_same = True call_lines_are_same = True match_types_are_same = True c1 = self.cl1.callgraph c2 = self.cl2.callgraph files1 = set(c1) files2 = set(c2) if files1 == files2: common_files = files2 else: common_files = files1 & files2 removed = files1 - files2 added = files2 - files1 for file in removed: logger.error( "{!r} file was removed from the callgraph".format(file)) for file in added: logger.error( "{!r} file was added to the callgraph".format(file)) files_are_same = False for file in common_files: funcs1 = set(c1[file]) funcs2 = set(c2[file]) if funcs1 == funcs2: common_funcs = funcs2 else: common_funcs = funcs1 & funcs2 removed = funcs1 - funcs2 added = funcs2 - funcs1 for func in removed: logger.error( "{!r} function from {!r} file was removed from the callgraph" .format(func, file)) for func in added: logger.error( "{!r} function from {!r} file was added to the callgraph" .format(func, file)) for func in common_funcs: called_in_files1 = set(c1[file][func].get("called_in", [])) called_in_files2 = set(c2[file][func].get("called_in", [])) if called_in_files1 == called_in_files2: common_called_in_files = called_in_files2 else: common_called_in_files = (called_in_files1 & called_in_files2) removed = called_in_files1 - called_in_files2 added = called_in_files2 - called_in_files1 for called_in_file in removed: logger.error( "{!r} function from {!r} file is no longer called in {!r} file" .format(func, file, called_in_file)) for called_in_file in added: logger.error( "{!r} function from {!r} file is now called in {!r} file" .format(func, file, called_in_file)) called_in_files_are_same = False for called_in_file in common_called_in_files: called_in_funcs1 = set( c1[file][func]["called_in"][called_in_file]) called_in_funcs2 = set( c2[file][func]["called_in"][called_in_file]) if called_in_funcs1 == called_in_funcs2: common_called_in_funcs = called_in_funcs2 else: common_called_in_funcs = (called_in_funcs1 & called_in_funcs2) removed = called_in_funcs1 - called_in_funcs2 added = called_in_funcs2 - called_in_funcs1 for called_in_func in removed: logger.error( "{!r} function from {!r} file is no longer called in {!r} func from {!r} file" .format(func, file, called_in_func, called_in_file)) for called_in_func in added: logger.error( "{!r} function from {!r} file is now called in {!r} func from {!r} file" .format(func, file, called_in_func, called_in_file)) called_in_funcs_are_same = False for called_in_func in common_called_in_funcs: call_lines1 = set(c1[file][func]["called_in"] [called_in_file][called_in_func]) call_lines2 = set(c2[file][func]["called_in"] [called_in_file][called_in_func]) if call_lines1 == call_lines2: common_call_lines = call_lines2 else: common_call_lines = call_lines1 & call_lines2 removed = call_lines1 - call_lines2 added = call_lines2 - call_lines1 for call_line in removed: logger.error( "{!r} function from {!r} file is no longer called in {!r} func from {!r} file on line {!r}" .format( func, file, called_in_func, called_in_file, call_line, )) for call_line in added: logger.error( "{!r} function from {!r} file is now called in {!r} func from {!r} file on line {!r}" .format( func, file, called_in_func, called_in_file, call_line, )) call_lines_are_same = False if common_call_lines: call_line = list(common_call_lines)[0] match_type1 = c1[file][func]["called_in"][ called_in_file][called_in_func][call_line][ "match_type"] match_type2 = c2[file][func]["called_in"][ called_in_file][called_in_func][call_line][ "match_type"] if match_type1 != match_type2: logger.error( "Match type of {!r} ({!r}) call in {!r} ({!r}) was changed from {!r} to {!r}" .format( func, file, called_in_func, called_in_file, match_type1, match_type2, )) match_types_are_same = False if (files_are_same and funcs_are_same and called_in_files_are_same and called_in_funcs_are_same and call_lines_are_same and match_types_are_same): logger.info("Callgraphs are the same") else: logger.info("Callgraphs are different") @staticmethod def __get_extension_list(work_dir): return [ f for f in os.listdir(work_dir) if os.path.isdir(os.path.join(work_dir, f)) ] def __ext_work_dirs_exist(self, ext_name): a = os.path.join(self.work_dir1, ext_name) b = os.path.join(self.work_dir2, ext_name) if os.path.exists(a) and os.path.exists(b): return True else: return False