def get_excluded_directories(dirname): assert isinstance(dirname, str) if not os.path.exists(dirname): raise ICE("%s does not exist" % dirname) elif not os.path.isdir(dirname): raise ICE("%s is not a directory" % dirname) canonical_name = os.path.abspath(dirname) if canonical_name not in tree: raise ICE("%s was not registered" % dirname) return tree[canonical_name].excluded_children
def get_config(filename): assert isinstance(filename, str) if not os.path.exists(filename): raise ICE("%s does not exist" % filename) elif not os.path.isfile(filename): raise ICE("%s is not a file" % filename) dirname = os.path.dirname(os.path.abspath(filename)) if dirname not in tree: raise ICE("%s was not registered" % dirname) return tree[dirname].config
def register_item(mh, name, options): assert isinstance(mh, Message_Handler) assert isinstance(name, str) if not os.path.exists(name): raise ICE("%s does not exist" % name) elif os.path.isfile(name): register_dir(mh, options, os.path.dirname(os.path.abspath(name)), False) elif os.path.isdir(name): register_dir(mh, options, os.path.abspath(name), True) else: raise ICE("%s is neither a file or directory")
def parse_block_subsystem(self, et_block): assert isinstance(et_block, ET.Element) assert et_block.tag == "Block" assert et_block.attrib["BlockType"] == "SubSystem" n_system = None system_name = et_block.attrib["Name"] props = self.parse_p_tags(et_block) if props.get("SFBlockType", None) == "MATLAB Function": # This could be a special "MATLAB Function" sub-system, in # which case we special case n_block = self.parse_block_matlab_function(et_block, props) else: # This is a normal sub-system. First we're finding the system # nested. for et_item in et_block: if et_item.tag == "System": if n_system: raise ICE("multiple systems in subsytem") self.name_stack.append(system_name) n_system = self.parse_system(et_item) self.name_stack.pop() # Then we build a sub-system block referencing the system n_block = Sub_System(et_block.attrib["SID"], system_name, n_system) return n_block
def parse_machine(self, et_machine): assert isinstance(et_machine, ET.Element) assert et_machine.tag == "machine" d_machine = {} # We expect to have a single tag called 'Children'. et_children = None for et_item in et_machine: if et_item.tag == "P": pass elif et_item.tag == "Children": if et_children: raise ICE("multiple children in Stateflow machine") et_children = et_item elif et_item.tag == "debug": # TODO: Unclear what the presence of this does. To # investigate. pass else: self.mh.error(self.loc(), "Unknown item %s in machine" % et_item.tag) for et_item in et_children: if et_item.tag == "target": # TODO: Unclear what exactly this does right now. pass elif et_item.tag == "chart": chart_id = int(et_item.attrib["id"]) d_machine[chart_id] = et_item else: self.mh.error(self.loc(), "Unknown item %s in Children" % et_item.tag) return d_machine
def function_length(node): assert isinstance(node, (Function_Definition, Script_File)) if isinstance(node, Script_File): return None elif node.t_end: return node.t_end.location.line - node.t_fun.location.line + 1 else: # If a function does not have an end, its length is from this # function up to and including the next function or the end of # the file n_cu = node.n_parent if not isinstance(n_cu, Function_File): raise ICE("unterminated function must be a child of a " "function file") this_function_idx = n_cu.l_functions.index(node) next_function_idx = this_function_idx + 1 if next_function_idx < len(n_cu.l_functions): # We have a function following this one return (n_cu.l_functions[next_function_idx].t_fun.location.line - node.t_fun.location.line) else: # This is the last function in the file return n_cu.file_length - node.t_fun.location.line + 1
def get_named_vertex(self, name): assert isinstance(name, str) if name not in self.names: raise ICE("attempted to get named vertex %s that" " does not exists" % name) return self.names[name]
def parse_block_matlab_function(self, et_block, props): assert isinstance(et_block, ET.Element) assert isinstance(props, dict) assert et_block.tag == "Block" assert et_block.attrib["BlockType"] == "SubSystem" assert props.get("SFBlockType", None) == "MATLAB Function" assert not self.is_external_harness # To find the program text for this, we need to look into the # Stateflow XML. First we construct a name by chaining # together all system names (except the top-level one) with /. sf_base_name = "/".join(self.name_stack + [et_block.attrib["Name"]]) # Get Stateflow chart if sf_base_name in self.sf_names: et_chart = self.sf_names[sf_base_name] else: self.mh.error(self.loc(), "Could not find %s in Stateflow" % sf_base_name) # Quickly rifle through Children to find the relevant eml # block. This could be more structured, and should be if we do # more stateflow support in the future. is_eml = False et_script = None for et_eml in et_chart.find("Children").iter("eml"): for et_item in et_eml: if et_item.tag == "P": if et_item.attrib["Name"] == "isEML": is_eml = bool(et_item.text) elif et_item.attrib["Name"] == "script": if et_script: raise ICE("multiple scripts in eml tag") et_script = et_item else: self.mh.error(self.loc(), "Unexpected child %s of eml" % et_item.tag) if et_script and not is_eml: raise ICE("script found, but isEML is not true") if et_script is None: raise ICE("referenced script for %s not found" % sf_base_name) return Matlab_Function(et_block.attrib["SID"], et_block.attrib["Name"], SLX_Reference(et_script))
def parse_style_configuration(self): self.match("IDENTIFIER") config_name = self.ct.value if config_name not in STYLE_CONFIGURATION: msg = "expected valid style configuration name" suggestions = difflib.get_close_matches(config_name, list(STYLE_CONFIGURATION) + KEYWORDS, n=1) if suggestions: msg += " (did you mean %s?)" % suggestions[0] self.mh.error(self.ct.location, msg) self.match("COLON") cfg_item = STYLE_CONFIGURATION[config_name] value = None if isinstance(cfg_item, Integer_Style_Configuration): value = self.parse_integer_number() if cfg_item.lower_limit is not None: if value < cfg_item.lower_limit: self.mh.error( self.ct.location, "%s must be at least %i" % (config_name, cfg_item.lower_limit)) if cfg_item.upper_limit is not None: if value > cfg_item.upper_limit: self.mh.error( self.ct.location, "%s can be at most %i" % (config_name, cfg_item.upper_limit)) elif isinstance(cfg_item, Regex_Style_Configuration): value = self.parse_regex() elif isinstance(cfg_item, Encoding_Style_Configuration): value = self.parse_encoding() elif isinstance(cfg_item, Set_Style_Configuration): value = self.parse_string() elif isinstance(cfg_item, String_Style_Configuration): value = self.parse_string() if isinstance(cfg_item, Choice_Style_Configuration): if value not in cfg_item.choices: self.mh.error( self.ct.location, "must be one of %s" % " or ".join(cfg_item.choices)) elif isinstance(cfg_item, Boolean_Style_Configuration): value = self.parse_boolean() else: raise ICE("unexpected cfg kind %s" % cfg_item.__class__.__name__) return Style_Configuration(config_name, value)
def get_root(name): assert isinstance(name, str) dirname = os.path.abspath(name) if dirname not in tree: raise ICE("%s was not registered" % dirname) while not tree[dirname].project_root: dirname = os.path.dirname(dirname) return dirname
def parse_docstrings(mh, cfg, parse_tree, tbuf): assert isinstance(mh, Message_Handler) assert isinstance(parse_tree, Compilation_Unit) assert isinstance(tbuf, Token_Buffer) approaching_docstring = False in_docstring = tbuf.tokens and tbuf.tokens[0].kind == "COMMENT" if in_docstring: ast_node = Docstring(cfg.style_config["copyright_regex"]) parse_tree.set_docstring(ast_node) else: ast_node = None for token in tbuf.tokens: # Recognise function docstrings if approaching_docstring: if token.first_in_statement: if token.kind == "COMMENT": in_docstring = True approaching_docstring = False elif in_docstring: if token.kind == "COMMENT": ast_node.add_comment(token) elif token.kind == "NEWLINE" and token.value.count("\n") == 1: pass else: in_docstring = False if token.kind == "KEYWORD" and token.value in ("function", "classdef"): # Recognise function docstrings approaching_docstring = True if token.ast_link is None: raise ICE("keyword is not linked to AST") elif not isinstance(token.ast_link, Definition): raise ICE("AST link is %s and not a Definition" % token.ast_link.__class__.__name__) ast_node = Docstring(cfg.style_config["copyright_regex"]) token.ast_link.set_docstring(ast_node)
def npath(node): assert isinstance(node, (Sequence_Of_Statements, Statement, Pragma, Function_Definition, Script_File)) if isinstance(node, Function_Definition): return npath(node.n_body) elif isinstance(node, Script_File): return npath(node.n_statements) elif isinstance(node, Sequence_Of_Statements): paths = 1 for n_statement in node.l_statements: if isinstance(n_statement, (If_Statement, For_Loop_Statement, Switch_Statement, Try_Statement, While_Statement)): paths *= npath(n_statement) return paths elif isinstance(node, If_Statement): paths = 0 for n_action in node.l_actions: paths += npath(n_action.n_body) if not node.has_else: paths += 1 return paths elif isinstance(node, Switch_Statement): paths = 0 for n_action in node.l_actions: paths += npath(n_action.n_body) if not node.has_otherwise: paths += 1 return paths elif isinstance(node, (For_Loop_Statement, While_Statement)): return 1 + npath(node.n_body) elif isinstance(node, Try_Statement): return npath(node.n_body) * 2 else: raise ICE("unexpected node %s" % node.__class__.__name__)
def sem_compilation_unit(self, n_cu): assert isinstance(n_cu, Compilation_Unit) n_cu.scope = self.scope if isinstance(n_cu, Class_File): self.sem_class_file(n_cu) elif isinstance(n_cu, Function_File): pass elif isinstance(n_cu, Script_File): pass else: raise ICE("unexpected compilation unit type %s" % n_cu.__class__.__name__)
def rec(root): is_leaf = True for subclass in root.__subclasses__(): rec(subclass) is_leaf = False if is_leaf: if issubclass(root, Style_Rule_File): rules["on_file"].append(root) elif issubclass(root, Style_Rule_Line): rules["on_line"].append(root) else: raise ICE( "Unable to categorize %s with base %s" % (root.__name__, " and ".join(b.__name__ for b in root.__bases__)))
def parse_model(self, et_model): assert isinstance(et_model, ET.Element) assert et_model.tag == "Model" n_model = Model(self.filename) n_system = None for et_item in et_model: if et_item.tag == "System": if n_system: raise ICE("multiple systems in Model") n_system = self.parse_system(et_item) n_model.set_system(n_system) return n_model
def parse_library(self, et_library): assert isinstance(et_library, ET.Element) assert et_library.tag == "Library" n_library = Library(self.filename) n_system = None for et_item in et_library: if et_item.tag == "System": if n_system: raise ICE("multiple systems in Library") n_system = self.parse_system(et_item) n_library.set_system(n_system) return n_library
def cnest(node): assert isinstance(node, (Sequence_Of_Statements, Statement, Pragma, Function_Definition, Script_File)) if isinstance(node, Function_Definition): return cnest(node.n_body) elif isinstance(node, Script_File): return cnest(node.n_statements) elif isinstance(node, Sequence_Of_Statements): return max(map(cnest, node.l_statements), default=0) elif isinstance(node, (Simple_Statement, Pragma)): return 0 elif isinstance(node, SPMD_Statement): return cnest(node.n_body) elif isinstance(node, If_Statement): return 1 + max((cnest(a.n_body) for a in node.l_actions), default=0) elif isinstance(node, Switch_Statement): return 1 + max((cnest(a.n_body) for a in node.l_actions), default=0) elif isinstance(node, (For_Loop_Statement, While_Statement)): return 1 + cnest(node.n_body) elif isinstance(node, Try_Statement): if node.n_handler: return 1 + max(cnest(node.n_body), cnest(node.n_handler)) else: return 1 + cnest(node.n_body) else: raise ICE("unexpected node %s" % node.__class__.__name__)
def __init__(self, graph, name=None): assert isinstance(graph, Graph) assert name is None or isinstance(name, str) self.graph = graph self.uid = graph.get_next_uid() self.out_edges = set() self.in_edges = set() self.name = name # Register vertex with host graph self.graph.vertices.add(self) # Register named vertex if name: if name in graph.names: raise ICE("attempted to create named vertex %s that" " already exists" % name) self.graph.names[name] = self
def evaluate(self, mh, config): assert isinstance(mh, Message_Handler) assert isinstance(config, Config) if isinstance(self.the_config, Integer_Style_Configuration): config.style_config[self.config_name] = self.value elif isinstance(self.the_config, Boolean_Style_Configuration): config.style_config[self.config_name] = self.value elif isinstance(self.the_config, String_Style_Configuration): # Also covers regex configuration config.style_config[self.config_name] = self.value elif isinstance(self.the_config, Set_Style_Configuration): config.style_config[self.config_name].add(self.value) else: raise ICE("unexpected config kind %s" % self.the_config.__class__.__name__)
def worst_offender_count(worst_offenders): for metric in worst_offenders: return len(worst_offenders[metric]) raise ICE("cannot determine length of wo table")
def process_wp(cls, wp): # Load file content content = wp.get_content() is_embedded = isinstance(wp, work_package.Embedded_MATLAB_WP) # Deal with command-line options primary_entity = wp.cfg.style_config["copyright_primary_entity"] if wp.options.primary_entity: primary_entity = wp.options.primary_entity old_entity = None if wp.options.update_year: action = "update_year" elif wp.options.merge: action = "merge" elif wp.options.change_entity is not None: action = "change_entity" old_entity = wp.options.change_entity elif wp.options.add_notice: action = "add_notice" else: raise ICE("none of the action are set") # Create lexer lexer = MATLAB_Lexer(wp.mh, content, wp.filename, wp.blockname) if wp.cfg.octave: lexer.set_octave_mode() if not wp.cfg.pragmas: # pragma: no cover lexer.process_pragmas = False # We're dealing with an empty file here. Lets just not do anything if len(lexer.text.strip()) == 0: return MH_Copyright_Result(wp, False) # Determine primary copyright holder. We use the command-line, # or try and deduce from the config files. if not primary_entity: if len(wp.cfg.style_config["copyright_entity"]) == 1: primary_entity = \ list(wp.cfg.style_config["copyright_entity"])[0] else: wp.mh.warning(lexer.get_file_loc(), "unable to determine primary copyright entity," " skipping this file") return MH_Copyright_Result(wp, False) # Compile magic regex cr_regex = re.compile(wp.cfg.style_config["copyright_regex"]) # Process tokens and take actions try: # Determine where the actual file content starts, and a # list of all copyright info. in_copyright_notice = ( wp.cfg.active("copyright_notice") and (not is_embedded or wp.cfg.style_config["copyright_in_embedded_code"])) old_copyright_info = [] file_start = 1 newlines = 1 comment_char = "%" while in_copyright_notice: token = lexer.token() if token is None: in_copyright_notice = False elif token.kind == "COMMENT": match = cr_regex.search(token.value) if match: if not token.raw_text.startswith("%"): comment_char = token.raw_text[0] old_copyright_info.append( (int(match.group("ystart")) if match.group("ystart") else None, int(match.group("yend")), match.group("org"), token)) file_start = token.location.line + 1 else: file_start = token.location.line in_copyright_notice = False elif token.kind == "NEWLINE": newlines = len(token.value) else: # Once we get a non-comment/non-copyright token, # the header has ended. We then emit messages if # we could not find anything. in_copyright_notice = False file_start = token.location.line # Update notices as instructed copyright_info = [] action_taken = False merged_copyright = [None, None, primary_entity] original_primary = None for ystart, yend, org, token in old_copyright_info: if ystart is not None and ystart > yend: wp.mh.error(token.location, "initial year is later than end year") if action == "update_year": if org == primary_entity: if yend < wp.options.year: if ystart is None: ystart = yend yend = wp.options.year action_taken = True elif yend > wp.options.year: wp.mh.error(token.location, "end year is later than %u" % wp.options.year) if ystart is not None: if ystart == yend: ystart = None action_taken = True copyright_info.append((ystart, yend, org)) elif action == "change_entity": if org == old_entity: org = primary_entity action_taken = True copyright_info.append((ystart, yend, org)) elif action == "merge": if org in wp.cfg.style_config["copyright_entity"] or \ org == primary_entity: # Actions are definitely taken if we have multiple # primaries, or we have a non-primary entity if org == primary_entity: if original_primary is None: original_primary = (ystart, yend, org) else: action_taken = True else: action_taken = True # Update the merged entity if merged_copyright[0] is None: merged_copyright[0] = ystart elif ystart is not None: merged_copyright[0] = min(merged_copyright[0], ystart) if merged_copyright[0] is None: merged_copyright[0] = yend else: merged_copyright[0] = min(merged_copyright[0], yend) if merged_copyright[1] is None: merged_copyright[1] = yend else: # yend is always not None merged_copyright[1] = max(merged_copyright[1], yend) else: copyright_info.append((ystart, yend, org)) elif action == "add_notice": copyright_info.append((ystart, yend, org)) else: raise ICE("unexpected action %s" % action) # Final updates for the merge and add action if action == "merge" and merged_copyright[1] is not None: # Clean merged copyright if merged_copyright[0] == merged_copyright[1]: merged_copyright[0] = None # Add new entry and sort list chronologically copyright_info.append(tuple(merged_copyright)) copyright_info.sort(key=year_span_key) # Check if we'd actually change anything if not action_taken and \ tuple(merged_copyright) != original_primary: action_taken = True elif action == "add_notice": if len(copyright_info) == 0: copyright_info.append((None, wp.options.year, primary_entity)) action_taken = True # Re-build file header if action_taken: new_content = [] for ystart, yend, org in copyright_info: sub = {"ystart" : ystart, "yend" : yend, "org" : org} if ystart is None: new_content.append("%s %s" % (comment_char, wp.options.template % sub)) else: new_content.append("%s %s" % (comment_char, wp.options.template_range % sub)) if newlines >= 2 or action == "add_notice": new_content.append("") new_content += lexer.context_line[file_start - 1:] wp.write_modified("\n".join(new_content) + "\n") return MH_Copyright_Result(wp, True) else: wp.mh.info(lexer.get_file_loc(), "no action taken") return MH_Copyright_Result(wp, False) except Error: # If there are any errors, we can stop here return MH_Copyright_Result(wp, False)
def sem_pass_1(mh, entrypoint, n_cu): assert isinstance(mh, Message_Handler) assert isinstance(n_cu, Compilation_Unit) assert isinstance(entrypoint, (Library_Declaration, Entrypoint_Declaration)) or \ entrypoint is None rv = Semantic_Analysis_Pass_1(mh) rv.sem_compilation_unit(n_cu) item = os.path.normpath(n_cu.dirname) if entrypoint: best_match = None for path in cfg_tree.get_path(entrypoint): search_item = os.path.normpath(path) if item.startswith(search_item): if best_match is None or len(best_match) < len(search_item): best_match = search_item if best_match is None: raise ICE("could not find %s on path" % n_cu.dirname) packages = item[len(best_match) + 1:] if os.sep != "/": packages = packages.replace(os.sep, "/") if packages: packages = packages.split("/") rv.pkg = packages else: packages = [] # We have an empty list, or a list of + directories, followed # by at most one @ directory, followed optionally by a private # directory. current_dir = best_match sequence = "package" for item in packages: current_dir = os.path.join(current_dir, item) if item.startswith("+"): if sequence == "package": pass else: mh.check(n_cu.loc(), "cannot nest package inside a %s" % sequence) return None elif item.startswith("@"): if sequence == "package": sequence = "class directory" else: mh.check( n_cu.loc(), "cannot nest class directory inside a %s" % sequence) return None elif item == "private": if sequence in ("package", "class directory"): sequence = "private directory" else: mh.check(n_cu.loc(), "cannot private directory inside a %s" % sequence) return None else: mh.check(n_cu.loc(), "is not on path and cannot be accessed" % sequence) return None return rv
def write_modified(self, content): raise ICE("logic error - must not be called for SL File WP")
def evaluate(self, mh, config): raise ICE("logic error - called evaluate() for project_root")
def evaluate(self, mh, config): raise ICE("logic error - called evaluate() for project directive")
def get_content(self): raise ICE("somhow called root class method")
def evaluate(self, mh, config): raise ICE("logic error - called evaluate() for exclude_dir")
def build_cfg_statement(graph, n_statement): assert isinstance(graph, Graph) assert isinstance(n_statement, (Statement, Metric_Justification_Pragma)) ctx = CFG_Context(Vertex(graph, n_statement)) if isinstance( n_statement, (Compound_Assignment_Statement, Global_Statement, Import_Statement, Naked_Expression_Statement, Persistent_Statement, Simple_Assignment_Statement, Metric_Justification_Pragma)): # All of these are simple statements. One entry, one obvious # exit. ctx.l_exits.append(ctx.v_entry) elif isinstance(n_statement, If_Statement): # If statements chain together the actions. current_link = ctx.v_entry for n_action in n_statement.l_actions: v_action = Vertex(graph, n_action) graph.add_edge(current_link, v_action) current_link = v_action action_ctx = build_cfg_sos(graph, n_action.n_body) ctx.merge_loops(action_ctx) ctx.merge_exits(action_ctx) if action_ctx.v_entry: graph.add_edge(v_action, action_ctx.v_entry) if not n_statement.has_else: # Add an implicit path for else, if not present ctx.l_exits.append(current_link) elif isinstance(n_statement, Switch_Statement): # Case statements in MATLAB are chained, because there is no # semantic check that the options do not overlap; hence they # must be evaluated in sequence. This is the same as above. In # the future, for the MISS_HIT subset, we could make sure # there is no overlap. current_link = ctx.v_entry for n_action in n_statement.l_actions: v_action = Vertex(graph, n_action) graph.add_edge(current_link, v_action) current_link = v_action action_ctx = build_cfg_sos(graph, n_action.n_body) ctx.merge_loops(action_ctx) ctx.merge_exits(action_ctx) if action_ctx.v_entry: graph.add_edge(v_action, action_ctx.v_entry) if not n_statement.has_otherwise: # Add an implicit path for otherwise, if not present ctx.l_exits.append(current_link) elif isinstance(n_statement, (For_Loop_Statement, While_Statement)): # Loops add a loop back to the loop statement. There are two # paths out of the loop statement: one into the loop and one # to the next statement. # # Any break statements found in the body are processed here. body_ctx = build_cfg_sos(graph, n_statement.n_body) if body_ctx.v_entry: graph.add_edge(ctx.v_entry, body_ctx.v_entry) for src in body_ctx.l_exits: graph.add_edge(src, ctx.v_entry) for src in body_ctx.l_loop_continues: graph.add_edge(src, ctx.v_entry) ctx.l_exits = [ctx.v_entry] + ctx.l_loop_breaks elif isinstance(n_statement, Break_Statement): ctx.l_loop_breaks.append(ctx.v_entry) elif isinstance(n_statement, Continue_Statement): ctx.l_loop_continues.append(ctx.v_entry) elif isinstance(n_statement, Return_Statement): graph.add_edge(ctx.v_entry, graph.get_named_vertex("end")) elif isinstance(n_statement, SPMD_Statement): spmd_ctx = build_cfg_sos(graph, n_statement.n_body) ctx.merge_loops(spmd_ctx) ctx.merge_exits(spmd_ctx) if spmd_ctx.v_entry: graph.add_edge(ctx.v_entry, spmd_ctx.v_entry) elif isinstance(n_statement, Try_Statement): try_ctx = build_cfg_sos(graph, n_statement.n_body) ctx.merge_loops(try_ctx) ctx.merge_exits(try_ctx) if n_statement.n_handler: handler_ctx = build_cfg_sos(graph, n_statement.n_handler) ctx.merge_loops(handler_ctx) ctx.merge_exits(handler_ctx) if handler_ctx.v_entry: graph.add_edge(ctx.v_entry, handler_ctx.v_entry) else: ctx.l_exits.append(ctx.v_entry) if try_ctx.v_entry: graph.add_edge(ctx.v_entry, try_ctx.v_entry) else: raise ICE("unknown statement kind %s" % n_statement.__class__.__name__) return ctx
def dump(self): raise ICE("dump is not defined")
def write_modified(self, content): raise ICE("somhow called root class method")