def match_control_line(self): match = self.match( r"(?<=^)[\t ]*(%(?!%)|##)[\t ]*((?:(?:\\r?\n)|[^\r\n])*)" r"(?:\r?\n|\Z)", re.M) if match: operator = match.group(1) text = match.group(2) if operator == '%': m2 = re.match(r'(end)?(\w+)\s*(.*)', text) if not m2: raise exceptions.SyntaxException( "Invalid control line: '%s'" % text, **self.exception_kwargs) isend, keyword = m2.group(1, 2) isend = (isend is not None) if isend: if not len(self.control_line): raise exceptions.SyntaxException( "No starting keyword '%s' for '%s'" % (keyword, text), **self.exception_kwargs) elif self.control_line[-1].keyword != keyword: raise exceptions.SyntaxException( "Keyword '%s' doesn't match keyword '%s'" % (text, self.control_line[-1].keyword), **self.exception_kwargs) self.append_node(parsetree.ControlLine, keyword, isend, text) else: self.append_node(parsetree.Comment, text) return True else: return False
def parse_until_text(self, *text): startpos = self.match_position while True: match = self.match(r'#.*\n') if match: continue match = self.match(r'(\"\"\"|\'\'\'|\"|\')') if match: m = self.match(r'.*?%s' % match.group(1), re.S) if not m: raise exceptions.SyntaxException( "Unmatched '%s'" % match.group(1), **self.exception_kwargs) else: match = self.match(r'(%s)' % r'|'.join(text)) if match: return (self.text[startpos:self.match_position - len(match.group(1))], match.group(1)) else: match = self.match(r".*?(?=\"|\'|#|%s)" % r'|'.join(text), re.S) if not match: raise exceptions.SyntaxException( "Expected: %s" % ','.join(text), **self.exception_kwargs)
def parse(self): self.encoding, self.text = self.decode_raw_stream( self.text, True, self.encoding, self.filename ) for preproc in self.preprocessor: self.text = preproc(self.text) # push the match marker past the # encoding comment. self.match_reg(self._coding_re) self.textlength = len(self.text) while True: if self.match_position > self.textlength: break if self.match_end(): break if self.match_expression(): continue if self.match_control_line(): continue if self.match_comment(): continue if self.match_tag_start(): continue if self.match_tag_end(): continue if self.match_python_block(): continue if self.match_text(): continue if self.match_position > self.textlength: break # TODO: no coverage here raise exceptions.MakoException("assertion failed") if len(self.tag): raise exceptions.SyntaxException( "Unclosed tag: <%%%s>" % self.tag[-1].keyword, **self.exception_kwargs, ) if len(self.control_line): raise exceptions.SyntaxException( "Unterminated control keyword: '%s'" % self.control_line[-1].keyword, self.text, self.control_line[-1].lineno, self.control_line[-1].pos, self.filename, ) return self.template
def match_tag_end(self): match = self.match(r'\</%[\t ]*(.+?)[\t ]*>') if match: if not len(self.tag): raise exceptions.SyntaxException("Closing tag without opening tag: </%%%s>" % match.group(1), **self.exception_kwargs) elif self.tag[-1].keyword != match.group(1): raise exceptions.SyntaxException("Closing tag </%%%s> does not match tag: <%%%s>" % (match.group(1), self.tag[-1].keyword), **self.exception_kwargs) self.tag.pop() return True else: return False
def parse_until_text(self, watch_nesting, *text): startpos = self.match_position text_re = r"|".join(text) brace_level = 0 paren_level = 0 bracket_level = 0 while True: match = self.match(r"#.*\n") if match: continue match = self.match(r"(\"\"\"|\'\'\'|\"|\')[^\\]*?(\\.[^\\]*?)*\1", re.S) if match: continue match = self.match(r"(%s)" % text_re) if match and not (watch_nesting and (brace_level > 0 or paren_level > 0 or bracket_level > 0)): return ( self.text[startpos:self.match_position - len(match.group(1))], match.group(1), ) elif not match: match = self.match(r"(.*?)(?=\"|\'|#|%s)" % text_re, re.S) if match: brace_level += match.group(1).count("{") brace_level -= match.group(1).count("}") paren_level += match.group(1).count("(") paren_level -= match.group(1).count(")") bracket_level += match.group(1).count("[") bracket_level -= match.group(1).count("]") continue raise exceptions.SyntaxException("Expected: %s" % ",".join(text), **self.exception_kwargs)
def parse_until_text(self, *text): startpos = self.match_position text_re = r'|'.join(text) brace_level = 0 while True: match = self.match(r'#.*\n') if match: continue match = self.match(r'(\"\"\"|\'\'\'|\"|\')((?<!\\)\\\1|.)*?\1', re.S) if match: continue match = self.match(r'(%s)' % text_re) if match: if match.group(1) == '}' and brace_level > 0: brace_level -= 1 continue return \ self.text[startpos:\ self.match_position-len(match.group(1))],\ match.group(1) match = self.match(r"(.*?)(?=\"|\'|#|%s)" % text_re, re.S) if match: brace_level += match.group(1).count('{') brace_level -= match.group(1).count('}') continue raise exceptions.SyntaxException("Expected: %s" % ','.join(text), **self.exception_kwargs)
def parse(code, mode, **exception_kwargs): try: return compiler_parse(code, mode) except SyntaxError, e: raise exceptions.SyntaxException( "(%s) %s (%s)" % (e.__class__.__name__, str(e), repr(code[0:50])), **exception_kwargs)
def append_node(self, nodecls, *args, **kwargs): kwargs.setdefault('source', self.text) kwargs.setdefault('lineno', self.matched_lineno) kwargs.setdefault('pos', self.matched_charpos) kwargs['filename'] = self.filename node = nodecls(*args, **kwargs) if len(self.tag): self.tag[-1].nodes.append(node) else: self.template.nodes.append(node) if isinstance(node, parsetree.Tag): if len(self.tag): node.parent = self.tag[-1] self.tag.append(node) elif isinstance(node, parsetree.ControlLine): if node.isend: self.control_line.pop() elif node.is_primary: self.control_line.append(node) elif len(self.control_line ) and not self.control_line[-1].is_ternary(node.keyword): raise exceptions.SyntaxException( "Keyword '%s' not a legal ternary for keyword '%s'" % (node.keyword, self.control_line[-1].keyword), **self.exception_kwargs)
def parse_until_text(self, watch_nesting, *text): startpos = self.match_position text_re = r'|'.join(text) brace_level = 0 paren_level = 0 bracket_level = 0 while True: match = self.match(r'#.*\n') if match: continue match = self.match(r'(\"\"\"|\'\'\'|\"|\')[^\\]*?(\\.[^\\]*?)*\1', re.S) if match: continue match = self.match(r'(%s)' % text_re) if match and not (watch_nesting and (brace_level > 0 or paren_level > 0 or bracket_level > 0)): return \ self.text[startpos: self.match_position - len(match.group(1))],\ match.group(1) elif not match: match = self.match(r"(.*?)(?=\"|\'|#|%s)" % text_re, re.S) if match: brace_level += match.group(1).count('{') brace_level -= match.group(1).count('}') paren_level += match.group(1).count('(') paren_level -= match.group(1).count(')') bracket_level += match.group(1).count('[') bracket_level -= match.group(1).count(']') continue raise exceptions.SyntaxException("Expected: %s" % ','.join(text), **self.exception_kwargs)
def parse(code, mode='exec', **exception_kwargs): """Parse an expression into AST""" try: return _ast_util.parse(code, '<unknown>', mode) except Exception: raise exceptions.SyntaxException( "(%s) %s (%r)" % (compat.exception_as().__class__.__name__, compat.exception_as(), code[0:50]), **exception_kwargs)
def parse(code, mode='exec', **exception_kwargs): """Parse an expression into AST""" try: if _ast: return _ast_util.parse(code, '<unknown>', mode) else: return compiler_parse(code, mode) except Exception, e: raise exceptions.SyntaxException( "(%s) %s (%s)" % (e.__class__.__name__, str(e), repr(code[0:50])), **exception_kwargs)
def parse(code, mode='exec', **exception_kwargs): """Parse an expression into AST""" try: if _ast: return _ast_util.parse(code, '<unknown>', mode) else: if isinstance(code, unicode): code = code.encode('ascii', 'backslashreplace') return compiler_parse(code, mode) except Exception, e: raise exceptions.SyntaxException( "(%s) %s (%r)" % (e.__class__.__name__, e, code[0:50]), **exception_kwargs)
def append_node(self, nodecls, *args, **kwargs): kwargs.setdefault("source", self.text) kwargs.setdefault("lineno", self.matched_lineno) kwargs.setdefault("pos", self.matched_charpos) kwargs["filename"] = self.filename node = nodecls(*args, **kwargs) if len(self.tag): self.tag[-1].nodes.append(node) else: self.template.nodes.append(node) # build a set of child nodes for the control line # (used for loop variable detection) # also build a set of child nodes on ternary control lines # (used for determining if a pass needs to be auto-inserted if self.control_line: control_frame = self.control_line[-1] control_frame.nodes.append(node) if ( not ( isinstance(node, parsetree.ControlLine) and control_frame.is_ternary(node.keyword) ) and self.ternary_stack and self.ternary_stack[-1] ): self.ternary_stack[-1][-1].nodes.append(node) if isinstance(node, parsetree.Tag): if len(self.tag): node.parent = self.tag[-1] self.tag.append(node) elif isinstance(node, parsetree.ControlLine): if node.isend: self.control_line.pop() self.ternary_stack.pop() elif node.is_primary: self.control_line.append(node) self.ternary_stack.append([]) elif self.control_line and self.control_line[-1].is_ternary( node.keyword ): self.ternary_stack[-1].append(node) elif self.control_line and not self.control_line[-1].is_ternary( node.keyword ): raise exceptions.SyntaxException( "Keyword '%s' not a legal ternary for keyword '%s'" % (node.keyword, self.control_line[-1].keyword), **self.exception_kwargs, )
def parse(code, mode="exec", **exception_kwargs): """Parse an expression into AST""" try: return _ast_util.parse(code, "<unknown>", mode) except Exception as e: raise exceptions.SyntaxException( "(%s) %s (%r)" % ( compat.exception_as().__class__.__name__, compat.exception_as(), code[0:50], ), **exception_kwargs, ) from e
def parse(code, mode='exec', **exception_kwargs): """Parse an expression into AST""" try: return _ast_util.parse(code, '<unknown>', mode) except Exception: import sys, traceback ################## Added by Christopher Sebastian print >> sys.stderr, '\nOriginal Exception:' ################## Added by Christopher Sebastian traceback.print_exc() ################## Added by Christopher Sebastian print >> sys.stderr ################## Added by Christopher Sebastian raise exceptions.SyntaxException( "(%s) %s :\n%s" % ( ################## Edited by Christopher Sebastian compat.exception_as().__class__.__name__, compat.exception_as(), (u'\n'.join(['%03d: %s'%(i+1,l) for i,l in enumerate(code.splitlines())])).encode('ascii', 'backslashreplace') ################## Edited by Christopher Sebastian ), **exception_kwargs)
def match_tag_start(self): match = self.match( r""" \<% # opening tag ([\w\.\:]+) # keyword ((?:\s+\w+|\s*=\s*|".*?"|'.*?')*) # attrname, = \ # sign, string expression \s* # more whitespace (/)?> # closing """, re.I | re.S | re.X, ) if match: keyword, attr, isend = match.groups() self.keyword = keyword attributes = {} if attr: for att in re.findall( r"\s*(\w+)\s*=\s*(?:'([^']*)'|\"([^\"]*)\")", attr ): key, val1, val2 = att text = val1 or val2 text = text.replace("\r\n", "\n") attributes[key] = text self.append_node(parsetree.Tag, keyword, attributes) if isend: self.tag.pop() else: if keyword == "text": match = self.match(r"(.*?)(?=\</%text>)", re.S) if not match: raise exceptions.SyntaxException( "Unclosed tag: <%%%s>" % self.tag[-1].keyword, **self.exception_kwargs ) self.append_node(parsetree.Text, match.group(1)) return self.match_tag_end() return True else: return False
def match_tag_start(self): match = self.match( r''' \<% # opening tag (\w+) # keyword ((?:\s+\w+|=|".*?"|'.*?')*) # attrname, = sign, string expression \s* # more whitespace (/)?> # closing ''', re.I | re.S | re.X) if match: (keyword, attr, isend) = (match.group(1).lower(), match.group(2), match.group(3)) self.keyword = keyword attributes = {} if attr: for att in re.findall( r"\s*(\w+)\s*=\s*(?:'([^']*)'|\"([^\"]*)\")", attr): (key, val1, val2) = att text = val1 or val2 text = text.replace('\r\n', '\n') attributes[key] = self.escape_code(text) self.append_node(parsetree.Tag, keyword, attributes) if isend: self.tag.pop() else: if keyword == 'text': match = self.match(r'(.*?)(?=\</%text>)', re.S) if not match: raise exceptions.SyntaxException( "Unclosed tag: <%%%s>" % self.tag[-1].keyword, **self.exception_kwargs) self.append_node(parsetree.Text, match.group(1)) return self.match_tag_end() return True else: return False
def writeline(self, line): """print a line of python, indenting it according to the current indent level. this also adjusts the indentation counter according to the content of the line. """ if not self.in_indent_lines: self._flush_adjusted_lines() self.in_indent_lines = True if ( line is None or re.match(r"^\s*#", line) or re.match(r"^\s*$", line) ): hastext = False else: hastext = True is_comment = line and len(line) and line[0] == '#' # see if this line should decrease the indentation level if ( not is_comment and (not hastext or self._is_unindentor(line)) ): if self.indent > 0: self.indent -= 1 # if the indent_detail stack is empty, the user # probably put extra closures - the resulting # module wont compile. if len(self.indent_detail) == 0: raise exceptions.SyntaxException( "Too many whitespace closures") self.indent_detail.pop() if line is None: return # write the line self.stream.write(self._indent_line(line) + "\n") self._update_lineno(len(line.split("\n"))) # see if this line should increase the indentation level. # note that a line can both decrase (before printing) and # then increase (after printing) the indentation level. if re.search(r":[ \t]*(?:#.*)?$", line): # increment indentation count, and also # keep track of what the keyword was that indented us, # if it is a python compound statement keyword # where we might have to look for an "unindent" keyword match = re.match(r"^\s*(if|try|elif|while|for|with)", line) if match: # its a "compound" keyword, so we will check for "unindentors" indentor = match.group(1) self.indent += 1 self.indent_detail.append(indentor) else: indentor = None # its not a "compound" keyword. but lets also # test for valid Python keywords that might be indenting us, # else assume its a non-indenting line m2 = re.match(r"^\s*(def|class|else|elif|except|finally)", line) if m2: self.indent += 1 self.indent_detail.append(indentor)
if self.match_tag_start(): continue if self.match_tag_end(): continue if self.match_python_block(): continue if self.match_text(): continue if self.match_position > self.textlength: break raise exceptions.CompileException("assertion failed") if len(self.tag): raise exceptions.SyntaxException( "Unclosed tag: <%%%s>" % self.tag[-1].keyword, **self.exception_kwargs) if len(self.control_line): raise exceptions.SyntaxException( "Unterminated control keyword: '%s'" % self.control_line[-1].keyword, self.text, self.control_line[-1].lineno, self.control_line[-1].pos, self.filename) return self.template def match_encoding(self): match = self.match(r'#.*coding[:=]\s*([-\w.]+).*\r?\n') if match: return match.group(1) else: return None