def python_code(tokenIterator, envName, endToken):
"""
Given an iterator of tokens, and the name of current environment, iterates through to the end of
the environment, and returns the text in the environment as python code. This function only
works for environments that are supposed to contain Python code.
"""
count = 0
pythonLine = ''
pythonCode = []
#parseNodes = []
startLine = 0
for token, lineNum in tokenIterator:
if not startLine:
#The first token is the endline at the end of \begin{envName}, so the environment starts on the same line as the first token in tokenIterator
startLine = lineNum
if endToken == token:
pythonCode = [line for line in pythonCode if line.strip()]
#We use this to figure out what the top level indent is, and strip that away so that Python can parse the code properly.
topLevelIndent = 1 if pythonCode[0][0] == '\t' else compute_indent(pythonCode[0])
try:
ast.parse(''.join(line[topLevelIndent:] for line in pythonCode))
except SyntaxError, e:
raise transExceptions.TranslationError(' '.join([parseTree.color("Error:", parseTree.bcolors.RED), 'Error in Python code found in', parseTree.color(envName,parseTree.bcolors.YELLOW),
'environment. Environment start line:', parseTree.color(str(startLine), parseTree.bcolors.GREEN), 'Python error:\n\n', str(e)]))
else:
return (startLine, pythonCode)
else:
if token.strip(' \t') == '\n':
pythonCode.append(pythonLine + token)
pythonLine = ''
else:
pythonLine += token
def parse_command(inlineCmd, startLineNum, tokenIterator, parent):
"""
Given an inline tex command, the line number at which the command occurs, and an iterator through the tex file's list of tokens, returns a parse Tree whose data is the
python code associated with the inline command, and whose children are the parsed arguments to the command.
"""
#Only exists for debugging purposes. Delete for actually running this.
#tokenList = list(tokenIterator)
#Only exists for debugging purposes. Delete for actually running this.
#tokenIterator = iter(tokenList)
args = []
data = [inlineCmd]
numArgs = 0
try:
numArgs = parseTree.inlineCommands[inlineCmd][1]
except KeyError:
try:
numArgs = parseTree.inlineCommandsPlayer[inlineCmd][1]
except IndexError:
r"""
If we get an index error, then that means that the inlineCmd does not take any arguments, in which case, the commands are followed by empty curly brackets, i.e. \name{}. This gets tokenized into
'\name', '{', '}' so we need to remove the curly brackets before moving on.
"""
clearingBraces = 2
while clearingBraces:
try:
token, lineNum = next(tokenIterator)
except StopIteration:
raise transExceptions.TranslationError(' '.join([parseTree.color("Error", parseTree.bcolors.RED), color_line(startLineNum), "Missing", "brackets for", parseTree.color(inlineCmd, parseTree.bcolors.GREEN)]))
if token == '{' or token == '}':
clearingBraces -= 1
else:
raise transExceptions.TranslationError(' '.join([parseTree.color("Error", parseTree.bcolors.RED), color_line(startLineNum), "Command", parseTree.color(inlineCmd, parseTree.bcolors.GREEN), "doesn't take any arguments."]))
numOpenBraces = 0
argumentTokens = []
tree = parseTree.InlineCommand(startLineNum, None, parent, data)
while len(args) < numArgs:
try:
token, lineNum = next(tokenIterator)
except StopIteration:
raise transExceptions.TranslationError(' '.join([parseTree.color("Error", parseTree.bcolors.RED), color_line(startLineNum), "Missing", parseTree.color(str(numArgs - len(args)), parseTree.bcolors.GREEN), "arguments for",
parseTree.color(inlineCmd, parseTree.bcolors.GREEN)]))
if token == '{':
numOpenBraces += 1
#We don't want to include the outermost brace. Only braces that may show up inside the argument.
if numOpenBraces == 1:
token = ''
elif token == '}':
numOpenBraces -= 1
if not numOpenBraces:
#If we're looking at a cond, then the first command is supposed to code: Some sort of boolean expression, with the next two arguments text.
if inlineCmd in parseTree.inlineCommands and not args:
args.append(parseTree.Code(lineNum=lineNum, parent=tree, data=[' '.join(token for (token, lineNum) in argumentTokens)]))
else:
args.append(parse_paragraph(iter(argumentTokens), tree))
argumentTokens = []
if numOpenBraces:
argumentTokens.append((token, lineNum))
tree.children = args
return tree
def code_command(command, tokenIterator, parent):
tree = parseTree.CodeCommands(parent=parent)
data = [command]
try:
#First token is {, which we don't want to include in our args.
next(tokenIterator)
token, lineNum = next(tokenIterator)
except StopIteration:
raise transExceptions.TranslationError(' '.join([parseTree.color("Error", parseTree.bcolors.RED), color_line(startLineNum), "LaTeX command:", command, "not terminated by a closing bracket '{'"]))
while token != '}':
data.append(token)
try:
token, lineNum = next(tokenIterator)
except StopIteration:
raise transExceptions.TranslationError(' '.join([parseTree.color("Error", parseTree.bcolors.RED), color_line(lineNum), "LaTeX command:", command, "not terminated by a closing bracket '{'"]))
tree.data = data
return tree
def parse_node(tokenIterator, node, startLine):
"""
Given a tokenIterator, a nodeType, and a line number at which the node begins, returns a parseTree for the node. This parseTree contains nothing as data, and has children that
consist of paragraphs, and any code environments.
"""
tree = node
try:
endToken = node.endToken
except AttributeError:
raise transExceptions.TranslationError(' '.join([error, "parse_node does not make any sense for the environment:", parseTree.color(tree.__class__.__name__, parseTree.bcolors.YELLOW), "Please inform Andrew Russell at [email protected] of",
"this error. Please send him: this error message, and a copy of the tex that generated the error."]))
paragraph = []
startLine = 0
previousLineNum = 0
for token, lineNum in tokenIterator:
if not startLine:
startLine = lineNum
tree.lineNum = startLine
if token == parseTree.BEGIN_CODE:
tree.children.append(code(tokenIterator, tree))
continue
elif token in parseTree.codeCommands:
tree.children.append(code_command(token, tokenIterator, tree))
continue
if token.strip(' ') == '\n':
#If the previous line number is strictly less than this one, then that means that the only token on this line is a new line, which means the line is blank, which means we're about to
#start a new paragraph.
if previousLineNum < lineNum:
tree.children.append(parse_paragraph(((token, lineNum) for (token, lineNum) in paragraph if token.strip()), tree))
paragraph = []
elif token == endToken:
return tree
elif token == parseTree.BEGIN_OPEN_SCENE or token == parseTree.BEGIN_NODE or token == parseTree.BEGIN_CHILD_NODE:
raise transExceptions.TranslationError(' '.join([error, color_line(startLine), "missing", parseTree.color(endToken, parseTree.bcolors.YELLOW), "for environment:",
parseTree.color(tree.__class__.__name__, parseTree.bcolors.BLUE)]))
else:
paragraph.append((token, lineNum))
previousLineNum = lineNum
else:
raise transExceptions.TranslationError(' '.join([error, color_line(startLine), "missing", parseTree.color(endToken, parseTree.bcolors.YELLOW), "for environment:",
parseTree.color(tree.__class__.__name__, parseTree.bcolors.BLUE)]))
def code(tokenIterator, parent):
tree = parseTree.Code(parent=parent)
data = []
pythonLine = ''
try:
token, startLineNum = next(tokenIterator)
except StopIteration:
raise transExceptions.TranslationError(' '.join([parseTree.color("Error", parseTree.bcolors.RED), color_line(startLineNum), "End of code block not found."]))
while token != parseTree.END_CODE:
try:
token, lineNum = next(tokenIterator)
except StopIteration:
raise transExceptions.TranslationError(' '.join([parseTree.color("Error", parseTree.bcolors.RED), color_line(startLineNum), "End of code block not found."]))
if token == parseTree.END_CODE:
continue
elif token == '\n':
data.append(pythonLine)
pythonLine = ''
else:
pythonLine += token
tree.data = data
tree.lineNum = startLineNum
return tree
def parse_environments(tokenIterator, root, startingNodeNum):
"""
Given an iterable of (token, line number) pairs, generates the parse tree associated of the latex source.
"""
#Group all error messages together, and print them together, so that the user can deal with them in batch, rather than one at a time.
errorText = []
currentScene = root
parseTree.nodeNum = startingNodeNum
for token, lineNum in tokenIterator:
"""
This loop iterates through the tokens until it reaches an open environment. Then, it determines which environment is being opened, and passes control off to a function that processes that
environment
"""
if parseTree.BEGIN in token:
try:
env = token[token.index('{')+1:token.index('}')]
except IndexError:
try:
token.index('{')
except IndexError:
raise transExceptions.TranslationError(' '.join([parseTree.color("Error Line:", parseTree.bcolors.RED), parseTree.color(str(lineNum), parseTree.bcolors.BLUE), "Missing", parseTree.color('{', parseTree.bcolors.YELLOW), "in token:",
parseTree.color(token, parseTree.bcolors.YELLOW)]))
else:
raise transExceptions.TranslationError(' '.join([parseTree.color("Error Line:", parseTree.bcolors.RED), parseTree.color(str(lineNum), parseTree.bcolors.BLUE), "Missing", parseTree.color('}', parseTree.bcolors.YELLOW), "in token:",
parseTree.color(token, parseTree.bcolors.YELLOW)]))
else:
try:
newTree = ENV_PROCESSORS[env](tokenIterator, currentScene)
except KeyError:
closeMatches = difflib.get_close_matches(env, ENV_PROCESSORS.keys(), 3, MIN_STRING_DIFFERENCE)
errorText.append(' '.join([parseTree.color("Error Line:", parseTree.bcolors.RED), parseTree.color(str(lineNum), parseTree.bcolors.BLUE), "Unrecognized environment:", parseTree.color(env, parseTree.bcolors.YELLOW),
"\n Did you mean:\n" if closeMatches else "",
'\n'.join([parseTree.color(match, parseTree.bcolors.GREEN) for match in closeMatches])]))
except transExceptions.TranslationError, e:
errorText.append(str(e))
else:
#open and close scenes are treated as children to root, in order to ensure proper tabbing.
if env == 'closeScene' or env == 'openScene':
root.children.append(newTree)
else:
currentScene.children.append(newTree)
#This is ugly, and should probably be refactored into duck typing, but I'm too lazy to do that now. It's big enough refactoring node type into explicit subclasses (which I really should have done from the
#beginning).
if isinstance(newTree, parseTree.CloseScene):
currentScene = root
elif isinstance(newTree, parseTree.OpenScene) and currentScene == root:
currentScene = newTree
elif isinstance(newTree, parseTree.OpenScene):
errorText.append(' '.join([error, color_line(lineNum), "New scene started before previous scene ended. Please add a", parseTree.color("closeScene", parseTree.bcolors.YELLOW),
"environment for previous scene."]))
def parse_children_command(cmd, startLineNum, tokenIterator, parent):
"""
Given a command that affects node transitions (childif, childelif, child, or continue or continueNewPage) returns the ParseTree associated with this node.
"""
try:
numArgs = childrenCommands[cmd]
except KeyError:
raise transExceptions.TranslationError(' '.join([parseTree.color("Error", parseTree.bcolors.RED), color_line(startLineNum), parseTree.color(cmd, parseTree.bcolors.GREEN), 'is not a valid node transition command.']))
args = []
argTokens = []
tree = parseTree.Link(lineNum=startLineNum, parent=parent, data=parent.data + [cmd])
if cmd == r'\childif' or cmd == r'\childelif':
numOpenBraces = 0
while len(args) < numArgs:
try:
token, lineNum = tokenIterator.next()
except StopIteration:
raise transExceptions.TranslationError(' '.join([parseTree.color("Error", parseTree.bcolors.RED), color_line(startLineNum), "Command", parseTree.color(cmd, parseTree.bcolors.GREEN), "is missing",
parseTree.color(str(numArgs - len(args)), parseTree.bcolors.GREEN), "arguments."]))
if token == '{':
if not numOpenBraces:
token = ''
numOpenBraces += 1
elif token == '}':
numOpenBraces -= 1
if numOpenBraces < 0:
raise transExceptions.TranslationError(' '.join([parseTree.color("Error", parseTree.bcolors.RED), color_line(lineNum), "Too many", parseTree.color('}', parseTree.bcolors.GREEN)]))
elif not numOpenBraces:
args.append(argTokens)
argTokens = []
if numOpenBraces:
argTokens.append(token)
if numOpenBraces > 0:
raise transExceptions.TranslationError(' '.join([parseTree.color("Error", parseTree.bcolors.RED), color_line(startLineNum), "Too many", parseTree.color('{', parseTree.bcolors.GREEN)]))
try:
ast.parse(' '.join(args[0]).strip())
except SyntaxError, e:
raise transExceptions.TranslationError(' '.join([parseTree.color("Error:", parseTree.bcolors.RED), 'Error in Python code found in', parseTree.color('childif',parseTree.bcolors.YELLOW), 'command on line:',
parseTree.color(str(lineNum), parseTree.bcolors.GREEN), 'Python error:\n\n', str(e)]))
args = [parseTree.Code(lineNum=startLineNum, parent=tree, data=[' '.join(args[0])]), parseTree.Destination(lineNum=startLineNum, parent=tree, data=args[1])]
"""
def translation_error_msg(startLine, numArgsProcessed):
return ' '.join([error, color_line(startLine), "Argument", parseTree.color(str(numArgsProcessed + 1), parseTree.bcolors.BLUE), "of", parseTree.color(str(nodeType.__name__), parseTree.bcolors.GREEN), "missing",
parseTree.color('}', parseTree.bcolors.YELLOW)])
import ast
import difflib
import transExceptions
import parseTree
#Mapping from the command, to the number of arguments the command takes.
childrenCommands = {r'\childif':2, r'\childelif':2, r'\child':2, r'\continue':1, r'\continueNewPage':1}
#A mapping from latex commands to a tuple containing the start of the python code, and the number of arguments the latex command takes.
error = parseTree.color("Error", parseTree.bcolors.RED)
def color_line(lineNum):
return parseTree.color_line(lineNum)
def compute_indent(codeLine):
count = 0
for char in codeLine:
if char == ' ':
count += 1
else:
return count
return count
