def find_symbol_name(code, cursor_pos):
'''
Given a string of code tokenize it until cursor_pos and return the final symbol name.
returns None if no symbol is found at cursor_pos.
>>> MathicsKernel.find_symbol_name('1 + Sin', 6)
'System`Sin'
>>> MathicsKernel.find_symbol_name('1 + ` Sin[Cos[2]] + x', 8)
'System`Sin'
>>> MathicsKernel.find_symbol_name('Sin `', 4)
'''
tokeniser = Tokeniser(SingleLineFeeder(code))
start_pos = None
end_pos = None
name = None
while True:
try:
token = tokeniser.next()
except ScanError:
continue
if token.tag == 'END':
break # ran out of tokens
# find first token which contains cursor_pos
if tokeniser.pos >= cursor_pos:
if token.tag == 'Symbol':
name = token.text
start_pos = token.pos
end_pos = tokeniser.pos
break
return start_pos, end_pos, name
def parse(self, feeder):
self.feeder = feeder
self.tokeniser = Tokeniser(feeder)
self.current_token = None
self.bracket_depth = 0
self.box_depth = 0
return self.parse_e()
def find_symbol_name(code, cursor_pos):
'''
Given a string of code tokenize it until cursor_pos and return the final symbol name.
returns None if no symbol is found at cursor_pos.
>>> MathicsKernel.find_symbol_name('1 + Sin', 6)
'System`Sin'
>>> MathicsKernel.find_symbol_name('1 + ` Sin[Cos[2]] + x', 8)
'System`Sin'
>>> MathicsKernel.find_symbol_name('Sin `', 4)
'''
tokeniser = Tokeniser(SingleLineFeeder(code))
start_pos = None
end_pos = None
name = None
while True:
try:
token = tokeniser.next()
except ScanError:
continue
if token.tag == 'END':
break # ran out of tokens
# find first token which contains cursor_pos
if tokeniser.pos >= cursor_pos:
if token.tag == 'Symbol':
name = token.text
start_pos = token.pos
end_pos = tokeniser.pos
break
return start_pos, end_pos, name
def tokens(self, code):
tokeniser = Tokeniser(SingleLineFeeder(code))
tokens = []
while True:
token = tokeniser.next()
if token.tag == 'END':
break
else:
tokens.append(token)
return tokens
def tokens(self, code):
tokeniser = Tokeniser(SingleLineFeeder(code))
tokens = []
while True:
token = tokeniser.next()
if token.tag == 'END':
break
else:
tokens.append(token)
return tokens
def parse(self, feeder):
self.feeder = feeder
self.tokeniser = Tokeniser(feeder)
self.current_token = None
self.bracket_depth = 0
self.box_depth = 0
return self.parse_e()
class Parser(object):
def __init__(self):
# no implicit times on these tokens
self.halt_tags = set([
'END', 'RawRightParenthesis', 'RawComma', 'RawRightBrace',
'RawRightBracket', 'RawColon', 'DifferentialD'
])
def parse(self, feeder):
self.feeder = feeder
self.tokeniser = Tokeniser(feeder)
self.current_token = None
self.bracket_depth = 0
self.box_depth = 0
return self.parse_e()
def next(self):
if self.current_token is None:
self.current_token = self.tokeniser.next()
return self.current_token
def next_noend(self):
'returns next token which is not END'
while True:
token = self.next()
if token.tag != 'END':
return token
self.incomplete(token.pos)
def consume(self):
self.current_token = None
def incomplete(self, pos):
self.tokeniser.incomplete()
self.backtrack(pos)
def expect(self, expected_tag):
token = self.next_noend()
if token.tag == expected_tag:
self.consume()
else:
self.tokeniser.sntx_message(token.pos)
raise InvalidSyntaxError()
def backtrack(self, pos):
self.tokeniser.pos = pos
self.current_token = None
def parse_e(self):
result = []
while self.next().tag != 'END':
result.append(self.parse_exp(0))
if len(result) > 1:
return Node('Times', *result)
if len(result) == 1:
return result[0]
else:
return None
def parse_exp(self, p):
result = self.parse_p()
while True:
if self.bracket_depth > 0:
token = self.next_noend()
else:
token = self.next()
tag = token.tag
method = getattr(self, 'e_' + tag, None)
if method is not None:
new_result = method(result, token, p)
elif tag in inequality_ops:
new_result = self.parse_inequality(result, token, p)
elif tag in binary_ops:
new_result = self.parse_binary(result, token, p)
elif tag in ternary_ops:
new_result = self.parse_ternary(result, token, p)
elif tag in postfix_ops:
new_result = self.parse_postfix(result, token, p)
elif tag not in self.halt_tags and flat_binary_ops['Times'] >= p:
# implicit times
q = flat_binary_ops['Times']
child = self.parse_exp(q + 1)
new_result = Node('Times', result, child).flatten()
else:
new_result = None
if new_result is None:
break
else:
result = new_result
return result
def parse_p(self):
token = self.next_noend()
tag = token.tag
method = getattr(self, 'p_' + tag, None)
if method is not None:
return method(token)
elif tag in prefix_ops:
self.consume()
q = prefix_ops[tag]
child = self.parse_exp(q)
return Node(tag, child)
else:
self.tokeniser.sntx_message(token.pos)
raise InvalidSyntaxError()
def parse_box(self, p):
result = None
while True:
token = self.next()
tag = token.tag
method = getattr(self, 'b_' + tag, None)
if method is not None:
new_result = method(result, token, p)
elif tag in ('OtherscriptBox', 'RightRowBox'):
break
elif tag == 'END':
self.incomplete(token.pos)
elif result is None and tag != 'END':
self.consume()
new_result = String(token.text)
else:
new_result = None
if new_result is None:
break
else:
result = new_result
if result is None:
result = String('')
return result
def parse_seq(self):
result = []
while True:
token = self.next_noend()
tag = token.tag
if tag == 'RawComma':
self.tokeniser.feeder.message('Syntax', 'com')
result.append(Symbol('Null'))
self.consume()
elif tag in ('RawRightBrace', 'RawRightBracket'):
if result:
self.tokeniser.feeder.message('Syntax', 'com')
result.append(Symbol('Null'))
break
else:
result.append(self.parse_exp(0))
token = self.next_noend()
tag = token.tag
if tag == 'RawComma':
self.consume()
continue
elif tag in ('RawRightBrace', 'RawRightBracket'):
break
return result
def parse_inequality(self, expr1, token, p):
tag = token.tag
q = flat_binary_ops[tag]
if q < p:
return None
self.consume()
head = expr1.get_head_name()
expr2 = self.parse_exp(q + 1)
if head == 'Inequality' and not expr1.parenthesised:
expr1.children.append(Symbol(tag))
expr1.children.append(expr2)
elif head in inequality_ops and head != tag and not expr1.parenthesised:
children = []
first = True
for child in expr1.children:
if not first:
children.append(Symbol(head))
children.append(child)
first = False
children.append(Symbol(tag))
children.append(expr2)
expr1 = Node('Inequality', *children)
else:
expr1 = Node(tag, expr1, expr2).flatten()
return expr1
def parse_binary(self, expr1, token, p):
tag = token.tag
q = binary_ops[tag]
if q < p:
return None
self.consume()
if tag not in right_binary_ops:
q += 1
expr2 = self.parse_exp(q)
# flatten or associate
if tag in nonassoc_binary_ops and expr1.get_head_name(
) == tag and not expr1.parenthesised:
self.tokeniser.sntx_message(token.pos)
raise InvalidSyntaxError()
result = Node(tag, expr1, expr2)
if tag in flat_binary_ops:
result.flatten()
return result
def parse_postfix(self, expr1, token, p):
tag = token.tag
q = postfix_ops[tag]
if q < p:
return None
self.consume()
return Node(tag, expr1)
# P methods
#
# p_xxx methods are called from parse_p.
# Called with one Token and return a Node.
# Used for prefix operators and brackets.
def p_Factorial(self, token):
self.consume()
q = prefix_ops['Not']
child = self.parse_exp(q)
return Node('Not', child)
def p_Factorial2(self, token):
self.consume()
q = prefix_ops['Not']
child = self.parse_exp(q)
return Node('Not', Node('Not', child))
def p_RawLeftParenthesis(self, token):
self.consume()
self.bracket_depth += 1
result = self.parse_exp(0)
self.expect('RawRightParenthesis')
self.bracket_depth -= 1
result.parenthesised = True
return result
def p_RawLeftBrace(self, token):
self.consume()
self.bracket_depth += 1
seq = self.parse_seq()
self.expect('RawRightBrace')
self.bracket_depth -= 1
return Node('List', *seq)
def p_LeftRowBox(self, token):
self.consume()
children = []
self.box_depth += 1
self.bracket_depth += 1
token = self.next()
while token.tag not in ('RightRowBox', 'OtherscriptBox'):
children.append(self.parse_box(0))
token = self.next()
if len(children) == 0:
result = String('')
elif len(children) == 1:
result = children[0]
else:
result = Node('RowBox', Node('List', *children))
self.expect('RightRowBox')
self.box_depth -= 1
self.bracket_depth -= 1
result.parenthesised = True
return result
def p_Number(self, token):
result = Number(token.text)
self.consume()
return result
def p_String(self, token):
result = String(token.text[1:-1])
self.consume()
return result
def p_Symbol(self, token):
symbol_name = special_symbols.get(token.text, token.text)
result = Symbol(symbol_name, context=None)
self.consume()
return result
def p_Filename(self, token):
result = Filename(token.text)
self.consume()
return result
def p_Span(self, token):
return self.e_Span(Number('1'), token, 0)
def p_Integral(self, token):
self.consume()
inner_prec, outer_prec = all_ops['Sum'] + 1, all_ops['Power'] - 1
expr1 = self.parse_exp(inner_prec)
self.expect('DifferentialD')
expr2 = self.parse_exp(outer_prec)
return Node('Integrate', expr1, expr2)
def p_Pattern(self, token):
self.consume()
text = token.text
if '.' in text:
name = text[:-2]
if name:
return Node(
'Optional',
Node('Pattern', Symbol(name, context=None), Node('Blank')))
else:
return Node('Optional', Node('Blank'))
pieces = text.split('_')
count = len(pieces) - 1
if count == 1:
name = 'Blank'
elif count == 2:
name = 'BlankSequence'
elif count == 3:
name = 'BlankNullSequence'
if pieces[-1]:
blank = Node(name, Symbol(pieces[-1], context=None))
else:
blank = Node(name)
if pieces[0]:
return Node('Pattern', Symbol(pieces[0], context=None), blank)
else:
return blank
def p_Minus(self, token):
self.consume()
q = prefix_ops['Minus']
expr = self.parse_exp(q)
if isinstance(expr, Number) and not expr.value.startswith('-'):
expr.value = '-' + expr.value
return expr
else:
return Node('Times', Number('-1'), expr).flatten()
def p_Plus(self, token):
self.consume()
q = prefix_ops['Minus']
# note flattening here even flattens e.g. + a + b
return Node('Plus', self.parse_exp(q)).flatten()
def p_PlusMinus(self, token):
self.consume()
q = prefix_ops['Minus']
return Node('PlusMinus', self.parse_exp(q))
def p_MinusPlus(self, token):
self.consume()
q = prefix_ops['Minus']
return Node('MinusPlus', self.parse_exp(q))
def p_Out(self, token):
self.consume()
text = token.text
if text == '%':
return Node('Out')
if text.endswith('%'):
n = str(-len(text))
else:
n = text[1:]
return Node('Out', Number(n))
def p_Slot(self, token):
self.consume()
text = token.text
if len(text) == 1:
n = Number('1')
else:
n = text[1:]
if n.isdigit():
n = Number(n)
else:
n = String(n)
return Node('Slot', n)
def p_SlotSequence(self, token):
self.consume()
text = token.text
if len(text) == 2:
n = '1'
else:
n = text[2:]
return Node('SlotSequence', Number(n))
def p_Increment(self, token):
self.consume()
q = prefix_ops['PreIncrement']
return Node('PreIncrement', self.parse_exp(q))
def p_Decrement(self, token):
self.consume()
q = prefix_ops['PreDecrement']
return Node('PreDecrement', self.parse_exp(q))
# E methods
#
# e_xxx methods are called from parse_e.
# They expect args (Node, Token precedence) and return Node or None.
# Used for binary and ternary operators.
# return None if precedence is too low.
def e_Span(self, expr1, token, p):
q = ternary_ops['Span']
if q < p:
return None
if expr1.get_head_name() == 'Span' and not expr1.parenthesised:
return None
self.consume()
# Span[expr1, expr2]
token = self.next()
if token.tag == 'Span':
expr2 = Symbol('All')
elif token.tag == 'END' and self.bracket_depth == 0:
# So that e.g. 'x = 1 ;;' doesn't wait for newline in the frontend
expr2 = Symbol('All')
return Node('Span', expr1, expr2)
else:
messages = list(self.feeder.messages)
try:
expr2 = self.parse_exp(q + 1)
except TranslateError:
expr2 = Symbol('All')
self.backtrack(token.pos)
self.feeder.messages = messages
token = self.next()
if token.tag == 'Span':
self.consume()
messages = list(self.feeder.messages)
try:
expr3 = self.parse_exp(q + 1)
return Node('Span', expr1, expr2, expr3)
except TranslateError:
self.backtrack(token.pos)
self.feeder.messages = messages
return Node('Span', expr1, expr2)
def e_RawLeftBracket(self, expr, token, p):
q = all_ops['Part']
if q < p:
return None
self.consume()
self.bracket_depth += 1
token = self.next_noend()
if token.tag == 'RawLeftBracket':
self.consume()
seq = self.parse_seq()
self.expect('RawRightBracket')
self.expect('RawRightBracket')
self.bracket_depth -= 1
return Node('Part', expr, *seq)
else:
seq = self.parse_seq()
self.expect('RawRightBracket')
self.bracket_depth -= 1
result = Node(expr, *seq)
result.parenthesised = True
return result
def e_Infix(self, expr1, token, p):
q = ternary_ops['Infix']
if q < p:
return None
self.consume()
expr2 = self.parse_exp(q + 1)
self.expect('Infix')
expr3 = self.parse_exp(q + 1)
return Node(expr2, expr1, expr3)
def e_Postfix(self, expr1, token, p):
q = left_binary_ops['Postfix']
if q < p:
return None
self.consume()
# postix has lowest prec and is left assoc
expr2 = self.parse_exp(q + 1)
return Node(expr2, expr1)
def e_Prefix(self, expr1, token, p):
q = 640
if 640 < p:
return None
self.consume()
expr2 = self.parse_exp(q)
return Node(expr1, expr2)
def e_ApplyList(self, expr1, token, p):
q = right_binary_ops['Apply']
if q < p:
return None
self.consume()
expr2 = self.parse_exp(q)
expr3 = Node('List', Number('1'))
return Node('Apply', expr1, expr2, expr3)
def e_Function(self, expr1, token, p):
q = postfix_ops['Function']
if q < p:
return None
# postfix or right-binary determined by symbol
self.consume()
if token.text == '&':
return Node('Function', expr1)
else:
expr2 = self.parse_exp(q)
return Node('Function', expr1, expr2)
def e_RawColon(self, expr1, token, p):
head_name = expr1.get_head_name()
if head_name == 'Symbol':
head = 'Pattern'
elif head_name in ('Blank', 'BlankSequence', 'BlankNullSequence',
'Pattern', 'Optional'):
head = 'Optional'
else:
return None
q = all_ops[head]
if p == 151:
return None
self.consume()
expr2 = self.parse_exp(q + 1)
return Node(head, expr1, expr2)
def e_Semicolon(self, expr1, token, p):
q = flat_binary_ops['CompoundExpression']
if q < p:
return None
self.consume()
# XXX this has to come before call to self.next()
pos = self.tokeniser.pos
messages = list(self.feeder.messages)
# So that e.g. 'x = 1;' doesn't wait for newline in the frontend
tag = self.next().tag
if tag == 'END' and self.bracket_depth == 0:
expr2 = Symbol('Null')
return Node('CompoundExpression', expr1, expr2).flatten()
# XXX look for next expr otherwise backtrack
try:
expr2 = self.parse_exp(q + 1)
except TranslateError:
self.backtrack(pos)
self.feeder.messages = messages
expr2 = Symbol('Null')
return Node('CompoundExpression', expr1, expr2).flatten()
def e_Minus(self, expr1, token, p):
q = left_binary_ops['Subtract']
if q < p:
return None
self.consume()
expr2 = self.parse_exp(q + 1)
if isinstance(expr2, Number) and not expr2.value.startswith('-'):
expr2.value = '-' + expr2.value
else:
expr2 = Node('Times', Number('-1'), expr2).flatten()
return Node('Plus', expr1, expr2).flatten()
def e_TagSet(self, expr1, token, p):
q = all_ops['Set']
if q < p:
return None
self.consume()
expr2 = self.parse_exp(q + 1)
# examine next token
token = self.next_noend()
tag = token.tag
if tag == 'Set':
head = 'TagSet'
elif tag == 'SetDelayed':
head = 'TagSetDelayed'
elif tag == 'Unset':
head = 'TagUnset'
else:
self.tokeniser.sntx_message(token.pos)
raise InvalidSyntaxError()
self.consume()
if head == 'TagUnset':
return Node(head, expr1, expr2)
expr3 = self.parse_exp(q + 1)
return Node(head, expr1, expr2, expr3)
def e_Unset(self, expr1, token, p):
q = all_ops['Set']
if q < p:
return None
self.consume()
return Node('Unset', expr1)
def e_Derivative(self, expr1, token, p):
q = postfix_ops['Derivative']
if q < p:
return None
n = 0
while self.next().tag == 'Derivative':
self.consume()
n += 1
head = Node('Derivative', Number(str(n)))
return Node(head, expr1)
def e_Divide(self, expr1, token, p):
q = left_binary_ops['Divide']
if q < p:
return None
self.consume()
expr2 = self.parse_exp(q + 1)
return Node('Times', expr1, Node('Power', expr2,
Number('-1'))).flatten()
def e_Alternatives(self, expr1, token, p):
q = flat_binary_ops['Alternatives']
if q < p:
return None
self.consume()
expr2 = self.parse_exp(q + 1)
return Node('Alternatives', expr1, expr2).flatten()
def e_MessageName(self, expr1, token, p):
leaves = [expr1]
while self.next().tag == 'MessageName':
self.consume()
token = self.next()
if token.tag == 'Symbol':
# silently convert Symbol to String
self.consume()
leaf = String(token.text)
elif token.tag == 'String':
leaf = self.p_String(token)
else:
self.tokeniser.sntx_message(token.pos)
raise InvalidSyntaxError()
leaves.append(leaf)
return Node('MessageName', *leaves)
# B methods
#
# b_xxx methods are called from parse_box.
# They expect args (Node, Token precedence) and return Node or None.
# The first argument may be None if the LHS is absent.
# Used for boxes.
def b_SqrtBox(self, box0, token, p):
if box0 is not None:
return None
self.consume()
q = misc_ops['SqrtBox']
box1 = self.parse_box(q)
if self.next().tag == 'OtherscriptBox':
self.consume()
box2 = self.parse_box(q)
return Node('RadicalBox', box1, box2)
else:
return Node('SqrtBox', box1)
def b_SuperscriptBox(self, box1, token, p):
q = misc_ops['SuperscriptBox']
if q < p:
return None
if box1 is None:
box1 = String('')
self.consume()
box2 = self.parse_box(q)
if self.next().tag == 'OtherscriptBox':
self.consume()
box3 = self.parse_box(misc_ops['SubsuperscriptBox'])
return Node('SubsuperscriptBox', box1, box3, box2)
else:
return Node('SuperscriptBox', box1, box2)
def b_SubscriptBox(self, box1, token, p):
q = misc_ops['SubscriptBox']
if q < p:
return None
if box1 is None:
box1 = String('')
self.consume()
box2 = self.parse_box(q)
if self.next().tag == 'OtherscriptBox':
self.consume()
box3 = self.parse_box(misc_ops['SubsuperscriptBox'])
return Node('SubsuperscriptBox', box1, box2, box3)
else:
return Node('SubscriptBox', box1, box2)
def b_UnderscriptBox(self, box1, token, p):
q = misc_ops['UnderscriptBox']
if q < p:
return None
if box1 is None:
box1 = String('')
self.consume()
box2 = self.parse_box(q)
if self.next().tag == 'OtherscriptBox':
self.consume()
box3 = self.parse_box(misc_ops['UnderoverscriptBox'])
return Node('UnderoverscriptBox', box1, box2, box3)
else:
return Node('UnderscriptBox', box1, box2)
def b_FractionBox(self, box1, token, p):
q = misc_ops['FractionBox']
if q < p:
return None
if box1 is None:
box1 = String('')
self.consume()
box2 = self.parse_box(q + 1)
return Node('FractionBox', box1, box2)
def b_FormBox(self, box1, token, p):
q = misc_ops['FormBox']
if q < p:
return None
if box1 is None:
box1 = Symbol('StandardForm') # RawForm
elif is_symbol_name(box1.value):
box1 = Symbol(box1.value, context=None)
else:
box1 = Node('Removed', String('$$Failure'))
self.consume()
box2 = self.parse_box(q)
return Node('FormBox', box2, box1)
def b_OverscriptBox(self, box1, token, p):
q = misc_ops['OverscriptBox']
if q < p:
return None
if box1 is None:
box1 = String('')
self.consume()
box2 = self.parse_box(q)
if self.next().tag == 'OtherscriptBox':
self.consume()
box3 = self.parse_box(misc_ops['UnderoverscriptBox'])
return Node('UnderoverscriptBox', box1, box3, box2)
else:
return Node('OverscriptBox', box1, box2)
class Parser(object):
def __init__(self):
# no implicit times on these tokens
self.halt_tags = set([
'END', 'RawRightParenthesis', 'RawComma', 'RawRightBrace',
'RawRightBracket', 'RawColon', 'DifferentialD'])
def parse(self, feeder):
self.feeder = feeder
self.tokeniser = Tokeniser(feeder)
self.current_token = None
self.bracket_depth = 0
self.box_depth = 0
return self.parse_e()
def next(self):
if self.current_token is None:
self.current_token = self.tokeniser.next()
return self.current_token
def next_noend(self):
'returns next token which is not END'
while True:
token = self.next()
if token.tag != 'END':
return token
self.incomplete(token.pos)
def consume(self):
self.current_token = None
def incomplete(self, pos):
self.tokeniser.incomplete()
self.backtrack(pos)
def expect(self, expected_tag):
token = self.next_noend()
if token.tag == expected_tag:
self.consume()
else:
self.tokeniser.sntx_message(token.pos)
raise InvalidSyntaxError()
def backtrack(self, pos):
self.tokeniser.pos = pos
self.current_token = None
def parse_e(self):
result = []
while self.next().tag != 'END':
result.append(self.parse_exp(0))
if len(result) > 1:
return Node('Times', *result)
if len(result) == 1:
return result[0]
else:
return None
def parse_exp(self, p):
result = self.parse_p()
while True:
if self.bracket_depth > 0:
token = self.next_noend()
else:
token = self.next()
tag = token.tag
method = getattr(self, 'e_' + tag, None)
if method is not None:
new_result = method(result, token, p)
elif tag in inequality_ops:
new_result = self.parse_inequality(result, token, p)
elif tag in binary_ops:
new_result = self.parse_binary(result, token, p)
elif tag in ternary_ops:
new_result = self.parse_ternary(result, token, p)
elif tag in postfix_ops:
new_result = self.parse_postfix(result, token, p)
elif tag not in self.halt_tags and flat_binary_ops['Times'] >= p:
# implicit times
q = flat_binary_ops['Times']
child = self.parse_exp(q + 1)
new_result = Node('Times', result, child).flatten()
else:
new_result = None
if new_result is None:
break
else:
result = new_result
return result
def parse_p(self):
token = self.next_noend()
tag = token.tag
method = getattr(self, 'p_' + tag, None)
if method is not None:
return method(token)
elif tag in prefix_ops:
self.consume()
q = prefix_ops[tag]
child = self.parse_exp(q)
return Node(tag, child)
else:
self.tokeniser.sntx_message(token.pos)
raise InvalidSyntaxError()
def parse_box(self, p):
result = None
while True:
token = self.next()
tag = token.tag
method = getattr(self, 'b_' + tag, None)
if method is not None:
new_result = method(result, token, p)
elif tag in ('OtherscriptBox', 'RightRowBox'):
break
elif tag == 'END':
self.incomplete(token.pos)
elif result is None and tag != 'END':
self.consume()
new_result = String(token.text)
else:
new_result = None
if new_result is None:
break
else:
result = new_result
if result is None:
result = String('')
return result
def parse_seq(self):
result = []
while True:
token = self.next_noend()
tag = token.tag
if tag == 'RawComma':
self.tokeniser.feeder.message('Syntax', 'com')
result.append(Symbol('Null'))
self.consume()
elif tag in ('RawRightBrace', 'RawRightBracket'):
if result:
self.tokeniser.feeder.message('Syntax', 'com')
result.append(Symbol('Null'))
break
else:
result.append(self.parse_exp(0))
token = self.next_noend()
tag = token.tag
if tag == 'RawComma':
self.consume()
continue
elif tag in ('RawRightBrace', 'RawRightBracket'):
break
return result
def parse_inequality(self, expr1, token, p):
tag = token.tag
q = flat_binary_ops[tag]
if q < p:
return None
self.consume()
head = expr1.get_head_name()
expr2 = self.parse_exp(q + 1)
if head == 'Inequality' and not expr1.parenthesised:
expr1.children.append(Symbol(tag))
expr1.children.append(expr2)
elif head in inequality_ops and head != tag and not expr1.parenthesised:
children = []
first = True
for child in expr1.children:
if not first:
children.append(Symbol(head))
children.append(child)
first = False
children.append(Symbol(tag))
children.append(expr2)
expr1 = Node('Inequality', *children)
else:
expr1 = Node(tag, expr1, expr2).flatten()
return expr1
def parse_binary(self, expr1, token, p):
tag = token.tag
q = binary_ops[tag]
if q < p:
return None
self.consume()
if tag not in right_binary_ops:
q += 1
expr2 = self.parse_exp(q)
# flatten or associate
if tag in nonassoc_binary_ops and expr1.get_head_name() == tag and not expr1.parenthesised:
self.tokeniser.sntx_message(token.pos)
raise InvalidSyntaxError()
result = Node(tag, expr1, expr2)
if tag in flat_binary_ops:
result.flatten()
return result
def parse_postfix(self, expr1, token, p):
tag = token.tag
q = postfix_ops[tag]
if q < p:
return None
self.consume()
return Node(tag, expr1)
# P methods
#
# p_xxx methods are called from parse_p.
# Called with one Token and return a Node.
# Used for prefix operators and brackets.
def p_Factorial(self, token):
self.consume()
q = prefix_ops['Not']
child = self.parse_exp(q)
return Node('Not', child)
def p_Factorial2(self, token):
self.consume()
q = prefix_ops['Not']
child = self.parse_exp(q)
return Node('Not', Node('Not', child))
def p_RawLeftParenthesis(self, token):
self.consume()
self.bracket_depth += 1
result = self.parse_exp(0)
self.expect('RawRightParenthesis')
self.bracket_depth -= 1
result.parenthesised = True
return result
def p_RawLeftBrace(self, token):
self.consume()
self.bracket_depth += 1
seq = self.parse_seq()
self.expect('RawRightBrace')
self.bracket_depth -= 1
return Node('List', *seq)
def p_LeftRowBox(self, token):
self.consume()
children = []
self.box_depth += 1
self.bracket_depth += 1
token = self.next()
while token.tag not in ('RightRowBox', 'OtherscriptBox'):
children.append(self.parse_box(0))
token = self.next()
if len(children) == 0:
result = String('')
elif len(children) == 1:
result = children[0]
else:
result = Node('RowBox', Node('List', *children))
self.expect('RightRowBox')
self.box_depth -= 1
self.bracket_depth -= 1
result.parenthesised = True
return result
def p_Number(self, token):
result = Number(token.text)
self.consume()
return result
def p_String(self, token):
result = String(token.text[1:-1])
self.consume()
return result
def p_Symbol(self, token):
result = Symbol(token.text, context=None)
self.consume()
return result
def p_Filename(self, token):
result = Filename(token.text)
self.consume()
return result
def p_Span(self, token):
return self.e_Span(Number('1'), token, 0)
def p_Integral(self, token):
self.consume()
inner_prec, outer_prec = all_ops['Sum'] + 1, all_ops['Power'] - 1
expr1 = self.parse_exp(inner_prec)
self.expect('DifferentialD')
expr2 = self.parse_exp(outer_prec)
return Node('Integrate', expr1, expr2)
def p_Pattern(self, token):
self.consume()
text = token.text
if '.' in text:
name = text[:-2]
if name:
return Node('Optional', Node('Pattern', Symbol(name, context=None), Node('Blank')))
else:
return Node('Optional', Node('Blank'))
pieces = text.split('_')
count = len(pieces) - 1
if count == 1:
name = 'Blank'
elif count == 2:
name = 'BlankSequence'
elif count == 3:
name = 'BlankNullSequence'
if pieces[-1]:
blank = Node(name, Symbol(pieces[-1], context=None))
else:
blank = Node(name)
if pieces[0]:
return Node('Pattern', Symbol(pieces[0], context=None), blank)
else:
return blank
def p_Minus(self, token):
self.consume()
q = prefix_ops['Minus']
expr = self.parse_exp(q)
if isinstance(expr, Number) and not expr.value.startswith('-'):
expr.value = '-' + expr.value
return expr
else:
return Node('Times', Number('-1'), expr).flatten()
def p_Plus(self, token):
self.consume()
q = prefix_ops['Minus']
# note flattening here even flattens e.g. + a + b
return Node('Plus', self.parse_exp(q)).flatten()
def p_PlusMinus(self, token):
self.consume()
q = prefix_ops['Minus']
return Node('PlusMinus', self.parse_exp(q))
def p_MinusPlus(self, token):
self.consume()
q = prefix_ops['Minus']
return Node('MinusPlus', self.parse_exp(q))
def p_Out(self, token):
self.consume()
text = token.text
if text == '%':
return Node('Out')
if text.endswith('%'):
n = str(-len(text))
else:
n = text[1:]
return Node('Out', Number(n))
def p_Slot(self, token):
self.consume()
text = token.text
if len(text) == 1:
n = Number('1')
else:
n = text[1:]
if n.isdigit():
n = Number(n)
else:
n = String(n)
return Node('Slot', n)
def p_SlotSequence(self, token):
self.consume()
text = token.text
if len(text) == 2:
n = '1'
else:
n = text[2:]
return Node('SlotSequence', Number(n))
def p_Increment(self, token):
self.consume()
q = prefix_ops['PreIncrement']
return Node('PreIncrement', self.parse_exp(q))
def p_Decrement(self, token):
self.consume()
q = prefix_ops['PreDecrement']
return Node('PreDecrement', self.parse_exp(q))
# E methods
#
# e_xxx methods are called from parse_e.
# They expect args (Node, Token precedence) and return Node or None.
# Used for binary and ternary operators.
# return None if precedence is too low.
def e_Span(self, expr1, token, p):
q = ternary_ops['Span']
if q < p:
return None
if expr1.get_head_name() == 'Span' and not expr1.parenthesised:
return None
self.consume()
# Span[expr1, expr2]
token = self.next()
if token.tag == 'Span':
expr2 = Symbol('All')
elif token.tag == 'END' and self.bracket_depth == 0:
# So that e.g. 'x = 1 ;;' doesn't wait for newline in the frontend
expr2 = Symbol('All')
return Node('Span', expr1, expr2)
else:
messages = list(self.feeder.messages)
try:
expr2 = self.parse_exp(q + 1)
except TranslateError:
expr2 = Symbol('All')
self.backtrack(token.pos)
self.feeder.messages = messages
token = self.next()
if token.tag == 'Span':
self.consume()
messages = list(self.feeder.messages)
try:
expr3 = self.parse_exp(q + 1)
return Node('Span', expr1, expr2, expr3)
except TranslateError:
self.backtrack(token.pos)
self.feeder.messages = messages
return Node('Span', expr1, expr2)
def e_RawLeftBracket(self, expr, token, p):
q = all_ops['Part']
if q < p:
return None
self.consume()
self.bracket_depth += 1
token = self.next_noend()
if token.tag == 'RawLeftBracket':
self.consume()
seq = self.parse_seq()
self.expect('RawRightBracket')
self.expect('RawRightBracket')
self.bracket_depth -= 1
return Node('Part', expr, *seq)
else:
seq = self.parse_seq()
self.expect('RawRightBracket')
self.bracket_depth -= 1
result = Node(expr, *seq)
result.parenthesised = True
return result
def e_Infix(self, expr1, token, p):
q = ternary_ops['Infix']
if q < p:
return None
self.consume()
expr2 = self.parse_exp(q + 1)
self.expect('Infix')
expr3 = self.parse_exp(q + 1)
return Node(expr2, expr1, expr3)
def e_Postfix(self, expr1, token, p):
q = left_binary_ops['Postfix']
if q < p:
return None
self.consume()
# postix has lowest prec and is left assoc
expr2 = self.parse_exp(q + 1)
return Node(expr2, expr1)
def e_Prefix(self, expr1, token, p):
q = 640
if 640 < p:
return None
self.consume()
expr2 = self.parse_exp(q)
return Node(expr1, expr2)
def e_ApplyList(self, expr1, token, p):
q = right_binary_ops['Apply']
if q < p:
return None
self.consume()
expr2 = self.parse_exp(q)
expr3 = Node('List', Number('1'))
return Node('Apply', expr1, expr2, expr3)
def e_Function(self, expr1, token, p):
q = postfix_ops['Function']
if q < p:
return None
# postfix or right-binary determined by symbol
self.consume()
if token.text == '&':
return Node('Function', expr1)
else:
expr2 = self.parse_exp(q)
return Node('Function', expr1, expr2)
def e_RawColon(self, expr1, token, p):
head_name = expr1.get_head_name()
if head_name == 'Symbol':
head = 'Pattern'
elif head_name in ('Blank', 'BlankSequence', 'BlankNullSequence', 'Pattern', 'Optional'):
head = 'Optional'
else:
return None
q = all_ops[head]
if p == 151:
return None
self.consume()
expr2 = self.parse_exp(q + 1)
return Node(head, expr1, expr2)
def e_Semicolon(self, expr1, token, p):
q = flat_binary_ops['CompoundExpression']
if q < p:
return None
self.consume()
# XXX this has to come before call to self.next()
pos = self.tokeniser.pos
messages = list(self.feeder.messages)
# So that e.g. 'x = 1;' doesn't wait for newline in the frontend
tag = self.next().tag
if tag == 'END' and self.bracket_depth == 0:
expr2 = Symbol('Null')
return Node('CompoundExpression', expr1, expr2).flatten()
# XXX look for next expr otherwise backtrack
try:
expr2 = self.parse_exp(q + 1)
except TranslateError:
self.backtrack(pos)
self.feeder.messages = messages
expr2 = Symbol('Null')
return Node('CompoundExpression', expr1, expr2).flatten()
def e_Minus(self, expr1, token, p):
q = left_binary_ops['Subtract']
if q < p:
return None
self.consume()
expr2 = self.parse_exp(q + 1)
if isinstance(expr2, Number) and not expr2.value.startswith('-'):
expr2.value = '-' + expr2.value
else:
expr2 = Node('Times', Number('-1'), expr2).flatten()
return Node('Plus', expr1, expr2).flatten()
def e_TagSet(self, expr1, token, p):
q = all_ops['Set']
if q < p:
return None
self.consume()
expr2 = self.parse_exp(q + 1)
# examine next token
token = self.next_noend()
tag = token.tag
if tag == 'Set':
head = 'TagSet'
elif tag == 'SetDelayed':
head = 'TagSetDelayed'
elif tag == 'Unset':
head = 'TagUnset'
else:
self.tokeniser.sntx_message(token.pos)
raise InvalidSyntaxError()
self.consume()
if head == 'TagUnset':
return Node(head, expr1, expr2)
expr3 = self.parse_exp(q + 1)
return Node(head, expr1, expr2, expr3)
def e_Unset(self, expr1, token, p):
q = all_ops['Set']
if q < p:
return None
self.consume()
return Node('Unset', expr1)
def e_Derivative(self, expr1, token, p):
q = postfix_ops['Derivative']
if q < p:
return None
n = 0
while self.next().tag == 'Derivative':
self.consume()
n += 1
head = Node('Derivative', Number(str(n)))
return Node(head, expr1)
def e_Divide(self, expr1, token, p):
q = left_binary_ops['Divide']
if q < p:
return None
self.consume()
expr2 = self.parse_exp(q + 1)
return Node('Times', expr1, Node('Power', expr2, Number('-1'))).flatten()
def e_Alternatives(self, expr1, token, p):
q = flat_binary_ops['Alternatives']
if q < p:
return None
self.consume()
expr2 = self.parse_exp(q + 1)
return Node('Alternatives', expr1, expr2).flatten()
def e_MessageName(self, expr1, token, p):
leaves = [expr1]
while self.next().tag == 'MessageName':
self.consume()
token = self.next()
if token.tag == 'Symbol':
# silently convert Symbol to String
self.consume()
leaf = String(token.text)
elif token.tag == 'String':
leaf = self.p_String(token)
else:
self.tokeniser.sntx_message(token.pos)
raise InvalidSyntaxError()
leaves.append(leaf)
return Node('MessageName', *leaves)
# B methods
#
# b_xxx methods are called from parse_box.
# They expect args (Node, Token precedence) and return Node or None.
# The first argument may be None if the LHS is absent.
# Used for boxes.
def b_SqrtBox(self, box0, token, p):
if box0 is not None:
return None
self.consume()
q = misc_ops['SqrtBox']
box1 = self.parse_box(q)
if self.next().tag == 'OtherscriptBox':
self.consume()
box2 = self.parse_box(q)
return Node('RadicalBox', box1, box2)
else:
return Node('SqrtBox', box1)
def b_SuperscriptBox(self, box1, token, p):
q = misc_ops['SuperscriptBox']
if q < p:
return None
if box1 is None:
box1 = String('')
self.consume()
box2 = self.parse_box(q)
if self.next().tag == 'OtherscriptBox':
self.consume()
box3 = self.parse_box(misc_ops['SubsuperscriptBox'])
return Node('SubsuperscriptBox', box1, box3, box2)
else:
return Node('SuperscriptBox', box1, box2)
def b_SubscriptBox(self, box1, token, p):
q = misc_ops['SubscriptBox']
if q < p:
return None
if box1 is None:
box1 = String('')
self.consume()
box2 = self.parse_box(q)
if self.next().tag == 'OtherscriptBox':
self.consume()
box3 = self.parse_box(misc_ops['SubsuperscriptBox'])
return Node('SubsuperscriptBox', box1, box2, box3)
else:
return Node('SubscriptBox', box1, box2)
def b_UnderscriptBox(self, box1, token, p):
q = misc_ops['UnderscriptBox']
if q < p:
return None
if box1 is None:
box1 = String('')
self.consume()
box2 = self.parse_box(q)
if self.next().tag == 'OtherscriptBox':
self.consume()
box3 = self.parse_box(misc_ops['UnderoverscriptBox'])
return Node('UnderoverscriptBox', box1, box2, box3)
else:
return Node('UnderscriptBox', box1, box2)
def b_FractionBox(self, box1, token, p):
q = misc_ops['FractionBox']
if q < p:
return None
if box1 is None:
box1 = String('')
self.consume()
box2 = self.parse_box(q + 1)
return Node('FractionBox', box1, box2)
def b_FormBox(self, box1, token, p):
q = misc_ops['FormBox']
if q < p:
return None
if box1 is None:
box1 = Symbol('StandardForm') # RawForm
elif is_symbol_name(box1.value):
box1 = Symbol(box1.value, context=None)
else:
box1 = Node('Removed', String('$$Failure'))
self.consume()
box2 = self.parse_box(q)
return Node('FormBox', box2, box1)
def b_OverscriptBox(self, box1, token, p):
q = misc_ops['OverscriptBox']
if q < p:
return None
if box1 is None:
box1 = String('')
self.consume()
box2 = self.parse_box(q)
if self.next().tag == 'OtherscriptBox':
self.consume()
box3 = self.parse_box(misc_ops['UnderoverscriptBox'])
return Node('UnderoverscriptBox', box1, box3, box2)
else:
return Node('OverscriptBox', box1, box2)