def parse(seq):
'Sequence(Token) -> object'
unarg = lambda f: lambda args: f(*args)
tokval = lambda x: x.value
flatten = lambda list: sum(list, [])
n = lambda s: a(Token('Name', s)) >> tokval
op = lambda s: a(Token('Op', s)) >> tokval
op_ = lambda s: skip(op(s))
date = some(lambda s: a(Token('Date', s))).named('date') >> tokval
id = some(lambda t: t.type in ['Name', 'Number', 'String']).named(
'id') >> tokval
make_chart_attr = lambda args: DefAttrs(u'chart', [Attr(*args)])
node_id = id # + maybe(port)
term = date + op_('-') + date
value = (id | term | date)
a_list = (id + maybe(op_('=') + id) + skip(maybe(op(','))) >> unarg(Attr))
attr_list = (many(op_('[') + many(a_list) + op_(']')) >> flatten)
chart_attr = id + (op_('=') | op_(':')) + value >> make_chart_attr
node_stmt = node_id + attr_list >> unarg(Node)
stmt = (chart_attr | node_stmt)
stmt_list = many(stmt + skip(maybe(op(';'))))
chart = (maybe(n('diagram')) + maybe(id) + op_('{') + stmt_list + op_('}')
>> unarg(Chart))
dotfile = chart + skip(finished)
return dotfile.parse(seq)
def test_str(self):
self.assertEqual(tokenize('"a\\"b\\_c"'),
[Token('STRING', '"a\\"b\\_c"'),
Token('NL', '')])
self.assertEqual(tokenize("'a\\'b\\_c'"),
[Token('STRING', "'a\\'b\\_c'"),
Token('NL', '')])
def parse(seq):
'Sequence(Token) -> object'
unarg = lambda f: lambda args: f(*args)
tokval = lambda x: x.value
flatten = lambda list: sum(list, [])
value_flatten = lambda l: sum([[l[0]]] + list(l[1:]), [])
n = lambda s: a(Token('Name', s)) >> tokval
op = lambda s: a(Token('Op', s)) >> tokval
op_ = lambda s: skip(op(s))
id = some(lambda t:
t.type in ['Name', 'Number', 'Color', 'String']).named('id') >> tokval
make_chart_attr = lambda args: DefAttrs(u'chart', [Attr(*args)])
node_id = id # + maybe(port)
pair = (
op_('(') + id + skip(maybe(op(','))) + id + op_(')')
>> tuple)
value = (id | pair)
value_list = (
value +
many(op_(',') + value)
>> value_flatten)
a_list = (
id +
maybe(op_('=') + id) +
skip(maybe(op(',')))
>> unarg(Attr))
attr_list = (
many(op_('[') + many(a_list) + op_(']'))
>> flatten)
chart_attr = id + (op_('=') | op_(':')) + value_list >> make_chart_attr
node_stmt = node_id + attr_list >> unarg(Node)
stmt = (
chart_attr
| node_stmt
)
stmt_list = many(stmt + skip(maybe(op(';'))))
chart_type = (
n('p') | n('pie') | n('piechart')
| n('p3') | n('pie3d') | n('piechart_3d')
| n('lc') | n('line') | n('linechart')
| n('lxy') | n('linechartxy')
| n('bhs') | n('holizontal_barchart')
| n('bvs') | n('vertical_barchart')
| n('bhg') | n('holizontal_bargraph')
| n('bvg') | n('vertical_bargraph')
| n('v') | n('venn') | n('venndiagram')
| n('s') | n('plot') | n('plotchart')
)
chart = (
chart_type +
maybe(id) +
op_('{') +
stmt_list +
op_('}')
>> unarg(Chart))
dotfile = chart + skip(finished)
return dotfile.parse(seq)
def parse(sequence, query):
tokval = lambda x: x.value
toktype = lambda t: (some(lambda x: x.type == t).named('(type %s)' % t) >>
tokval)
operation = lambda s: a(Token('Op', s)) >> tokval
operation_ = lambda s: skip(operation(s))
create_param = lambda param_name: query.get_aliased_param(param_name)
make_and = lambda params: And(params[0], params[1])
make_or = lambda params: Or(params[0], params[1])
make_not = lambda inner: Not(inner)
word = toktype('Word')
inner_bracket = forward_decl()
left_of_and = forward_decl()
right_of_and = forward_decl()
left_of_or = forward_decl()
not_ = forward_decl()
bracket = operation_('(') + inner_bracket + operation_(')')
and_ = left_of_and + operation_('&') + right_of_and >> make_and
or_ = left_of_or + operation_('|') + inner_bracket >> make_or
param = word >> create_param
not_.define(operation_('!') + (bracket | param))
not_ = not_ >> make_not
left_of_or.define(and_ | bracket | not_ | param)
left_of_and.define(bracket | not_ | param)
right_of_and.define(left_of_and)
inner_bracket.define(or_ | and_ | bracket | not_ | param)
definition = (bracket | inner_bracket) + finished
return definition.parse(sequence)
def language_element(key, tok_type, combinator=a):
"""
Parser to match language element by using a certain combinator
:param key: exact key of the token, e.g.: `begin`, `model`, `)`
:param tok_type: predefined token type to be matched
:param combinator: use the combinator to create a parser
:return: a parser that matches elements using the above condition
"""
if combinator == a:
return combinator(Token(tok_type, key))
elif combinator == some:
return combinator(lambda tok: tok == Token(tok_type, key))
elif combinator == maybe:
return combinator(a(Token(tok_type, key)))
else:
raise Exception("Parser creation error")
def test_error_info(self):
tokenize = make_tokenizer([
('keyword', (r'(is|end)',)),
('id', (r'[a-z]+',)),
('space', (r'[ \t]+',)),
('nl', (r'[\n\r]+',)),
])
try:
list(tokenize('f is ф'))
except LexerError as e:
self.assertEqual(six.text_type(e),
'cannot tokenize data: 1,6: "f is \u0444"')
else:
self.fail('must raise LexerError')
sometok = lambda type: some(lambda t: t.type == type)
keyword = lambda s: a(Token('keyword', s))
id = sometok('id')
is_ = keyword('is')
end = keyword('end')
nl = sometok('nl')
equality = id + skip(is_) + id >> tuple
expr = equality + skip(nl)
file = many(expr) + end
msg = """\
spam is eggs
eggs isnt spam
end"""
toks = [x for x in tokenize(msg) if x.type != 'space']
try:
file.parse(toks)
except NoParseError as e:
self.assertEqual(e.msg,
"got unexpected token: 2,11-2,14: id 'spam'")
self.assertEqual(e.state.pos, 4)
self.assertEqual(e.state.max, 7)
# May raise KeyError
t = toks[e.state.max]
self.assertEqual(t, Token('id', 'spam'))
self.assertEqual((t.start, t.end), ((2, 11), (2, 14)))
else:
self.fail('must raise NoParseError')
def test_lexer():
assert list(tokenize_property('P17')) == [Token('PID', 'P17')]
assert list(tokenize_property('P17/P297')) == [
Token('PID', 'P17'),
Token('SLASH', '/'),
Token('PID', 'P297')
]
assert list(tokenize_property('(P17/P297)')) == ([
Token('LBRA', '('),
Token('PID', 'P17'),
Token('SLASH', '/'),
Token('PID', 'P297'),
Token('RBRA', ')')
])
def test_lexer(self):
self.assertEqual(list(tokenize_property('P17')), [Token('PID', 'P17')])
self.assertEqual(
list(tokenize_property('P17/P297')),
[Token('PID', 'P17'),
Token('SLASH', '/'),
Token('PID', 'P297')])
self.assertEqual(list(tokenize_property('(P17/P297)')), [
Token('LBRA', '('),
Token('PID', 'P17'),
Token('SLASH', '/'),
Token('PID', 'P297'),
Token('RBRA', ')')
])
def test_id(self):
with self.assertRaises(LexerError):
tokenize('-abc')
with self.assertRaises(LexerError):
tokenize('_abc')
with self.assertRaises(LexerError):
tokenize('abc-')
with self.assertRaises(LexerError):
tokenize('abc_')
self.assertEqual(
tokenize('abc_d0-3'),
[Token('ID', 'abc_d0-3'), Token('NL', '')])
self.assertEqual(
tokenize('`1.233 444\\` ee`'),
[Token('QUOTE_ID', '`1.233 444\\` ee`'),
Token('NL', '')])
with self.assertRaises(LexerError):
tokenize('01-')
def create_grammar():
tokval = lambda x: x.value
toktype = lambda t: some(lambda x: x.type == t) >> tokval
op = lambda s: a(Token('Op', s)) >> tokval
op_ = lambda s: skip(op(s))
n = lambda s: a(Token('Name', s)) >> tokval
null = n('null')
true = n('true')
false = n('false')
number = toktype('Number')
string = toktype('String')
value = forward_decl()
member = string + op_(':') + value
object_ = (op_('{') + maybe(member + many(op_(',') + member)) + op_('}'))
array = (op_('[') + maybe(value + many(op_(',') + value)) + op_(']'))
value.define(null | true | false | object_ | array | number | string)
json_text = object_ | array
json_file = json_text + skip(finished)
return json_file
def create_parser():
# operator: '~=' | '>=' | '<=' | '<' | '>' | '='
operator = some(lambda tok: tok.type == 'CMP') >> choose_class
# value: STRING | WORD
word = some(lambda tok: tok.type == 'WORD') >> Text
string = some(lambda tok: tok.type == 'STRING') >> QuotedText
value = string | word
# function: WORD '(' ')'
open_brace = skip(a(Token('BR', '(')))
close_brace = skip(a(Token('BR', ')')))
function = word + open_brace + close_brace >> Function
# field_expr: WORD operator value
fieldexpr = (word + operator + (function | value)) >> (lambda x: x[1]
([x[0], x[2]]))
OR = a(Token('OP', 'OR')) >> choose_class
AND = a(Token('OP', 'AND')) >> choose_class
def eval(data):
arg1, lst = data
for f, arg2 in lst:
arg1 = f([arg1, arg2])
return arg1
def eval(data):
lft, args = data
return reduce(lambda arg1, (f, arg2): f([arg1, arg2]), args, lft)
expr = forward_decl()
basexpr = open_brace + expr + close_brace | fieldexpr
andexpr = (basexpr + many(AND + basexpr)) >> eval
orexpr = (andexpr + many(OR + andexpr)) >> eval
expr.define(orexpr)
return expr
def _parse(seq):
const = lambda x: lambda _: x
tokval = lambda x: x.value
toktype = lambda t: some(lambda x: x.type == t) >> tokval
op = lambda s: a(Token(u'Op', s)) >> tokval
op_ = lambda s: skip(op(s))
def make_string(args):
context, value = args
if not context: context = 'any:'
return String(unescape_str(value[1:-1]), context[:-1])
def make_regex(args):
context, value = args
value, modifiers = value.rsplit('/', 1)
value = value[1:]
if not context: context = 'any:'
return Regex(unescape_regex(value), modifiers, context[:-1])
def make_or(args):
return Or(*args)
def make_and(args):
return And(*args)
def make_not(x):
return Not(x)
context = maybe(toktype(u'Prefix'))
string = (context + toktype(u'String')) >> make_string
regex = (context + toktype(u'Regex')) >> make_regex
par_term = forward_decl()
simple_term = forward_decl()
term = forward_decl()
not_term = forward_decl()
and_term = forward_decl()
or_term = forward_decl()
par_term.define(op_(u'(') + term + op_(u')'))
simple_term.define(par_term | string | regex)
not_term.define(op_('not') + not_term >> make_not | simple_term)
and_term.define(not_term + op_('and') + and_term >> make_and | not_term)
or_term.define(and_term + op_('or') + or_term >> make_or | and_term)
term.define(or_term)
eof = skip(toktype(u'EOF'))
filter_expr = (term + eof) | (eof >> const(Any()))
return filter_expr.parse(seq)
def parse(seq):
"""Sequence(Token) -> object"""
unarg = lambda f: lambda args: f(*args)
tokval = lambda x: x.value
flatten = lambda list: sum(list, [])
n = lambda s: a(Token(u'Name', s)) >> tokval
op = lambda s: a(Token(u'Op', s)) >> tokval
op_ = lambda s: skip(op(s))
id_types = [u'Name', u'Number', u'String']
id = some(lambda t: t.type in id_types).named(u'id') >> tokval
make_graph_attr = lambda args: DefAttrs(u'graph', [Attr(*args)])
make_edge = lambda x, xs, attrs: Edge([x] + xs, attrs)
node_id = id # + maybe(port)
a_list = (
id + maybe(op_(u'=') + id) + skip(maybe(op(u','))) >> unarg(Attr))
attr_list = (many(op_(u'[') + many(a_list) + op_(u']')) >> flatten)
attr_stmt = (
(n(u'graph') | n(u'node') | n(u'edge')) + attr_list >> unarg(DefAttrs))
graph_attr = id + op_(u'=') + id >> make_graph_attr
node_stmt = node_id + attr_list >> unarg(Node)
# We use a forward_decl becaue of circular definitions like (stmt_list ->
# stmt -> subgraph -> stmt_list)
subgraph = forward_decl()
edge_rhs = skip(op(u'->') | op(u'--')) + (subgraph | node_id)
edge_stmt = ((subgraph | node_id) + oneplus(edge_rhs) + attr_list >>
unarg(make_edge))
stmt = (attr_stmt | edge_stmt | subgraph | graph_attr | node_stmt)
stmt_list = many(stmt + skip(maybe(op(u';'))))
subgraph.define(
skip(n(u'subgraph')) + maybe(id) + op_(u'{') + stmt_list +
op_(u'}') >> unarg(SubGraph))
graph = (maybe(n(u'strict')) + maybe(n(u'graph') | n(u'digraph')) +
maybe(id) + op_(u'{') + stmt_list + op_(u'}') >> unarg(Graph))
dotfile = graph + skip(finished)
return dotfile.parse(seq)
def parse(seq):
'Sequence(Token) -> object'
tokval = lambda x: x.value
op = lambda s: a(Token('Op', s)) >> tokval
op_ = lambda s: skip(op(s))
id = some(lambda t: t.type in ['Name', 'Number', 'Color', 'String']).named(
'id') >> tokval
date = some(lambda t: t.type == 'Date').named('date') >> tokval
make_node = lambda args: Node(*args)
node_stmt = id + op_(':') + date + maybe(op_('-') + date) >> make_node
chart = (many(node_stmt + skip(maybe(op(';')))) >> Chart)
dotfile = chart + skip(finished)
return dotfile.parse(seq)
def lex_string_expr(string):
"""Lex a string expression."""
tokenizer = lexer.make_tokenizer([
('concat', [r'#']),
('string', [r'"[^"]+"']),
('name', [r'[A-Za-z_][A-Za-z_0-9\-:?\'\.\s]*']),
('space', [r'[ \t\r\n]+']),
])
try:
return remove_whitespace_tokens(tokenizer(string))
except lexer.LexerError:
# If we fail to lex the string, it is not a valid string expression so
# just return it as a single token
return [Token('string', string)]
def _parse_rule(seq):
tokval = lambda x: x.value
toktype = lambda t: some(lambda x: x.type == t) >> tokval
sep = lambda s: a(Token(u'Sep', s)) >> tokval
s_sep = lambda s: skip(sep(s))
level = toktype(u'Level')
comparator = toktype(u'Comparator') >> COMPARATORS.get
number = toktype(u'Number') >> float
historical = toktype(u'Historical')
unit = toktype(u'Unit')
operator = toktype(u'Operator')
logical_operator = toktype(u'LogicalOperator') >> LOGICAL_OPERATORS.get
exp = comparator + (
(number + maybe(unit)) | historical) + maybe(operator + number)
rule = (level + s_sep(':') + exp + many(logical_operator + exp))
overall = rule + skip(finished)
return overall.parse(seq)
def test_error_info():
tokenize = make_tokenizer([
Spec('keyword', r'(is|end)'),
Spec('id', r'[a-z]+'),
Spec('space', r'[ \t]+'),
Spec('nl', r'[\n\r]+'),
])
try:
list(tokenize('f is ф'))
except LexerError as e:
pass
else:
ok_(False, 'must raise LexerError')
keyword = lambda s: tok('keyword', s)
id = tok('id')
is_ = keyword('is')
end = keyword('end')
nl = tok('nl')
equality = id + skip(is_) + id >> tuple
expr = equality + skip(nl)
file = many(expr) + end
msg = """\
rake is eggs
eggs isnt spam
end"""
toks = [x for x in tokenize(msg) if x.type != 'space']
try:
file.parse(toks)
except ParserError as e:
msg, pos, i = e.args
eq_(msg, "got unexpected token: id 'spam'")
eq_(pos, ((2, 11), (2, 14)))
# May raise KeyError
t = toks[i]
eq_(t, Token('id', 'spam'))
else:
ok_(False, 'must raise ParserError')
def parse(seq):
'Sequence(Token) -> grammar dict'
make_patterns = lambda x: ('patterns', x)
n = lambda s: a(Token('Name', s)) >> tokval
# plan syntax
f_add = n('add') + skip(space)
f_apply = n('apply') + skip(space)
f_lookup = n('lookup') + skip(maybe(space))
f_parallel = n('parallel') + skip(space)
f_sequential = n('sequential') + skip(space)
f_firstmatch = n('firstmatch') + skip(space)
f_parse = n('parse') + skip(space) + name >> list
f_decompose = n('decompose') + skip(space) + name >> list
func_clause = oneplus(f_add | f_apply | f_lookup | f_parallel
| f_sequential | f_firstmatch) + maybe(
f_parse | f_decompose) >> unfoldl >> tuple
stage_clause = skip(
n('stage')) + skip(space) + name + skip(space) + func_clause + skip(
maybe(space))
return_clause = n('return') + skip(space) + skip(
n('if')) + skip(space) + name + skip(maybe(space))
for_clause = skip(n('for')) + skip(space) + name + skip(op(':')) + skip(
maybe(space)) + many(stage_clause | return_clause) >> tuple
plan_dict = oneplus(for_clause) >> dict
plan = n('plan') + skip(maybe(space)) + plan_dict >> tuple
# pattern syntax
pattern = skip(n('pattern')) + skip(space) + fullgloss_parser() + skip(
space) + skip(op('|')) + skip(space) + fullgloss_parser() + skip(
maybe(space)) >> unarg(Pattern)
sec_header = skip(n('section')) + skip(space) + name + skip(maybe(space))
section = sec_header + many(pattern)
sections = many(section) >> dict
patterns = sections >> make_patterns
grammar = plan + patterns + skip(finished) >> dict
return grammar.parse(seq)
def parse(seq):
"""Sequence(Token) -> object"""
id_tokens = ['Name', 'Number', 'String', 'RackHeight', 'Units']
rackitem_tokens = ['QuotedRackItem', 'RackItem']
tokval = lambda x: x.value
op = lambda s: a(Token('Op', s)) >> tokval
op_ = lambda s: skip(op(s))
_id = some(lambda t: t.type in id_tokens) >> tokval
keyword = lambda s: a(Token('Name', s)) >> tokval
rackheight = some(lambda t: t.type == 'RackHeight') >> tokval
number = some(lambda t: t.type == 'Number').named('number') >> tokval
rackitem = some(lambda t: t.type in rackitem_tokens) >> tokval
def make_num_rackitem(num, text, attr):
return RackItem(num, text.strip(), attr)
def make_nonnum_rackitem(text, attr):
return RackItem(None, text.strip(), attr)
def make_nonvalued_attr(rackheight):
return Attr(rackheight, None)
#
# parts of syntax
#
option_stmt = (
_id +
maybe(op_('=') + _id)
>> create_mapper(Attr)
)
option_list = (
maybe(op_('[') + option_stmt + many(op_(',') + option_stmt) + op_(']'))
>> create_mapper(oneplus_to_list, default_value=[])
)
# attributes statement::
# default_shape = box;
# default_fontsize = 16;
# 12U;
# ascending;
#
attribute_stmt = (
_id + op_('=') + _id
>> create_mapper(Attr)
)
rackheight_stmt = (
rackheight
>> make_nonvalued_attr
)
ascending_stmt = (
keyword('ascending')
>> make_nonvalued_attr
)
# field statement::
# 1: A
# 2: B [attr = value, attr = value];
# * C [attr = value, attr = value];
# * D [attr = value, attr = value];
#
numbered_field_item_stmt = (
number +
op_(':') +
rackitem +
option_list
>> create_mapper(make_num_rackitem)
)
nonnumbered_field_item_stmt = (
(op_('-') | op_('*')) +
rackitem +
option_list
>> create_mapper(make_nonnum_rackitem)
)
field_item_stmt = (
numbered_field_item_stmt |
nonnumbered_field_item_stmt
)
# rack statement::
# rack {
# 1: A;
# }
#
# rack definition
rack_inline_stmt = (
attribute_stmt |
rackheight_stmt |
field_item_stmt
)
rack_inline_stmt_list = (
many(rack_inline_stmt + skip(maybe(op(';'))))
)
rack_stmt = (
skip(keyword('rack')) +
maybe(_id) +
op_('{') +
rack_inline_stmt_list +
op_('}')
>> create_mapper(Rack)
)
# extension statement (plugin)::
# plugin attributes [name = Name];
#
extension_stmt = (
keyword('plugin') +
_id +
option_list
>> create_mapper(Extension)
)
#
# diagram statement::
# rackdiag {
# A;
# }
#
diagram_id = (
(keyword('diagram') | keyword('rackdiag')) +
maybe(_id)
>> list
)
diagram_inline_stmt = (
ascending_stmt |
extension_stmt |
rack_stmt |
field_item_stmt |
attribute_stmt |
rackheight_stmt
)
diagram_inline_stmt_list = (
many(diagram_inline_stmt + skip(maybe(op(';'))))
)
diagram = (
maybe(diagram_id) +
op_('{') +
diagram_inline_stmt_list +
op_('}')
>> create_mapper(Diagram)
)
dotfile = diagram + skip(finished)
return dotfile.parse(seq)
s.append(child)
added.add(child)
def __flatten_helper(lst: Iterable[Any]) -> Generator[Any, None, None]:
for x in lst:
if isinstance(x, Iterable):
yield from __flatten_helper(x)
else:
yield x
def __flatten(lst: Sequence[Any]) -> List[Any]:
return list(__flatten_helper(lst))
__toktype = lambda t: some(lambda x: x.type == t) # type: ignore
__tokop = lambda typ: skip(some(lambda x: x.type == typ))
__tokkw = lambda name: skip(a(Token('ID', name)))
__tokkw_keep = lambda name: a(Token('ID', name))
__identity = __wrap_result(lambda n, s, e: n)
def __make_array(n):
r = __collapse_results(n, depth=1)
re
if n is None:
return __Result([], (0, 0), (0, 0))
else:
return __Result(
[x.value for x in [n[0]] + n[1] if x.value is not None],
min(x.start for x in [n[0]] + n[1] if x.start > (0, 0)),
max(x.end for x in [n[0]] + n[1] if x.end > (0, 0)),
)
def parse(seq):
"""Sequence(Token) -> object"""
const = lambda x: lambda _: x
tokval = lambda x: x.value
toktype = lambda t: some(lambda x: x.type == t) >> tokval
op = lambda s: a(Token('OP', s)) >> tokval
op_ = lambda s: skip(op(s))
n = lambda s: a(Token('NAME', s)) >> tokval
def make_array(n):
if n is None:
return []
else:
return [n[0]] + n[1]
def make_object(n):
return dict(make_array(n))
def make_int(n):
return '%s' % int(n)
def make_real(n):
return '%s' % float(n)
def unescape(s):
std = {
'"': '"',
'\\': '\\',
'/': '/',
'b': '\b',
'f': '\f',
'n': '\n',
'r': '\r',
't': '\t',
}
def sub(m):
if m.group('standard') is not None:
return std[m.group('standard')]
else:
return unichr(int(m.group('unicode'), 16))
return re_esc.sub(sub, s)
def make_string(n):
return n
#return unescape(n[1:-1])
def make_all_models(models):
return dict(models)
# all_attrs = []
# for i in attrs:
# attr = i[0]
# if attr not in all_attrs:
# all_attrs.append(attr)
# else:
# raise Exception('Attribute %s is already defined in class'%attr)
def make_fields(n):
#return dict(n)
return Field(n)
def make_params(n):
return n
null = toktype('NONE') >> const("None")
true = toktype('TRUE') >> const("True")
false = toktype('FALSE') >> const("False")
number = toktype('INT') >> make_int
real = toktype('REAL') >> make_real
string = toktype('STRING') >> make_string
value = forward_decl()
name = toktype('NAME')
field = toktype('FIELD') + maybe(op_('(') + many(value) +
op_(')')) >> tuple
member = string + op_(':') + value >> tuple
attrs = forward_decl()
params = forward_decl()
models = many(name + op_('::') + many(attrs)) >> make_all_models
attrs.define(name + op_(':') + field + many(params) >> make_fields)
params.define(name + op_('=') + value >> tuple)
value.define(null | true | false | name | number | real | string)
parser_text = models
parser_file = parser_text + skip(finished)
return parser_file.parse(seq)
def make_parser(type_, value, param, parse):
nonlocal actor_set
if type_ not in valid_token_types and type_ not in ('ACTOR', 'VALUE'):
exit_bad_rule(f'"{type_}" is not a valid token type')
if type_ == 'ACTOR':
actor_set = True
if value:
exit_bad_rule("ACTOR token cannot be used with value check")
if type_ == 'VALUE':
if value:
exit_bad_rule("VALUE token cannot be used with value check")
if param is None or parse not in ('INT', 'FLOAT', 'BOOL', 'STRING'):
exit_bad_rule("VALUE token not of form VALUE(parameter=TYPE)")
parse = f'__assert_{parse.lower()}(__value)'
if value:
try:
value = eval_(value)
except:
exit_bad_rule("Failed to parse custom rule value check")
if type_ == 'ID':
out.append(skip(a(Token('ID', value)) | a(Token('QUOTE_ID', value))))
else:
out.append(skip(a(Token(type_, value))))
return True
elif type_ == 'ACTOR':
if param is not None:
@__wrap_result
def handle_secondary_only(t, start, end):
return {'.actor_secondary': '' if t is None else t}
if param != '.name':
exit_bad_rule("ACTOR token parameter must be .name")
if parse is None:
exit_bad_rule("ACTOR .name must be set if present")
try:
name = eval_(parse)
except:
exit_bad_rule("Failed to parse actor primary name")
params['.actor_name'] = name
out.append(maybe(__tokop('AT') + id_) >> handle_secondary_only)
else:
@__wrap_result
def handle_actor(t, start, end):
name, secondary = t
return {'.actor_name': name, '.actor_secondary': '' if secondary is None else secondary}
out.append((id_ + maybe(__tokop('AT') + id_)) >> handle_actor)
elif not param:
out.append(skip(some(lambda x: x.type == type_ if type_ != 'ID' else x.type in ('ID', 'QUOTE_ID'))))
else:
parse = parse or '__value'
try:
f = eval_(f'lambda __type, __value: {parse}')
except:
exit_bad_rule("Failed to parse custom rule")
@__wrap_result
def inner(t, start, end):
try:
return {param: f(type_, t)}
except Exception as e:
emit_error(str(e), start, end)
raise LogError()
value_wrapper = {
'INT': __int,
'FLOAT': __float,
'BOOL': __bool,
'STRING': __string,
'ID': __identifier,
}.get(type_, __identity)
if param[0] == '.':
value_wrapper = __identity
if type_ != 'VALUE':
out.append(some(lambda x: x.type == type_ if type_ != 'ID' else x.type in ('ID', 'QUOTE_ID')) >> value_wrapper >> inner)
else:
out.append(__value >> value_wrapper >> inner)
return False
def parse(seq):
"""Sequence(Token) -> object"""
const = lambda x: lambda _: x
tokval = lambda x: x.value
toktype = lambda t: some(lambda x: x.type == t) >> tokval
op = lambda s: a(Token('Op', s)) >> tokval
op_ = lambda s: skip(op(s))
n = lambda s: a(Token('Name', s)) >> tokval
def make_array(n):
if n is None:
return []
else:
return [n[0]] + n[1]
def make_object(n):
return dict(make_array(n))
def make_number(n):
try:
return int(n)
except ValueError:
return float(n)
def unescape(s):
std = {
'"': '"',
'\\': '\\',
'/': '/',
'b': '\b',
'f': '\f',
'n': '\n',
'r': '\r',
't': '\t',
}
def sub(m):
if m.group('standard') is not None:
return std[m.group('standard')]
else:
return chr(int(m.group('unicode'), 16))
return re_esc.sub(sub, s)
def make_string(n):
return unescape(n[1:-1])
null = n('null') >> const(None)
true = n('true') >> const(True)
false = n('false') >> const(False)
number = toktype('Number') >> make_number
string = toktype('String') >> make_string
value = forward_decl()
member = string + op_(':') + value >> tuple
object = (op_('{') + maybe(member + many(op_(',') + member)) + op_('}') >>
make_object)
array = (op_('[') + maybe(value + many(op_(',') + value)) + op_(']') >>
make_array)
value.define(null | true | false | object | array | number | string)
json_text = object | array
json_file = json_text + skip(finished)
return json_file.parse(seq)
def make_token(self, token_type, value):
"""Create a token type with a value."""
return Token(token_type, value, (self.last_lnum, self.lastpos),
(self.lnum, self.pos))
tokenize = make_tokenizer([
(u'keyword', (ur'(is|end)',)),
(u'id', (ur'[a-z]+',)),
(u'space', (ur'[ \t]+',)),
(u'nl', (ur'[\n\r]+',)),
])
try:
list(tokenize(u'f is ф'))
except LexerError, e:
self.assertEqual(unicode(e),
u'cannot tokenize data: 1,6: "f is \u0444"')
else:
self.fail(u'must raise LexerError')
sometok = lambda type: some(lambda t: t.type == type)
keyword = lambda s: a(Token(u'keyword', s))
id = sometok(u'id')
is_ = keyword(u'is')
end = keyword(u'end')
nl = sometok(u'nl')
equality = id + skip(is_) + id >> tuple
expr = equality + skip(nl)
file = many(expr) + end
msg = """\
spam is eggs
eggs isnt spam
end"""
toks = [x for x in tokenize(msg) if x.type != u'space']
def parse(seq):
"""Sequence(Token) -> object"""
tokval = lambda x: x.value
op = lambda s: a(Token('Op', s)) >> tokval
op_ = lambda s: skip(op(s))
_id = some(lambda t: t.type in ['Name', 'Number', 'String']) >> tokval
keyword = lambda s: a(Token('Name', s)) >> tokval
separator = some(lambda t: t.type == 'Separator') >> tokval
def make_separator(sep):
return Separator(sep[0:3], sep[3:-3].strip())
#
# parts of syntax
#
option_stmt = (_id + maybe(op_('=') + _id) >> create_mapper(Attr))
option_list = (maybe(
op_('[') + option_stmt + many(op_(',') + option_stmt) + op_(']')) >>
create_mapper(oneplus_to_list, default_value=[]))
# attributes statement::
# default_shape = box;
# default_fontsize = 16;
#
attribute_stmt = (_id + op_('=') + _id >> create_mapper(Attr))
# node statement::
# A;
# B [attr = value, attr = value];
#
node_stmt = (_id + option_list >> create_mapper(Node))
# separator statement::
# === message ===
# ... message ...
#
separator_stmt = (separator >> make_separator)
# edge statement::
# A -> B;
# C -> D {
# D -> E;
# }
#
edge_block = forward_decl()
edge_relation = (op('<<--') | op('<--') | op('<<-') | op('<-') | op('->')
| op('->>') | op('-->') | op('-->>') | op('=>'))
edge_stmt = (_id + edge_relation + _id + many(edge_relation + _id) +
option_list + maybe(edge_block) >> create_mapper(Edge))
edge_block_inline_stmt_list = (many(edge_stmt + skip(maybe(op(';')))
| separator_stmt))
edge_block.define(
op_('{') + edge_block_inline_stmt_list + op_('}') >> Statements)
# group statement::
# group {
# A;
# }
#
group_inline_stmt_list = (many((attribute_stmt | node_stmt) +
skip(maybe(op(';')))))
group_stmt = (skip(keyword('group')) + skip(maybe(_id)) + op_('{') +
group_inline_stmt_list + op_('}') >> Group)
# combined fragment (alt, loop) statement::
# loop {
# A -> B;
# }
# alt {
# D -> E;
# }
#
fragment_stmt = forward_decl()
fragment_inline_stmt = (attribute_stmt | fragment_stmt | edge_stmt
| node_stmt)
fragment_inline_stmt_list = (many(fragment_inline_stmt +
skip(maybe(op(';')))))
fragment_types = (keyword('alt') | keyword('loop'))
fragment_stmt.define(fragment_types + maybe(_id) + op_('{') +
fragment_inline_stmt_list +
op_('}') >> create_mapper(Fragment))
# extension statement (class, plugin)::
# class red [color = red];
# plugin attributes [name = Name];
#
extension_stmt = ((keyword('class') | keyword('plugin')) + _id +
option_list >> create_mapper(Extension))
# diagram statement::
# seqdiag {
# A -> B;
# }
#
diagram_id = (
(keyword('diagram') | keyword('seqdiag')) + maybe(_id) >> list)
diagram_inline_stmt = (extension_stmt | attribute_stmt | fragment_stmt
| group_stmt | edge_stmt | separator_stmt
| node_stmt)
diagram_inline_stmt_list = (many(diagram_inline_stmt +
skip(maybe(op(';')))))
diagram = (maybe(diagram_id) + op_('{') + diagram_inline_stmt_list +
op_('}') >> create_mapper(Diagram))
dotfile = diagram + skip(finished)
return dotfile.parse(seq)
def parse(self, seq):
const = lambda x: lambda _: x
unarg = lambda f: lambda x: f(*x)
tokval = lambda x: x.value
toktype = lambda s: some(lambda x: x.type == s) >> tokval
def sumstr(s):
#import pdb; pdb.set_trace()
return "".join(str(x) for x in s)
keyword = lambda s: some(lambda x: x.type == 'KEYWORD' and x.value.
lower() == s)
op = lambda s: a(Token('OP', s)) >> tokval
op_ = lambda s: skip(op(s))
onename = many(
toktype('NAME') | toktype('KEYWORD') | toktype('HEADER')
| toktype('NUMBER')) >> sumstr
name = toktype('NAME') | toktype('KEYWORD') | toktype('HEADER')
number = toktype('NUMBER') >> get_num
get_string = lambda v: v[1:-1]
string = (toktype('STRING') >> get_string) | (
toktype('CHAR') >> get_char)
nl = skip(toktype('NL'))
typed_func = lambda k, c: keyword(k) + op_('(') + c + op_(')')
simple_func = lambda k: typed_func(k, onename)
assign_func = lambda k: keyword(k) + op_('(') + name + op_('=') + (
name | string) + op_(')')
# statement components
vkey = op_('[') + many(name) + op_(']')
prefix = toktype('TARGET') | assign_func('if') | typed_func(
'platform', string)
context_statement = (keyword('context') + op_('(') + number +
op_(')')) | keyword('context')
index_statement = keyword('index') + op_('(') + onename + op_(
',') + number + op_(')')
deadkey = (keyword('deadkey')
| keyword('dk')) + op_('(') + (number | name) + op_(')')
context = context_statement | keyword('beep') | keyword('nul') | simple_func('use') | \
(index_statement >> AnyIndex) | string | (deadkey >> DeadKey) | \
simple_func('reset') | simple_func('save') | assign_func('set') | \
simple_func('outs')
match = (simple_func('any') >> AnyIndex) | string | (
deadkey >> DeadKey) | simple_func('notany')
# top level statements
stripspace = lambda s: s.replace(' ', '')
header = (((toktype('HEADER') >> stripspace) + string) |
(toktype('HEADERN') + number)) + nl
group = (simple_func('group') >> self.set_group) + maybe(
toktype('USINGKEYS')) + nl
store = maybe(many(prefix)) + simple_func('store') + many(string | simple_func('outs') | \
(vkey >> VKey) | (deadkey >> DeadKey) | keyword('beep')) + nl
begin = keyword('begin') + onename + op_('>') + simple_func('use') + nl
rule = maybe(many(prefix)) + maybe(many(match)) + maybe(toktype('PLUS') + \
((vkey >> VKey) | string | (simple_func('any') >> AnyIndex))) + op_('>') + many(context) + nl
matchrule = (keyword('match')
| keyword('nomatch')) + op_('>') + many(context) + nl
# a file is a sequence of certain types of statement
make_header = lambda s: Store([(Token('HEADER', 'header'), '&' + s[0]),
[s[1]]])
kmfile = many((header >> make_header) | (store >> self.make_store) | \
skip(begin >> self.store_begin)) + \
maybe(many(group | (rule >> self.make_rule) | \
(matchrule >> self.make_match_rule) | \
(store >> self.make_store))) + finished
return kmfile.parse(seq)
def tokenize(string: str) -> List[Token]:
t = make_tokenizer(__tokens)
pstack: List[Token] = []
indent = ['']
if string and string[-1] not in ('\r', '\n'):
string = string + '\n'
num_lines = len(re.findall(r'\r\n|\r|\n', string))
tokens: List[Token] = []
buffered: List[Token] = []
space_since_nl = False
buffering = 0
emit_token = lambda tok: (buffered if buffering else tokens).append(tok)
gen = peekable(t(string))
for x in gen:
assert x.start is not None
assert x.end is not None
start, end = x.start, x.end
if x.type == 'COMMENT':
continue
elif x.type == 'LPAREN':
pstack.append(x)
elif x.type == 'RPAREN':
if not pstack:
raise LexerError(start, 'no parentheses to close')
if pstack.pop().type != 'LPAREN':
raise LexerError(start, "expecting ']' but got ')'")
elif x.type == 'LSQUARE':
pstack.append(x)
elif x.type == 'RSQUARE':
if not pstack:
raise LexerError(start, 'no bracket to close')
if pstack.pop().type != 'LSQUARE':
raise LexerError(start, "expecting ')' but got ']'")
elif x.type == 'NL':
if pstack:
continue
space_since_nl = False
if tokens and tokens[-1].type == 'NL' and buffering != 2:
continue
x = Token('NL', '', start=start, end=end)
elif x.type == 'ID' and x.name == 'entrypoint':
if space_since_nl:
raise LexerError(start, 'entrypoint must be unindented')
buffering = 2
elif x.type == 'SP':
space_since_nl = True
nxt = gen.peek(None)
next_comment = (nxt is not None and nxt.type == 'COMMENT')
if buffering == 1 and not next_comment:
buffering = 0
if tokens and tokens[-1].type == 'NL' and not buffering and not next_comment:
indent_diff = __compare_indent(indent[-1], x.name, start)
if indent_diff < 0:
found = False
while indent:
s = indent.pop()
if s == x.name:
indent.append(s)
break
emit_token(Token('DEDENT', '', start=start, end=end))
if not indent:
raise LexerError(end, 'dedent to unknown level')
elif indent_diff > 0:
indent.append(x.name)
emit_token(Token('INDENT', '', start=start, end=end))
if not buffering and buffered:
tokens.extend(buffered)
buffered = []
continue
if x.type == 'NL' and buffering:
buffering = 1
elif buffering == 1:
buffering = 0
if x.type != 'INDENT' and tokens and tokens[-1].type == 'NL' and not space_since_nl \
and not buffering:
while len(indent) > 1:
s = indent.pop()
emit_token(Token('DEDENT', '', start=start, end=end))
if not buffering and buffered:
tokens.extend(buffered)
buffered = []
emit_token(x)
if pstack:
raise LexerError((num_lines + 1, 0), 'unclosed parentheses/brackets')
if buffering or buffered:
raise LexerError((num_lines + 1, 0), 'unexpecte end of file')
while indent[-1]:
indent.pop()
tokens.append(Token('DEDENT', '', start=(num_lines + 1, 0), end=(num_lines + 1, 0)))
return tokens
return make_hidCode_range('SysCode', rangeVals)
def make_consCode_range(rangeVals):
return make_hidCode_range('ConsCode', rangeVals)
## Base Rules
const = lambda x: lambda _: x
unarg = lambda f: lambda x: f(*x)
flatten = lambda list: sum(list, [])
tokenValue = lambda x: x.value
tokenType = lambda t: some(lambda x: x.type == t) >> tokenValue
operator = lambda s: a(Token('Operator', s)) >> tokenValue
parenthesis = lambda s: a(Token('Parenthesis', s)) >> tokenValue
eol = a(Token('EndOfLine', ';'))
def listElem(item):
return [item]
def listToTuple(items):
return tuple(items)
# Flatten only the top layer (list of lists of ...)
def oneLayerFlatten(items):
mainList = []
def parse(
seq: List[Token],
gen_actor: Callable[[str, str], Actor],
exported_tco: bool = True,
custom_action_parser_pfx: Optional[Parser] = None,
custom_action_parser_reg: Optional[Parser] = None,
custom_query_parser_op: Optional[Parser] = None,
custom_query_parser_pfx: Optional[Parser] = None,
custom_query_parser_reg: Optional[Parser] = None
) -> Tuple[List[RootNode], List[Actor]]:
actors: Dict[Tuple[str, str], Actor] = {}
nid = 0
def next_id() -> int:
nonlocal nid
rv, nid = nid, nid + 1
return rv
def check_function(p, type_, start, end):
if len(p) == 1:
p = p[0]
if isinstance(p, dict):
actor = (p.pop('.actor_name'), p.pop('.actor_secondary'))
name = p.pop('.name')
negated = p.pop('.negated')
params = pdict = p
prepare_params = False
else:
actor_name, actor_secondary, name, params = p
actor = (actor_name, actor_secondary or '')
negated = False
prepare_params = True
function_name = f'EventFlow{type_.capitalize()}{name}'
if actor not in actors:
actors[actor] = gen_actor(*actor)
mp = getattr(actors[actor], ['actions', 'queries'][type_ == 'query'])
if function_name not in mp:
emit_warning(f'no {type_} with name "{function_name}" found, using empty call', start, end)
if type_ == 'action':
actors[actor].register_action(Action(actor, function_name, []))
else:
actors[actor].register_query(Query(actor, function_name, [], IntType, False))
function = mp[function_name]
if prepare_params:
try:
pdict = function.prepare_param_dict([p for name, p in params if name is None])
for name, p in params:
if name is not None:
if name in pdict:
emit_warning(f'keyword argument name {name} matches positional argument name', start, end)
pdict[name] = p
except AssertionError as e:
emit_error(str(e), start, end)
raise LogError()
return function_name, function, pdict, negated
@__wrap_result
def make_action(n, start, end):
action_name, action, pdict, _ = check_function(n, 'action', start, end)
return (), (ActionNode(f'Event{next_id()}', action, pdict),)
@__wrap_result
def make_case(n, start, end):
if isinstance(n, tuple) and len(n) == 2:
return ([x.value for x in n[0]], n[1])
return n
def _get_query_num_values(query, start, end):
num_values = query.num_values
if num_values == 999999999:
emit_warning(f'maximum value for {query.name} unknown; assuming 50', start, end, print_source=False)
emit_warning(f'setting a maximum value in functions.csv may reduce generated bfevfl size', start, end)
num_values = 50
return num_values
@__wrap_result
def make_switch(n, start, end):
p, branches = n[:-1], n[-1]
cases = branches[0] + branches[2]
default = branches[1]
query_name, query, pdict, _ = check_function(p, 'query', start, end)
sw = SwitchNode(f'Event{next_id()}', query, pdict)
entrypoints = []
enum_values = {}
if query.rv.type.startswith('enum['):
enum_values_list = query.rv.type[5:-1].split(',')
enum_values = {v.strip(): i for i, v in enumerate(enum_values_list)}
for values, block in cases:
eps, node, connector = block
entrypoints.extend(eps)
sw.add_out_edge(node)
connector.add_out_edge(sw.connector)
for value in values:
if isinstance(value, str):
if not enum_values:
emit_error(f'Query "{query.name}" does not return an enum', start, end)
raise LogError()
if value not in enum_values:
emit_error(f'Enum "{value}" is not a valid return value of "{query.name}"', start, end)
raise LogError()
value = enum_values[value]
sw.add_case(node, value)
num_values = _get_query_num_values(query, start, end)
default_values = set(range(num_values)) - set(sum((v for v, n in cases), []))
if default_values:
if default is not None:
_, default, connector = default
connector.add_out_edge(sw.connector)
default_branch = default or sw.connector
sw.add_out_edge(default_branch)
for value in default_values:
sw.add_case(default_branch, value)
elif default:
emit_warning(f'default branch for {query_name} call is dead code, ignoring', start, end)
return entrypoints, (sw,)
@__wrap_result
def make_bool_function(p, start, end):
query_name, query, pdict, negated = check_function(p, 'query', start, end)
num_values = _get_query_num_values(query, start, end)
if num_values > 2:
emit_warning(f'call to {query_name} treated as boolean function but may not be', start, end)
return ((query, pdict), [({0}, query.inverted != negated), (set(range(1, num_values)), (not query.inverted) != negated)])
@__wrap_result
def make_in(p, start, end):
p, values = p[:-1], p[-1]
query_name, query, pdict, _ = check_function(p, 'query', start, end)
num_values = _get_query_num_values(query, start, end)
matched = set()
unmatched = set(range(num_values))
enum_values = {}
if query.rv.type.startswith('enum['):
enum_values_list = query.rv.type[5:-1].split(',')
enum_values = {v.strip(): i for i, v in enumerate(enum_values_list)}
for value in values:
if isinstance(value.value, str):
if not enum_values:
emit_error(f'Query "{query.name}" does not return an enum', start, end)
raise LogError()
if value.value not in enum_values:
emit_error(f'Enum "{value.value}" is not a valid return value of "{query.name}"', start, end)
raise LogError()
value.value = enum_values[value.value]
if 0 > value.value or num_values <= value.value:
emit_warning('{value.value} never returned by {query_name}, ignored', start, end)
continue
matched.add(value.value)
unmatched.remove(value.value)
if not matched or not unmatched:
emit_warning(f'always true or always false check', start, end)
return ((query, pdict), [(matched, True), (unmatched, False)])
@__wrap_result
def make_cmp(p, start, end):
p, op, value = p[:-2], p[-2], p[-1]
query_name, query, pdict, _ = check_function(p, 'query', start, end)
num_values = _get_query_num_values(query, start, end)
if isinstance(value.value, str):
if query.rv.type.startswith('enum['):
enum_values_list = query.rv.type[5:-1].split(',')
value.value = enum_values_list.index(value.value)
if value.value == -1:
emit_error(f'Enum "{value.value}" is not a valid return value of "{query.name}"', start, end)
raise LogError()
else:
emit_error(f'Query "{query.name}" does not return an enum', start, end)
raise LogError()
if op == '==' or op == '!=':
matched = {value.value} if 0 <= value.value < num_values else set()
unmatched = set(i for i in range(num_values) if i != value.value)
elif op == '<' or op == '>=':
matched = set(range(min(num_values, value.value)))
unmatched = set(range(value.value, num_values))
else:
matched = set(range(min(num_values, value.value + 1)))
unmatched = set(range(value.value + 1, num_values))
if op in ('!=', '>=', '>'):
matched, unmatched = unmatched, matched
if not matched or not unmatched:
emit_warning(f'always true or always false check', start, end)
return ((query, pdict), [(matched, True), (unmatched, False)])
def _predicate_replace(values, old, new):
for i in range(len(values)):
if values[i][1] == old:
values[i] = (values[i][0], new)
@__wrap_result
def make_or(p, start, end):
left, right = p
if isinstance(right, TypedValue):
left, right = right, left
if isinstance(left, TypedValue):
if isinstance(right, TypedValue):
return TypedValue(type=BoolType, value=left.value or right.value)
if not left.value:
return right
else:
return TypedValue(type=BoolType, value=True)
# todo: can probably optimize for smaller output
_predicate_replace(left[1], False, right)
return left
@__wrap_result
def make_and(p, start, end):
left, right = p
if isinstance(right, TypedValue):
left, right = right, left
if isinstance(left, TypedValue):
if isinstance(right, TypedValue):
return TypedValue(type=BoolType, value=left.value and right.value)
if left.value:
return right
else:
return TypedValue(type=BoolType, value=False)
# todo: can probably optimize for smaller output
_predicate_replace(left[1], True, right)
return left
@__wrap_result
def make_not(p, start, end):
if isinstance(p, TypedValue):
p.value = not p.value
return p
_predicate_replace(p[1], True, None)
_predicate_replace(p[1], False, True)
_predicate_replace(p[1], None, False)
return p
def _expand_table(table, current, next_):
((query, pdict), values) = table
sw = SwitchNode(f'Event{next_id()}', query, pdict)
for match, action in values:
if isinstance(action, tuple):
to = _expand_table(action, current, next_)
elif action:
to = current
else:
to = next_
for value in match:
sw.add_case(to, value)
sw.add_out_edge(to)
return sw
@__wrap_result
def make_ifelse(n, start, end):
if_, block, elifs, else_ = n
cond_branches = [(if_, block)] + elifs + ([(None, else_)] if else_ else [])
entrypoints = []
next_ = next_connector = ConnectorNode(f'Connector{next_id()}')
for table, body in cond_branches[::-1]:
eps, node, branch_connector = body
entrypoints.extend(eps)
if table is None:
next_ = node
next_connector = branch_connector
elif isinstance(table, TypedValue):
if table.value:
next_ = node
next_connector = branch_connector
else:
continue
else:
next_ = _expand_table(table, node, next_)
branch_connector.add_out_edge(next_.connector)
next_connector.add_out_edge(next_.connector)
next_connector = next_.connector
return entrypoints, (next_,)
@__wrap_result
def make_fork(n_, start, end):
for entrypoints, node, connector in n_:
for ep in entrypoints:
__replace_node(ep, connector, None)
__replace_node(node, connector, None)
eps = __flatten([ep for ep, _, _ in n_])
n = [x for _, x, _ in n_]
fork_id, join_id = next_id(), next_id()
join = JoinNode(f'Event{join_id}')
fork = ForkNode(f'Event{fork_id}', join, n)
for node in n:
fork.add_out_edge(node)
return eps, (fork, join)
@__wrap_result
def make_while(n, start, end):
table, (eps, node, connector) = n
next_connector = ConnectorNode(f'Connector{next_id()}')
if isinstance(table, TypedValue):
if table.value:
# while true
connector.add_out_edge(node)
return eps, (node, connector)
else:
# while false
if eps:
emit_error('entrypoints in while-false not supported', start, end)
raise LogError()
return [], (connector, connector)
else:
next_ = _expand_table(table, node, next_connector)
connector.add_out_edge(next_)
return eps, (next_, next_connector)
@__wrap_result
def make_do_while(n, start, end):
(eps, node, connector), table = n
next_connector = ConnectorNode(f'Connector{next_id()}')
if isinstance(table, TypedValue):
if table.value:
# do while true
connector.add_out_edge(node)
return eps, (node, connector)
else:
next_ = _expand_table(table, node, next_connector)
connector.add_out_edge(next_)
return eps, (node, next_connector)
@__wrap_result
def make_subflow_param(n, start, end):
return (n,)
@__wrap_result
def make_subflow(n, start, end):
ns, name, params = n
param_dict = {k[0][0]: k[0][1] for k in params}
return (), (SubflowNode(f'Event{next_id()}', ns or '', name, param_dict),)
@__wrap_result
def make_none(_, start, end):
return None
@__wrap_result
def make_return(_, start, end):
return (), (TerminalNode,)
@__wrap_result
def named_value(n, start, end):
return n
@__wrap_result
def unnamed_value(n, start, end):
return (None, n)
@__wrap_result
def make_param(n, start, end):
return n
def verify_params(name, root, params_list):
for param, type_, value in params_list:
if value is not None and type_ != value.type:
emit_error(f'variable definition for {param} in {name} is of type {type_} but has default value of type {value.type}')
raise LogError()
params = {param: type_ for param, type_, value in params_list}
for node in find_postorder(root):
if isinstance(node, (ActionNode, SwitchNode)):
for param, value in node.params.items():
if isinstance(value.value, Argument):
if value.value not in params:
emit_error(f'variable {value.value} not defined in flow {name}')
raise LogError()
expected_type = params[value.value]
actual_type = value.type
if param.startswith('EntryVariableKey'):
if param[16:].startswith('Int_'):
actual_type = IntType
elif param[16:].startswith('Bool_'):
actual_type = BoolType
elif param[16:].startswith('Float_'):
actual_type = FloatType
elif param[16:].startswith('String_'):
actual_type = StringType
else:
value.type = ArgumentType
if actual_type != AnyType and expected_type != actual_type:
emit_error(f'variable {value.value} has the wrong type, defined to be {expected_type} but used as {actual_type}')
raise LogError()
if isinstance(node, SubflowNode):
for param, value in node.params.items():
if isinstance(value.value, Argument):
if value.value not in params:
emit_error(f'variable {value.value} not defined in flow {name}')
raise LogError()
value.type = params[value.value]
@__wrap_result
def make_flow(n, start, end):
local, name, params, body = n
entrypoints, body_root, body_connector = body
verify_params(name, body_root, params)
vardefs = [RootNode.VarDef(name=n, type=t, initial_value=v.value if v else None) for n, t, v in params]
valueless_vardefs = [RootNode.VarDef(name=n, type=t, initial_value=None) for n, t, v in params]
node = RootNode(name, local is not None, False, vardefs)
for e in entrypoints:
e.vardefs = valueless_vardefs[:]
node.add_out_edge(body_root)
body_connector.add_out_edge(TerminalNode)
return list(entrypoints) + [node]
@__wrap_result
def link_ep_block(n, start, end):
connector = ConnectorNode(f'Connector{next_id()}')
ep, block_info = n
block_info = [x for x in block_info if x is not None]
if block_info:
eps, block = (__flatten(p) for p in zip(*(x for x in block_info if x is not None)))
else:
eps, block = [], ()
if not block:
if ep is not None:
ep_node = RootNode(ep, True, True, [])
ep_node.add_out_edge(connector)
eps.append(ep_node)
return (eps, connector, connector)
for n1, n2 in zip(block, block[1:] + [connector]):
n1_conn = n1
if isinstance(n1, tuple):
n1, n1_conn = n1
if isinstance(n2, tuple):
n2, _ = n2
if isinstance(n1, SwitchNode):
n1.connector.add_out_edge(n2)
else:
n1_conn.add_out_edge(n2)
if ep is not None:
ep_node = RootNode(ep, True, True, [])
ep_node.add_out_edge(block[0])
eps.append(ep_node)
return (eps, block[0], connector)
@__wrap_result
def link_block(n, start, end):
connector = ConnectorNode(f'Connector{next_id()}')
n = [n[0]] + n[1]
eps, blocks, connectors = zip(*n)
eps = __flatten(eps)
for connector, block in zip(connectors[:-1], blocks[1:]):
connector.add_out_edge(block)
return (eps, blocks[0], connectors[-1])
@__wrap_result
def collect_flows(n, start, end):
if n is None:
return []
else:
return __flatten([x for x in n if x is not None])
block = forward_decl()
# pass = PASS NL
pass_ = (__tokkw('pass') + __tokop('NL')) >> make_none
# return = RETURN NL
return_ = (__tokkw('return') + __tokop('NL')) >> make_return
# id_value = id ASSIGN value | value
id_value = ((id_ + __tokop('ASSIGN') + __value) >> named_value) | (__value >> unnamed_value)
# function_params = [id_value {COMMA id_value}]
function_params = maybe(id_value + many(__tokop('COMMA') + id_value)) >> __make_array
# actor_name = id [ AT id ]
actor_name = id_ + maybe(__tokop('AT') + id_)
# function_name = id
function_name = id_
# base_function = actor_name DOT action_name LPAREN function_params RPAREN
base_function = (
actor_name + __tokop('DOT') + function_name +
__tokop('LPAREN') + function_params + __tokop('RPAREN')
)
# function = custom_query_parser_reg | base_function | custom_query_parser_pfx | LPAREN function RPAREN
function = forward_decl()
if custom_query_parser_reg is not None:
function_ = custom_query_parser_reg | base_function
else:
function_ = base_function
if custom_query_parser_pfx is not None:
function_ = function_ | custom_query_parser_pfx
function_ = function_ | (__tokop('LPAREN') + function + __tokop('RPAREN'))
function.define(function_)
# action = (custom_action_parser_reg | base_function | custom_action_parser_pfx) NL
if custom_action_parser_reg is not None:
action = custom_action_parser_reg | base_function
else:
action = base_function
if custom_action_parser_pfx is not None:
action = action | custom_action_parser_pfx
action = action + __tokop('NL') >> make_action
# int_or_enum = INT | QUOTE_ID
int_or_enum = (__toktype('INT') >> __int) | (__toktype('QUOTE_ID') >> __identifier)
# __intlist = int_or_enum {COMMA int_or_enum} [COMMA] | LPAREN __intlist RPAREN
__intlist = forward_decl()
__intlist.define((int_or_enum + many(__tokop('COMMA') + int_or_enum) + maybe(__tokop('COMMA')) >> __make_array) | \
__tokop('LPAREN') + __intlist + __tokop('RPAREN'))
# case = CASE __intlist block
case = __tokkw('case') + __intlist + block >> make_case
# default = DEFAULT block
default = __tokkw('default') + block >> make_case
# cases = { case } [ default ] { case } | pass
cases = many(case) + maybe(default) + many(case) | pass_
# switch = SWITCH function COLON NL INDENT cases DEDENT
switch = __tokkw('switch') + function + __tokop('COLON') + __tokop('NL') + \
__tokop('INDENT') + cases + __tokop('DEDENT') >> make_switch
predicate = forward_decl()
predicate0 = forward_decl()
predicate1 = forward_decl()
predicate2 = forward_decl()
predicate3 = forward_decl()
# bool_function = function
bool_function = function >> make_bool_function
# in_predicate = function IN __intlist
in_predicate = function + __tokkw('in') + __intlist >> make_in
# not_in_predicate = function IN __intlist
not_in_predicate = function + __tokkw('not') + __tokkw('in') + __intlist >> make_in >> make_not
# cmp_predicate = function CMP INT
cmp_predicate = function + __toktype('CMP') + int_or_enum >> make_cmp
# not_predicate = NOT predicate0
not_predicate = __tokkw('not') + predicate0 >> make_not
# const_predicate = TRUE | FALSE
const_predicate = (__tokkw_keep('true') | __tokkw_keep('false')) >> __bool
# paren_predicate = LPAREN predicate RPAREN
paren_predicate = __tokop('LPAREN') + predicate + __tokop('RPAREN')
# predicate0 = not_predicate | const_predicate | bool_function | paren_predicate
predicate0.define(not_predicate | const_predicate | bool_function | paren_predicate)
# predicate1 = custom_query_parser_op | in_predicate | not_in_predicate | cmp_predicate | predicate0
predicate1_ = in_predicate | not_in_predicate | cmp_predicate | predicate0
if custom_query_parser_op is not None:
predicate1_ = (custom_query_parser_op >> make_bool_function) | predicate1_
predicate1.define(predicate1_)
# and_predicate = predicate1 AND predicate2
and_predicate = predicate1 + __tokkw('and') + predicate2 >> make_and
# predicate2 = and_predicate | predicate1
predicate2.define(and_predicate | predicate1)
# or_predicate = predicate2 OR predicate3
or_predicate = predicate2 + __tokkw('or') + predicate3 >> make_or
# predicate3 = or_predicate | predicate2
predicate3.define(or_predicate | predicate2)
# predicate = predicate3
predicate.define(predicate3)
# if = IF predicate block
if_ = __tokkw('if') + predicate + block
# elif = ELIF predicate block
elif_ = __tokkw('elif') + predicate + block
# else = ELSE block
else_ = __tokkw('else') + block
# ifelse = if { elif } [ else ]
ifelse = if_ + many(elif_) + maybe(else_) >> make_ifelse
# branches = { BRANCH block }
# branchless case handled implicitly by lack of INDENT
branches = many(__tokkw('branch') + block)
# fork = FORK COLON NL INDENT branches DEDENT
fork = __tokkw('fork') + __tokop('COLON') + __tokop('NL') + \
__tokop('INDENT') + branches + __tokop('DEDENT') >> make_fork
# while = WHILE predicate block
while_ = __tokkw('while') + predicate + block >> make_while
# do_while = DO block WHILE predicate NL
do_while = __tokkw('do') + block + __tokkw('while') + predicate + __tokop('NL') >> make_do_while
# flow_name = [id COLON COLON] id
flow_name = maybe(id_ + __tokop('COLON') + __tokop('COLON')) + id_
# subflow_param = id ASSIGN nonenum_value
subflow_param = id_ + __tokop('ASSIGN') + __nonenum_value >> make_subflow_param
# subflow_params = [subflow_param { COMMA subflow_param }]
subflow_params = maybe(subflow_param + many(__tokop('COMMA') + subflow_param)) >> __make_array
# run = RUN flow_name LPAREN subflow_params RPAREN NL
run = (
__tokkw('run') + flow_name + __tokop('LPAREN') + subflow_params + __tokop('RPAREN') + __tokop('NL')
) >> make_subflow
# stmt = action | switch | ifelse | fork | while_ | do_while | run | pass_ | return | NL
stmt = action | switch | ifelse | fork | while_ | do_while | run | pass_ | return_ | (__tokop('NL') >> make_none)
# entrypoint = ENTRYPOINT id COLON NL
entrypoint = __tokkw('entrypoint') + id_ + __tokop('COLON') + __tokop('NL')
# stmts = stmt { stmt }
stmts = stmt + many(stmt) >> __make_array
# ep_block_body = [entrypoint] stmts
ep_block_body = maybe(entrypoint) + stmts >> link_ep_block
# block_body = ep_block_body { ep_block_body }
block_body = ep_block_body + many(ep_block_body) >> link_block
# block = COLON NL INDENT block_body DEDENT
block.define(__tokop('COLON') + __tokop('NL') + __tokop('INDENT') + block_body + __tokop('DEDENT'))
# type = INT | FLOAT | STR | BOOL | ANY
type_atom = (__tokkw_keep('int') | __tokkw_keep('float') | __tokkw_keep('str') | __tokkw_keep('bool') | __tokkw_keep('any')) >> __type
# flow_param = ID COLON TYPE [ASSIGN base_value]
flow_param = id_ + __tokop('COLON') + type_atom + maybe(__tokop('ASSIGN') + __base_value) >> make_param
# flow_params = [flow_param { COMMA flow_param }]
flow_params = maybe(flow_param + many(__tokop('COMMA') + flow_param)) >> __make_array
# flow = [LOCAL] FLOW ID LPAREN flow_params RPAREN block
flow = (
maybe(a(Token('ID', 'local'))) + __tokkw('flow') + id_ + __tokop('LPAREN') + flow_params + __tokop('RPAREN') + block
) >> make_flow
# file = { flow | NL }
evfl_file = many(flow | (__tokop('NL') >> make_none)) >> collect_flows
parser = evfl_file + skip(finished)
roots: List[RootNode] = parser.parse(seq).value
local_roots = {r.name: r for r in roots if r.local}
exported_roots = {r.name: r for r in roots if not r.local}
for n in roots:
__collapse_connectors(n)
__process_local_calls(roots, local_roots, exported_roots, exported_tco)
for n in roots:
__replace_node(n, TerminalNode, None)
return list(exported_roots.values()), list(actors.values())