def _parse_pattern(pattern, data, file_path):
source = base.Source(data=data, path=file_path)
tokens = lexer.lex(source)
match_result = All(pattern, Peek('EOF')).getitem(0).parse(tokens)
if not match_result:
raise match_result.to_error()
return match_result.value
def _parse_pattern(pattern, data, file_path, import_path):
source = base.Source(data=data, path=file_path)
tokens = lexer.lex(source)
match_result = (All(pattern,
Peek('EOF')).map(lambda args: args[0]).parse(tokens))
if not match_result:
raise match_result.to_error()
return match_result.value
def _make_module_parser(*, global_variable, function, class_, module_name):
global_stmt = Any(
global_variable.map(lambda x: ('var', x)),
function.map(lambda x: ('func', x)),
class_.map(lambda x: ('cls', x)),
)
module_pattern = All(
import_stmt.map(lambda x: ('import', x)).repeat(),
global_stmt.repeat(),
).flatten()
def module_callback(m):
imports = []
vars = []
funcs = []
clss = []
for kind, node in m.value:
if kind == 'import':
imports.append(node)
elif kind == 'var':
vars.append(node)
elif kind == 'func':
funcs.append(node)
elif kind == 'cls':
clss.append(node)
else:
raise base.Error([node.mark], f'FUBAR: {kind}: {node}')
return ast.Module(
mark=m.mark,
name=module_name,
imports=tuple(imports),
vars=tuple(vars),
funcs=tuple(funcs),
clss=tuple(clss),
)
return module_pattern.fatmap(module_callback)
def _make_header_parser(module_name: str, imports: typing.Tuple[ast.Import,
...]):
import_map = _make_import_map(imports=imports, module_name=module_name)
importable_id = _make_importable_id(
module_name=module_name,
import_map=import_map,
)
exportable_id = _make_exportable_id(
module_name=module_name,
import_map=import_map,
)
type_ = _make_type_parser(
importable_id=importable_id,
global_dict=None,
)
# for any statement or expression we want to skip over.
blob = Forward(lambda: Any(
AnyTokenBut('{', '}', '(', ')', ';', ','),
All('(', blob.repeat(), ')'),
brace_blob,
))
inner_blob = Forward(lambda: Any(
AnyTokenBut('{', '}'),
brace_blob,
))
brace_blob = All('{', inner_blob.repeat(), '}')
return _make_combined_parser(
module_name=module_name,
importable_id=importable_id,
exportable_id=exportable_id,
type_=type_,
expression=blob.repeat().map(lambda x: None),
block=brace_blob.map(lambda x: None),
global_dict=None,
module_scope=None,
)
translation_unit = Forward(lambda: Struct(cst.TranslationUnit, [
['stmts', Any(
inline_blob,
import_stmt,
native_typedef,
struct_definition,
function_definition,
).repeat()],
]))
inline_blob = Struct(cst.InlineBlob, [
'inline',
['type', Any(
All('*', '*').valmap('fwd'),
All('*').valmap('hdr'),
All().valmap('src'),
)],
['text', Required('STR')],
])
import_path_pattern = All(
All('ID'),
All('.', 'ID').getitem(1).repeat(),
).flatten().map('.'.join)
import_stmt = Struct(cst.Import, [
'import',
['path', import_path_pattern],
])
Any(',').optional(),
Required(']'),
]),
Struct(cst.VoidType, ['void']),
Struct(cst.BoolType, ['bool']),
Struct(cst.IntType, ['int']),
Struct(cst.DoubleType, ['double']),
Struct(cst.StringType, ['string']),
Struct(cst.Typename, [['name', 'ID']]),
))
value_expression = Forward(lambda: postfix)
file_ = Forward(lambda: Struct(cst.File, [
['statements', Any(
All(line_comment),
All(import_),
All(inline),
All(class_),
All(function),
All('NEWLINE').valmap(()),
).repeat().flatten().map(tuple)],
]))
module_name = All(
All('ID'),
All('.', 'ID').getitem(1).repeat(),
).flatten().map('.'.join)
import_ = Struct(cst.Import, [
'from',
def _make_source_parser(module_name: str, header: ast.Module, global_dict):
def ensure_global_exists(m):
name = m.value
if name not in global_dict:
raise base.Error([m.mark], f'Name {repr(name)} does not exist')
return name
import_map = _make_import_map(
imports=header.imports,
module_name=module_name,
)
importable_id = _make_importable_id(
module_name=module_name,
import_map=import_map,
).fatmap(ensure_global_exists)
exportable_id = _make_exportable_id(
module_name=module_name,
import_map=import_map,
).fatmap(ensure_global_exists)
type_ = _make_type_parser(
importable_id=importable_id,
global_dict=global_dict,
)
module_scope = {
key: global_dict[val]
for key, val in _prelude_table.items()
}
for imp in header.imports:
if imp.name in global_dict:
module_scope[imp.name] = global_dict[imp.name]
else:
raise base.Error(
[imp.mark],
f'{repr(imp.name)} is not defined',
)
def get_func_def(scope, name, mark):
if name not in scope:
raise base.Error([mark], f'{name} is not defined')
if not isinstance(scope[name], ast.Function):
raise base.Error([scope[name].mark, mark],
f'{name} is not a function')
return scope[name]
def fcall(mark, f, args):
if len(f.params) != len(args):
raise base.Error(
[f.mark, mark],
f'Expected {len(f.params)} args but got {len(args)}')
for param, arg in zip(f.params, args):
if not types.convertible(arg.type, param.type, global_dict):
raise base.Error(
[param.mark, arg.mark],
f'Expected type {param.type} but got {arg.type}')
return ast.FunctionCall(mark=mark, f=f, args=args)
# TODO
expression = Forward(lambda: atom)
args = All(
'(',
expression.join(',').fatmap(
lambda m: lambda scope: tuple(e(scope) for e in m.value)),
Any(')').required(),
).map(lambda args: args[1])
atom = Any(
Any('INT').fatmap(lambda m: lambda scope: ast.IntLiteral(
mark=m.mark, value=m.value)),
Any('STR').fatmap(lambda m: lambda scope: ast.StringLiteral(
mark=m.mark,
value=m.value,
)),
All('ID', args).fatmap(
lambda m: lambda scope: fcall(
mark=m.mark,
f=get_func_def(scope=scope, name=m.value[0], mark=m.mark),
args=m.value[1](scope),
), ),
)
statement = Forward(lambda: Any(
block,
All('return', expression, ';').fatmap(
lambda m: lambda scope: ast.Return(
mark=m.mark,
expr=m.value[1](scope),
), ),
All(expression, ';').fatmap(lambda m: lambda scope: ast.
ExpressionStatement(
mark=m.mark,
expr=m.value[0](scope),
)),
))
block = (All('{', statement.repeat(),
Any('}').required()).fatmap(lambda m: lambda scope: ast.Block(
mark=m.mark,
stmts=tuple(stmt(scope) for stmt in m.value[1]),
)))
return _make_combined_parser(
module_name=module_name,
importable_id=importable_id,
exportable_id=exportable_id,
type_=type_,
expression=expression,
block=block,
global_dict=global_dict,
module_scope=module_scope,
)
def _make_combined_parser(*, module_name, importable_id, exportable_id, type_,
expression, block, global_dict, module_scope):
# We can perform some basic validations if the global_dict
# is available to us.
validate = global_dict is not None
global_variable = All(
type_,
exportable_id,
'=',
expression.required(),
Any(';').required(),
).fatmap(lambda m: ast.GlobalVariable(
mark=m.mark,
type=m.value[0],
name=m.value[1],
expr=m.value[3] if global_dict is None else m.value[3](global_dict),
))
parameter = All(type_, 'ID').fatmap(lambda m: ast.Parameter(
mark=m.mark,
type=m.value[0],
name=m.value[1],
))
parameters = All(
'(',
parameter.join(',').map(tuple),
')',
).map(lambda args: tuple(args[1]))
############
# Function
############
function_pattern = All(
native, # 0: native
type_, # 1: return type
exportable_id, # 2: name
parameters, # 3: parameters
Any(
block,
Any(';').map(lambda x: None),
), # 4: body
)
def function_callback(m):
native = m.value[0]
rtype = m.value[1]
name = m.value[2]
params = m.value[3]
body_thunk = m.value[4]
if body_thunk is None:
body = None
else:
scope = Scope(module_scope)
for param in params:
scope[param.name] = param
body = body_thunk(scope)
if validate:
if native and body:
raise base.Error([m.mark],
'Native functions cannot have a body')
if not native and not body:
raise base.Error([m.mark],
'Non-native functions must have a body')
return ast.Function(
mark=m.mark,
rtype=rtype,
name=name,
params=params,
body=body,
)
function = function_pattern.fatmap(function_callback)
############
# Field
############
field_thunk = (All(type_, 'ID',
';').fatmap(lambda m: lambda scope: ast.Field(
mark=m.mark,
type=m.value[0],
name=f'{scope["@class_name"]}.{m.value[1]}',
)))
#################
# Method (thunk)
#################
method_pattern = All(
type_, # 0: return type
'ID', # 1: name
parameters, # 2: parameters
Any(
block,
Any(';').map(lambda x: None),
), # 3: body
)
def method_callback(m):
def inner_callback(outer_scope):
scope = Scope(outer_scope)
rtype = m.value[0]
name = f'{scope["@class_name"]}#{m.value[1]}'
params = m.value[2]
body_thunk = m.value[3]
if body_thunk is None:
body = None
else:
for param in params:
scope[param.name] = param
body = body_thunk(scope)
return ast.Method(
mark=m.mark,
rtype=rtype,
name=name,
params=params,
body=body,
)
return inner_callback
method_thunk = method_pattern.fatmap(method_callback)
############
# Class
############
base_class = Any(
All(':', importable_id).map(lambda args: args[1]),
All().map(lambda args: None),
)
class_pattern = All(
native, # 0: native
'class', # 1
exportable_id, # 2: name
base_class, # 3: base/super class
'{', # 4
field_thunk.repeat(), # 5: fields
method_thunk.repeat(), # 6: methods
'}', # 7
)
def class_callback(m):
native = m.value[0]
class_name = m.value[2]
declared_base = m.value[3]
field_thunks = m.value[5]
method_thunks = m.value[6]
base = (ast.OBJECT if declared_base is None
and class_name != ast.OBJECT else declared_base)
scope = Scope(module_scope)
scope['@class_name'] = class_name
fields = tuple(ft(scope) for ft in field_thunks)
methods = tuple(mt(scope) for mt in method_thunks)
if validate:
for method in methods:
if native and method.body:
raise base.Error(
[m.mark, method.mark],
'Native classes cannot have method bodies',
)
if not native and not method.body:
raise base.Error(
[m.mark, method.mark],
'Non-native classes cannot have native methods',
)
if native and fields:
raise base.Error(
[m.mark, fields[0].mark],
'Native classes cannot have fields',
)
return ast.Class(
mark=m.mark,
native=native,
name=class_name,
base=base,
fields=fields,
methods=methods,
)
class_ = class_pattern.fatmap(class_callback)
module = _make_module_parser(
global_variable=global_variable,
function=function,
class_=class_,
module_name=module_name,
)
return module
def _import_path_to_file_path(import_path):
return os.path.join(
_source_root,
import_path.replace('.', os.path.sep) + '.nc',
)
import_stmt = All(
'import',
All('ID',
All('.', 'ID').map(lambda args: args[1]).repeat()).map(
lambda args: '.'.join([args[0]] + args[1])),
Any(
All('as', 'ID').map(lambda args: args[1]),
All().map(lambda x: None),
),
).fatmap(lambda m: ast.Import(
mark=m.mark,
name=m.value[1],
alias=m.value[1].split('.')[-1] if m.value[2] is None else m.value[2],
))
_prelude_table = {
symbol: f'{ast.PRELUDE}.{symbol}'
for symbol in ast.PRELUDE_SYMBOLS
}
_builtin_mark = base.Mark(
source=base.Source(
path='<builtin>',
from mtots.parser.combinator import AnyTokenBut
from mtots.parser.combinator import Forward
from mtots.parser.combinator import Peek
from mtots.parser.combinator import Required
from mtots.parser.combinator import Token
def Struct(*args, **kwargs):
return combinator.Struct(*args, include_mark=True, **kwargs)
module = Forward(lambda: Struct(cst.Module, [
[
'statements',
Any(
All(import_),
All(global_variable),
All(function),
All(class_),
All('NEWLINE').valmap(()),
).repeat().flatten().map(tuple)
],
]))
import_ = Struct(cst.Import, [
'import',
[
'name',
All(
All('ID'),
All('.', 'ID').getitem(1).repeat(),
def parse(s):
return (All(source, Peek('EOF')).map(lambda args: args[0]).parse(
lexer.lex_string(s)))
def parse(s):
return (All(import_stmt.repeat(),
Peek('EOF')).map(lambda args: args[0]).parse(
lexer.lex_string(s)))
from mtots.parser.combinator import AnyTokenBut
from mtots.parser.combinator import Forward
from mtots.parser.combinator import Peek
from mtots.parser.combinator import Token
import os
MAIN_IMPORT_PATH = '__main__'
# Useful for skipping blocks of code
# for the header parser
blob = Forward(lambda: Any(
brace_blob,
AnyTokenBut('{', '}'),
))
brace_blob = All('{', blob.repeat(), '}')
# These are modifiers that affect the type of a function
# i.e. does the type of the function pointer need to know
# this about the function it's pointinng to?
func_decl_modifier = Any(
# Windows calling conventions
Token('ID', '__cdecl'),
Token('ID', '__clrcall'),
Token('ID', '__stdcall'),
Token('ID', '__fastcall'),
Token('ID', '__thiscall'),
Token('ID', '__vectorcall'),
)
# These are modifiers that affect the definition of a function
