def _parse_pattern(pattern, data, file_path): source = base.Source(data=data, path=file_path) tokens = lexer.lex(source) match_result = All(pattern, Peek('EOF')).getitem(0).parse(tokens) if not match_result: raise match_result.to_error() return match_result.value
def _parse_pattern(pattern, data, file_path, import_path): source = base.Source(data=data, path=file_path) tokens = lexer.lex(source) match_result = (All(pattern, Peek('EOF')).map(lambda args: args[0]).parse(tokens)) if not match_result: raise match_result.to_error() return match_result.value
def _make_module_parser(*, global_variable, function, class_, module_name): global_stmt = Any( global_variable.map(lambda x: ('var', x)), function.map(lambda x: ('func', x)), class_.map(lambda x: ('cls', x)), ) module_pattern = All( import_stmt.map(lambda x: ('import', x)).repeat(), global_stmt.repeat(), ).flatten() def module_callback(m): imports = [] vars = [] funcs = [] clss = [] for kind, node in m.value: if kind == 'import': imports.append(node) elif kind == 'var': vars.append(node) elif kind == 'func': funcs.append(node) elif kind == 'cls': clss.append(node) else: raise base.Error([node.mark], f'FUBAR: {kind}: {node}') return ast.Module( mark=m.mark, name=module_name, imports=tuple(imports), vars=tuple(vars), funcs=tuple(funcs), clss=tuple(clss), ) return module_pattern.fatmap(module_callback)
def _make_header_parser(module_name: str, imports: typing.Tuple[ast.Import, ...]): import_map = _make_import_map(imports=imports, module_name=module_name) importable_id = _make_importable_id( module_name=module_name, import_map=import_map, ) exportable_id = _make_exportable_id( module_name=module_name, import_map=import_map, ) type_ = _make_type_parser( importable_id=importable_id, global_dict=None, ) # for any statement or expression we want to skip over. blob = Forward(lambda: Any( AnyTokenBut('{', '}', '(', ')', ';', ','), All('(', blob.repeat(), ')'), brace_blob, )) inner_blob = Forward(lambda: Any( AnyTokenBut('{', '}'), brace_blob, )) brace_blob = All('{', inner_blob.repeat(), '}') return _make_combined_parser( module_name=module_name, importable_id=importable_id, exportable_id=exportable_id, type_=type_, expression=blob.repeat().map(lambda x: None), block=brace_blob.map(lambda x: None), global_dict=None, module_scope=None, )
translation_unit = Forward(lambda: Struct(cst.TranslationUnit, [ ['stmts', Any( inline_blob, import_stmt, native_typedef, struct_definition, function_definition, ).repeat()], ])) inline_blob = Struct(cst.InlineBlob, [ 'inline', ['type', Any( All('*', '*').valmap('fwd'), All('*').valmap('hdr'), All().valmap('src'), )], ['text', Required('STR')], ]) import_path_pattern = All( All('ID'), All('.', 'ID').getitem(1).repeat(), ).flatten().map('.'.join) import_stmt = Struct(cst.Import, [ 'import', ['path', import_path_pattern], ])
Any(',').optional(), Required(']'), ]), Struct(cst.VoidType, ['void']), Struct(cst.BoolType, ['bool']), Struct(cst.IntType, ['int']), Struct(cst.DoubleType, ['double']), Struct(cst.StringType, ['string']), Struct(cst.Typename, [['name', 'ID']]), )) value_expression = Forward(lambda: postfix) file_ = Forward(lambda: Struct(cst.File, [ ['statements', Any( All(line_comment), All(import_), All(inline), All(class_), All(function), All('NEWLINE').valmap(()), ).repeat().flatten().map(tuple)], ])) module_name = All( All('ID'), All('.', 'ID').getitem(1).repeat(), ).flatten().map('.'.join) import_ = Struct(cst.Import, [ 'from',
def _make_source_parser(module_name: str, header: ast.Module, global_dict): def ensure_global_exists(m): name = m.value if name not in global_dict: raise base.Error([m.mark], f'Name {repr(name)} does not exist') return name import_map = _make_import_map( imports=header.imports, module_name=module_name, ) importable_id = _make_importable_id( module_name=module_name, import_map=import_map, ).fatmap(ensure_global_exists) exportable_id = _make_exportable_id( module_name=module_name, import_map=import_map, ).fatmap(ensure_global_exists) type_ = _make_type_parser( importable_id=importable_id, global_dict=global_dict, ) module_scope = { key: global_dict[val] for key, val in _prelude_table.items() } for imp in header.imports: if imp.name in global_dict: module_scope[imp.name] = global_dict[imp.name] else: raise base.Error( [imp.mark], f'{repr(imp.name)} is not defined', ) def get_func_def(scope, name, mark): if name not in scope: raise base.Error([mark], f'{name} is not defined') if not isinstance(scope[name], ast.Function): raise base.Error([scope[name].mark, mark], f'{name} is not a function') return scope[name] def fcall(mark, f, args): if len(f.params) != len(args): raise base.Error( [f.mark, mark], f'Expected {len(f.params)} args but got {len(args)}') for param, arg in zip(f.params, args): if not types.convertible(arg.type, param.type, global_dict): raise base.Error( [param.mark, arg.mark], f'Expected type {param.type} but got {arg.type}') return ast.FunctionCall(mark=mark, f=f, args=args) # TODO expression = Forward(lambda: atom) args = All( '(', expression.join(',').fatmap( lambda m: lambda scope: tuple(e(scope) for e in m.value)), Any(')').required(), ).map(lambda args: args[1]) atom = Any( Any('INT').fatmap(lambda m: lambda scope: ast.IntLiteral( mark=m.mark, value=m.value)), Any('STR').fatmap(lambda m: lambda scope: ast.StringLiteral( mark=m.mark, value=m.value, )), All('ID', args).fatmap( lambda m: lambda scope: fcall( mark=m.mark, f=get_func_def(scope=scope, name=m.value[0], mark=m.mark), args=m.value[1](scope), ), ), ) statement = Forward(lambda: Any( block, All('return', expression, ';').fatmap( lambda m: lambda scope: ast.Return( mark=m.mark, expr=m.value[1](scope), ), ), All(expression, ';').fatmap(lambda m: lambda scope: ast. ExpressionStatement( mark=m.mark, expr=m.value[0](scope), )), )) block = (All('{', statement.repeat(), Any('}').required()).fatmap(lambda m: lambda scope: ast.Block( mark=m.mark, stmts=tuple(stmt(scope) for stmt in m.value[1]), ))) return _make_combined_parser( module_name=module_name, importable_id=importable_id, exportable_id=exportable_id, type_=type_, expression=expression, block=block, global_dict=global_dict, module_scope=module_scope, )
def _make_combined_parser(*, module_name, importable_id, exportable_id, type_, expression, block, global_dict, module_scope): # We can perform some basic validations if the global_dict # is available to us. validate = global_dict is not None global_variable = All( type_, exportable_id, '=', expression.required(), Any(';').required(), ).fatmap(lambda m: ast.GlobalVariable( mark=m.mark, type=m.value[0], name=m.value[1], expr=m.value[3] if global_dict is None else m.value[3](global_dict), )) parameter = All(type_, 'ID').fatmap(lambda m: ast.Parameter( mark=m.mark, type=m.value[0], name=m.value[1], )) parameters = All( '(', parameter.join(',').map(tuple), ')', ).map(lambda args: tuple(args[1])) ############ # Function ############ function_pattern = All( native, # 0: native type_, # 1: return type exportable_id, # 2: name parameters, # 3: parameters Any( block, Any(';').map(lambda x: None), ), # 4: body ) def function_callback(m): native = m.value[0] rtype = m.value[1] name = m.value[2] params = m.value[3] body_thunk = m.value[4] if body_thunk is None: body = None else: scope = Scope(module_scope) for param in params: scope[param.name] = param body = body_thunk(scope) if validate: if native and body: raise base.Error([m.mark], 'Native functions cannot have a body') if not native and not body: raise base.Error([m.mark], 'Non-native functions must have a body') return ast.Function( mark=m.mark, rtype=rtype, name=name, params=params, body=body, ) function = function_pattern.fatmap(function_callback) ############ # Field ############ field_thunk = (All(type_, 'ID', ';').fatmap(lambda m: lambda scope: ast.Field( mark=m.mark, type=m.value[0], name=f'{scope["@class_name"]}.{m.value[1]}', ))) ################# # Method (thunk) ################# method_pattern = All( type_, # 0: return type 'ID', # 1: name parameters, # 2: parameters Any( block, Any(';').map(lambda x: None), ), # 3: body ) def method_callback(m): def inner_callback(outer_scope): scope = Scope(outer_scope) rtype = m.value[0] name = f'{scope["@class_name"]}#{m.value[1]}' params = m.value[2] body_thunk = m.value[3] if body_thunk is None: body = None else: for param in params: scope[param.name] = param body = body_thunk(scope) return ast.Method( mark=m.mark, rtype=rtype, name=name, params=params, body=body, ) return inner_callback method_thunk = method_pattern.fatmap(method_callback) ############ # Class ############ base_class = Any( All(':', importable_id).map(lambda args: args[1]), All().map(lambda args: None), ) class_pattern = All( native, # 0: native 'class', # 1 exportable_id, # 2: name base_class, # 3: base/super class '{', # 4 field_thunk.repeat(), # 5: fields method_thunk.repeat(), # 6: methods '}', # 7 ) def class_callback(m): native = m.value[0] class_name = m.value[2] declared_base = m.value[3] field_thunks = m.value[5] method_thunks = m.value[6] base = (ast.OBJECT if declared_base is None and class_name != ast.OBJECT else declared_base) scope = Scope(module_scope) scope['@class_name'] = class_name fields = tuple(ft(scope) for ft in field_thunks) methods = tuple(mt(scope) for mt in method_thunks) if validate: for method in methods: if native and method.body: raise base.Error( [m.mark, method.mark], 'Native classes cannot have method bodies', ) if not native and not method.body: raise base.Error( [m.mark, method.mark], 'Non-native classes cannot have native methods', ) if native and fields: raise base.Error( [m.mark, fields[0].mark], 'Native classes cannot have fields', ) return ast.Class( mark=m.mark, native=native, name=class_name, base=base, fields=fields, methods=methods, ) class_ = class_pattern.fatmap(class_callback) module = _make_module_parser( global_variable=global_variable, function=function, class_=class_, module_name=module_name, ) return module
def _import_path_to_file_path(import_path): return os.path.join( _source_root, import_path.replace('.', os.path.sep) + '.nc', ) import_stmt = All( 'import', All('ID', All('.', 'ID').map(lambda args: args[1]).repeat()).map( lambda args: '.'.join([args[0]] + args[1])), Any( All('as', 'ID').map(lambda args: args[1]), All().map(lambda x: None), ), ).fatmap(lambda m: ast.Import( mark=m.mark, name=m.value[1], alias=m.value[1].split('.')[-1] if m.value[2] is None else m.value[2], )) _prelude_table = { symbol: f'{ast.PRELUDE}.{symbol}' for symbol in ast.PRELUDE_SYMBOLS } _builtin_mark = base.Mark( source=base.Source( path='<builtin>',
from mtots.parser.combinator import AnyTokenBut from mtots.parser.combinator import Forward from mtots.parser.combinator import Peek from mtots.parser.combinator import Required from mtots.parser.combinator import Token def Struct(*args, **kwargs): return combinator.Struct(*args, include_mark=True, **kwargs) module = Forward(lambda: Struct(cst.Module, [ [ 'statements', Any( All(import_), All(global_variable), All(function), All(class_), All('NEWLINE').valmap(()), ).repeat().flatten().map(tuple) ], ])) import_ = Struct(cst.Import, [ 'import', [ 'name', All( All('ID'), All('.', 'ID').getitem(1).repeat(),
def parse(s): return (All(source, Peek('EOF')).map(lambda args: args[0]).parse( lexer.lex_string(s)))
def parse(s): return (All(import_stmt.repeat(), Peek('EOF')).map(lambda args: args[0]).parse( lexer.lex_string(s)))
from mtots.parser.combinator import AnyTokenBut from mtots.parser.combinator import Forward from mtots.parser.combinator import Peek from mtots.parser.combinator import Token import os MAIN_IMPORT_PATH = '__main__' # Useful for skipping blocks of code # for the header parser blob = Forward(lambda: Any( brace_blob, AnyTokenBut('{', '}'), )) brace_blob = All('{', blob.repeat(), '}') # These are modifiers that affect the type of a function # i.e. does the type of the function pointer need to know # this about the function it's pointinng to? func_decl_modifier = Any( # Windows calling conventions Token('ID', '__cdecl'), Token('ID', '__clrcall'), Token('ID', '__stdcall'), Token('ID', '__fastcall'), Token('ID', '__thiscall'), Token('ID', '__vectorcall'), ) # These are modifiers that affect the definition of a function