def load_compiler_suite(compiler_suite: str) -> \ Tuple[PreprocessorFactoryFunc, ParserFactoryFunc, TransformerFactoryFunc, CompilerFactoryFunc]: """ Extracts a compiler suite from file or string `compiler_suite` and returns it as a tuple (preprocessor, parser, ast, compiler). Returns: 4-tuple (preprocessor function, parser class, ast transformer function, compiler class) """ global RX_SECTION_MARKER assert isinstance(compiler_suite, str) source = load_if_file(compiler_suite) dhpath = relative_path(os.path.dirname('.'), DHPARSER_PARENTDIR) imports = DHPARSER_IMPORTS.format(dhparser_parentdir=dhpath) if is_python_code(compiler_suite): sections = split_source(compiler_suite, source) _, imports, preprocessor_py, parser_py, ast_py, compiler_py, _ = sections # TODO: Compile in one step and pick parts from namespace later ? preprocessor = compile_python_object(imports + preprocessor_py, r'get_(?:\w+_)?preprocessor$') parser = compile_python_object(imports + parser_py, r'get_(?:\w+_)?grammar$') ast = compile_python_object(imports + ast_py, r'get_(?:\w+_)?transformer$') else: # Assume source is an ebnf grammar. # Is there really any reasonable application case for this? lg_dir = suspend_logging() compiler_py, messages, _ = compile_source(source, None, get_ebnf_grammar(), get_ebnf_transformer(), get_ebnf_compiler(compiler_suite, source)) resume_logging(lg_dir) if has_errors(messages): raise DefinitionError(only_errors(messages), source) preprocessor = get_ebnf_preprocessor parser = get_ebnf_grammar ast = get_ebnf_transformer compiler = compile_python_object(imports + compiler_py, r'get_(?:\w+_)?compiler$') if callable(preprocessor) and callable(parser) and callable(Callable) and callable(compiler): return preprocessor, parser, ast, compiler raise ValueError('Could not generate compiler suite from source code!')
def grammar_instance(grammar_representation) -> Tuple[Grammar, str]: """ Returns a grammar object and the source code of the grammar, from the given `grammar`-data which can be either a file name, ebnf-code, python-code, a Grammar-derived grammar class or an instance of such a class (i.e. a grammar object already). """ if isinstance(grammar_representation, str): # read grammar grammar_src = load_if_file(grammar_representation) if is_python_code(grammar_src): parser_py = grammar_src # type: str messages = [] # type: List[Error] else: lg_dir = suspend_logging() result, messages, _ = compile_source( grammar_src, None, get_ebnf_grammar(), get_ebnf_transformer(), get_ebnf_compiler()) parser_py = cast(str, result) resume_logging(lg_dir) if has_errors(messages): raise DefinitionError(only_errors(messages), grammar_src) imports = DHPARSER_IMPORTS.format( dhparser_parentdir=relative_path('.', DHPARSER_PARENTDIR)) grammar_class = compile_python_object(imports + parser_py, r'\w+Grammar$') if inspect.isclass(grammar_class) and issubclass(grammar_class, Grammar): parser_root = grammar_class() else: raise ValueError('Could not compile or Grammar class!') else: # assume that dsl_grammar is a ParserHQ-object or Grammar class grammar_src = '' if isinstance(grammar_representation, Grammar): parser_root = grammar_representation else: # assume ``grammar_representation`` is a grammar class and get the root object parser_root = grammar_representation() return parser_root, grammar_src
def compileDSL(text_or_file: str, preprocessor: Optional[PreprocessorFunc], dsl_grammar: Union[str, Grammar], ast_transformation: TransformationFunc, compiler: Compiler) -> Any: """ Compiles a text in a domain specific language (DSL) with an EBNF-specified grammar. Returns the compiled text or raises a compilation error. Raises: CompilationError if any errors occurred during compilation """ assert isinstance(text_or_file, str) assert isinstance(compiler, Compiler) parser, grammar_src = grammar_instance(dsl_grammar) result, messages, AST = compile_source(text_or_file, preprocessor, parser, ast_transformation, compiler) if has_errors(messages): src = load_if_file(text_or_file) raise CompilationError(only_errors(messages), src, grammar_src, AST, result) return result
def test_load_if_file(self): # an error should be raised if file expected but not found error_raised = False try: load_if_file('this_is_code_and_not_a_file') except FileNotFoundError: error_raised = True assert error_raised # multiline text will never be mistaken for a file assert load_if_file('this_is_code_and_not_a_file\n') # neither will text that does not look like a file name s = "this is code * and not a file" assert s == load_if_file(s) # not a file and not mistaken for a file assert self.code1 == load_if_file(self.code1) # not a file and not mistaken for a file either assert self.code2 == load_if_file(self.code2) # file correctly loaded assert self.code2 == load_if_file(self.filename)
def extract_symbols(ebnf_text_or_file: str) -> SymbolsDictType: r""" Extracts all defined symbols from an EBNF-grammar. This can be used to prepare grammar-tests. The symbols will be returned as lists of strings which are grouped by the sections to which they belong and returned as an ordered dictionary, they keys of which are the section names. In order to define a section in the ebnf-source, add a comment-line starting with "#:", followed by the section name. It is recommended to use valid file names as section names. Example: #: components expression = term { EXPR_OP~ term} term = factor { TERM_OP~ factor} factor = [SIGN] ( NUMBER | VARIABLE | group ) { VARIABLE | group } group = "(" expression ")" #: leaf_expressions EXPR_OP = /\+/ | /-/ TERM_OP = /\*/ | /\// SIGN = /-/ NUMBER = /(?:0|(?:[1-9]\d*))(?:\.\d+)?/~ VARIABLE = /[A-Za-z]/~ If no sections have been defined in the comments, there will be only one group with the empty string as a key. :param ebnf_text_or_file: Either an ebnf-grammar or the file-name of an ebnf-grammar :return: Ordered dictionary mapping the section names of the grammar to lists of symbols that appear under that section. """ def trim_section_name(name: str) -> str: return re.sub(r'[^\w-]', '_', name.replace('#:', '').strip()) ebnf = load_if_file(ebnf_text_or_file) deflist = RX_DEFINITION_OR_SECTION.findall(ebnf) if not deflist: if ebnf_text_or_file.find('\n') < 0 and ebnf_text_or_file.endswith( '.ebnf'): deflist = '#: ' + os.path.splitext(ebnf_text_or_file)[0] else: deflist = '#: ALL' symbols = collections.OrderedDict() # type: SymbolsDictType if deflist[0][:2] != '#:': curr_section = '' symbols[curr_section] = [] for df in deflist: if df[:2] == '#:': curr_section = trim_section_name(df) if curr_section in symbols: raise AssertionError('Section name must not be repeated: ' + curr_section) symbols[curr_section] = [] else: symbols[curr_section].append( df) # no worry, curr_section is always defined return symbols
def compile_source(source: str, preprocessor: Optional[PreprocessorFunc], # str -> str parser: GrammarCallable, # str -> Node (concrete syntax tree (CST)) transformer: TransformationFunc, # Node (CST) -> Node (abstract ST (AST)) compiler: CompilerCallable, # Node (AST), Source -> Any # out_source_data: list = NOPE, # Tuple[str, SourceMapFunc] *, preserve_AST: bool = False) \ -> Tuple[Optional[Any], List[Error], Optional[Node]]: """Compiles a source in four stages: 1. Pre-Processing (if needed) 2. Parsing 3. AST-transformation 4. Compiling. The later stages AST-transformation, compilation will only be invoked if no fatal errors occurred in any of the earlier stages of the processing pipeline. :param source: The input text for compilation or a the name of a file containing the input text. :param preprocessor: text -> text. A preprocessor function or None, if no preprocessor is needed. :param parser: A parsing function or grammar class :param transformer: A transformation function that takes the root-node of the concrete syntax tree as an argument and transforms it (in place) into an abstract syntax tree. :param compiler: A compiler function or compiler class instance :param preserve_AST: Preserves the AST-tree. :returns: The result of the compilation as a 3-tuple (result, errors, abstract syntax tree). In detail: 1. The result as returned by the compiler or ``None`` in case of failure 2. A list of error or warning messages 3. The root-node of the abstract syntax tree if `preserve_ast` is True or `None` otherwise. """ ast = None # type: Optional[Node] original_text = load_if_file(source) # type: str source_name = source if is_filename(source) else 'source' compiler.source = original_text log_file_name = logfile_basename( source, compiler) if is_logging() else '' # type: str if not hasattr(parser, 'free_char_parsefunc__') or parser.history_tracking__: # log only for custom parser/transformer/compilers log_syntax_trees = get_config_value('log_syntax_trees') else: log_syntax_trees = set() # preprocessing errors = [] if preprocessor is None: source_text = original_text # type: str source_mapping = gen_neutral_srcmap_func(source_text, source_name) # lambda i: SourceLocation(source_name, 0, i) # type: SourceMapFunc else: _, source_text, source_mapping, errors = preprocessor( original_text, source_name) if has_errors(errors, FATAL): return None, errors, None # parsing syntax_tree = parser(source_text, source_mapping=source_mapping) # type: RootNode for e in errors: syntax_tree.add_error(None, e) syntax_tree.source = original_text syntax_tree.source_mapping = source_mapping if 'cst' in log_syntax_trees: log_ST(syntax_tree, log_file_name + '.cst') if parser.history_tracking__: log_parsing_history(parser, log_file_name) # assert is_error(syntax_tree.error_flag) or str(syntax_tree) == strip_tokens(source_text), \ # str(syntax_tree) # Ony valid if neither tokens or whitespace are dropped early result = None if not is_fatal(syntax_tree.error_flag): # AST-transformation if is_error(syntax_tree.error_flag): # catch Python exception, because if an error has occurred # earlier, the syntax tree might not look like expected, # which could (fatally) break AST transformations. try: transformer(syntax_tree) except Exception as e: syntax_tree.new_error( syntax_tree, "AST-Transformation failed due to earlier parser errors. " "Crash Message: %s: %s" % (e.__class__.__name__, str(e)), AST_TRANSFORM_CRASH) else: transformer(syntax_tree) if 'ast' in log_syntax_trees: log_ST(syntax_tree, log_file_name + '.ast') if not is_fatal(syntax_tree.error_flag): if preserve_AST: ast = copy.deepcopy(syntax_tree) # Compilation if is_error(syntax_tree.error_flag): # assume Python crashes are merely a consequence of earlier # errors, so let's catch them try: result = compiler(syntax_tree) except Exception as e: # raise e node = syntax_tree # type: Node if isinstance(compiler, Compiler) and compiler.context: node = compiler.context[-1] st = traceback.format_list( traceback.extract_tb(e.__traceback__)) trace = ''.join(filter_stacktrace(st)) syntax_tree.new_error( node, "Compilation failed, most likely, due to errors earlier " "in the processing pipeline. Crash Message: %s: %s\n%s" % (e.__class__.__name__, str(e), trace), COMPILER_CRASH) else: # assume Python crashes are programming mistakes, so let # the exceptions through result = compiler(syntax_tree) messages = syntax_tree.errors_sorted # type: List[Error] # Obsolete, because RootNode adjusts error locations whenever an error is added: # adjust_error_locations(messages, original_text, source_mapping) return result, messages, ast