def selftest() -> bool:
"""Run a simple self-test of DHParser.
"""
print("DHParser selftest...")
print("\nSTAGE I: Trying to compile EBNF-Grammar:\n")
builtin_ebnf_parser = get_ebnf_grammar()
docstring = str(builtin_ebnf_parser.__doc__) # type: str
ebnf_src = docstring[docstring.find('@'):]
ebnf_transformer = get_ebnf_transformer()
ebnf_compiler = get_ebnf_compiler('EBNF')
result, errors, _ = compile_source(ebnf_src, None, builtin_ebnf_parser,
ebnf_transformer, ebnf_compiler)
generated_ebnf_parser = cast(str, result)
if errors:
print("Selftest FAILED :-(")
print("\n\n".join(str(err) for err in errors))
return False
print(generated_ebnf_parser)
print("\n\nSTAGE 2: Selfhosting-test: "
"Trying to compile EBNF-Grammar with generated parser...\n")
selfhosted_ebnf_parser = compileDSL(ebnf_src, None, generated_ebnf_parser,
ebnf_transformer, ebnf_compiler)
# ebnf_compiler.gen_transformer_skeleton()
print(selfhosted_ebnf_parser)
return True
def grammar_provider(ebnf_src: str, branding="DSL", additional_code: str = '') -> ParserFactoryFunc:
"""
Compiles an EBNF-grammar and returns a grammar-parser provider
function for that grammar.
Args:
ebnf_src(str): Either the file name of an EBNF grammar or
the EBNF-grammar itself as a string.
branding (str or bool): Branding name for the compiler
suite source code.
additional_code: Python code added to the generated source. This typically
contains the source code of semantic actions referred to in the
generated source, e.g. filter-functions, resume-point-search-functions
Returns:
A provider function for a grammar object for texts in the
language defined by ``ebnf_src``.
"""
grammar_src = compileDSL(ebnf_src, nil_preprocessor, get_ebnf_grammar(),
get_ebnf_transformer(), get_ebnf_compiler(branding, ebnf_src))
log_name = get_config_value('compiled_EBNF_log')
if log_name:
if is_logging():
append_log(log_name, grammar_src)
else:
print(grammar_src)
imports = DHPARSER_IMPORTS.format(dhparser_parentdir=relative_path('.', DHPARSER_PARENTDIR))
grammar_factory = compile_python_object('\n'.join([imports, additional_code, grammar_src]),
r'get_(?:\w+_)?grammar$')
if callable(grammar_factory):
grammar_factory.python_src__ = grammar_src
return grammar_factory
raise ValueError('Could not compile grammar provider!')
def compile_EBNF(self, text: str):
from DHParser.compile import compile_source
from DHParser.ebnf import get_ebnf_preprocessor, get_ebnf_grammar, get_ebnf_transformer, \
get_ebnf_compiler
compiler = get_ebnf_compiler("EBNFServerAnalyse", text)
result, messages, _ = compile_source(text, get_ebnf_preprocessor(),
get_ebnf_grammar(),
get_ebnf_transformer(), compiler)
# TODO: return errors as well as (distilled) information about symbols for code propositions
return None
def compile_EBNF(text: str, diagnostics_signature: bytes) -> (str, bytes):
from DHParser.compile import compile_source
from DHParser.ebnf import get_ebnf_preprocessor, get_ebnf_grammar, get_ebnf_transformer, \
get_ebnf_compiler
from DHParser.toolkit import json_dumps
compiler = get_ebnf_compiler("EBNFServerAnalyse", text)
result, messages, _ = compile_source(text, get_ebnf_preprocessor(),
get_ebnf_grammar(),
get_ebnf_transformer(), compiler)
# TODO: return errors as well as (distilled) information about symbols for code propositions
signature = b''.join(msg.signature() for msg in messages)
if signature != diagnostics_signature:
# publish diagnostics only if the same diagnostics have not been reported earlier, already
diagnostics = [msg.diagnosticObj() for msg in messages]
return json_dumps(
diagnostics), signature # TODO: bytes is not a JSON-type proper!
else:
return '[]', signature
def load_compiler_suite(compiler_suite: str) -> \
Tuple[PreprocessorFactoryFunc, ParserFactoryFunc,
TransformerFactoryFunc, CompilerFactoryFunc]:
"""
Extracts a compiler suite from file or string `compiler_suite`
and returns it as a tuple (preprocessor, parser, ast, compiler).
Returns:
4-tuple (preprocessor function, parser class,
ast transformer function, compiler class)
"""
global RX_SECTION_MARKER
assert isinstance(compiler_suite, str)
source = load_if_file(compiler_suite)
dhpath = relative_path(os.path.dirname('.'), DHPARSER_PARENTDIR)
imports = DHPARSER_IMPORTS.format(dhparser_parentdir=dhpath)
if is_python_code(compiler_suite):
sections = split_source(compiler_suite, source)
_, imports, preprocessor_py, parser_py, ast_py, compiler_py, _ = sections
# TODO: Compile in one step and pick parts from namespace later ?
preprocessor = compile_python_object(imports + preprocessor_py,
r'get_(?:\w+_)?preprocessor$')
parser = compile_python_object(imports + parser_py, r'get_(?:\w+_)?grammar$')
ast = compile_python_object(imports + ast_py, r'get_(?:\w+_)?transformer$')
else:
# Assume source is an ebnf grammar.
# Is there really any reasonable application case for this?
lg_dir = suspend_logging()
compiler_py, messages, _ = compile_source(source, None, get_ebnf_grammar(),
get_ebnf_transformer(),
get_ebnf_compiler(compiler_suite, source))
resume_logging(lg_dir)
if has_errors(messages):
raise DefinitionError(only_errors(messages), source)
preprocessor = get_ebnf_preprocessor
parser = get_ebnf_grammar
ast = get_ebnf_transformer
compiler = compile_python_object(imports + compiler_py, r'get_(?:\w+_)?compiler$')
if callable(preprocessor) and callable(parser) and callable(Callable) and callable(compiler):
return preprocessor, parser, ast, compiler
raise ValueError('Could not generate compiler suite from source code!')
def raw_compileEBNF(ebnf_src: str, branding="DSL") -> EBNFCompiler:
"""
Compiles an EBNF grammar file and returns the compiler object
that was used and which can now be queried for the result as well
as skeleton code for preprocessor, transformer and compiler objects.
Args:
ebnf_src(str): Either the file name of an EBNF grammar or
the EBNF grammar itself as a string.
branding (str): Branding name for the compiler suite source
code.
Returns:
An instance of class ``ebnf.EBNFCompiler``
Raises:
CompilationError if any errors occurred during compilation
"""
grammar = get_ebnf_grammar()
compiler = get_ebnf_compiler(branding, ebnf_src)
transformer = get_ebnf_transformer()
compileDSL(ebnf_src, nil_preprocessor, grammar, transformer, compiler)
return compiler
def grammar_instance(grammar_representation) -> Tuple[Grammar, str]:
"""
Returns a grammar object and the source code of the grammar, from
the given `grammar`-data which can be either a file name, ebnf-code,
python-code, a Grammar-derived grammar class or an instance of
such a class (i.e. a grammar object already).
"""
if isinstance(grammar_representation, str):
# read grammar
grammar_src = load_if_file(grammar_representation)
if is_python_code(grammar_src):
parser_py = grammar_src # type: str
messages = [] # type: List[Error]
else:
lg_dir = suspend_logging()
result, messages, _ = compile_source(
grammar_src, None,
get_ebnf_grammar(), get_ebnf_transformer(), get_ebnf_compiler())
parser_py = cast(str, result)
resume_logging(lg_dir)
if has_errors(messages):
raise DefinitionError(only_errors(messages), grammar_src)
imports = DHPARSER_IMPORTS.format(
dhparser_parentdir=relative_path('.', DHPARSER_PARENTDIR))
grammar_class = compile_python_object(imports + parser_py, r'\w+Grammar$')
if inspect.isclass(grammar_class) and issubclass(grammar_class, Grammar):
parser_root = grammar_class()
else:
raise ValueError('Could not compile or Grammar class!')
else:
# assume that dsl_grammar is a ParserHQ-object or Grammar class
grammar_src = ''
if isinstance(grammar_representation, Grammar):
parser_root = grammar_representation
else:
# assume ``grammar_representation`` is a grammar class and get the root object
parser_root = grammar_representation()
return parser_root, grammar_src
def test_copy2(self):
# test if Node.__deepcopy__ goes sufficiently deep for ast-
# transformation and compiling to perform correctly after copy
ebnf = 'term = term ("*"|"/") factor | factor\nfactor = /[0-9]+/~'
parser = get_ebnf_grammar()
transform = get_ebnf_transformer()
compiler = get_ebnf_compiler()
tree = parser(ebnf)
tree_copy = copy.deepcopy(tree)
transform(tree_copy)
res1 = compiler(tree_copy)
t2 = copy.deepcopy(tree_copy)
res2 = compiler(t2)
diff = ''.join([a for a, b in zip(res1, res2) if a != b])
assert diff.isnumeric() # differences should only be ID-Numbers
tree_copy = copy.deepcopy(tree)
transform(tree_copy)
res3 = compiler(tree_copy)
diff = ''.join([a for a, b in zip(res2, res3) if a != b])
assert diff.isnumeric() # differences should only be ID-Numbers
transform(tree)
res4 = compiler(tree)
diff = ''.join([a for a, b in zip(res3, res4) if a != b])
assert diff.isnumeric() # differences should only be ID-Numbers
def main():
"""Creates a project (if a project name has been passed as command line
parameter) or runs a quick self-test.
"""
access_presets()
set_preset_value('syntax_variant', 'heuristic')
finalize_presets()
if len(sys.argv) > 1:
if sys.argv[1].lower() == "--selftest":
if not selftest():
print("Selftest FAILED :-(\n")
sys.exit(1)
print("Selftest SUCCEEDED :-)\n")
else:
if sys.argv[1].lower() in ('-ast', '--ast'):
try:
with open(sys.argv[2], 'r', encoding='utf-8') as f:
ebnf = f.read()
syntax_tree = get_ebnf_grammar()(ebnf)
if is_error(syntax_tree.error_flag):
print('\n\n'.join(
str(err) for err in syntax_tree.errors_sorted))
sys.exit(1)
get_ebnf_transformer()(syntax_tree)
if is_error(syntax_tree.error_flag):
print('\n\n'.join(
str(err) for err in syntax_tree.errors_sorted))
sys.exit(1)
print(syntax_tree.as_sxpr(compact=True))
except IndexError:
print("Usage: dhparser.py -ast FILENAME.ebnf")
sys.exit(1)
except FileNotFoundError:
print('File "%s" not found!' % sys.arg[2])
sys.exit(1)
except IOError as e:
print('Could not read file "%s": %s' %
(sys.argv[2], str(e)))
elif sys.argv[1].lower() in ('-cpu', '--cpu', '-mem', '--mem'):
try:
func = partial(compile_on_disk, source_file=sys.argv[2])
if sys.argv[1].lower() in ('-cpu', '--cpu'):
_errors = cpu_profile(func, 1)
else:
_errors = mem_profile(func)
if _errors:
print('\n'.join(str(err) for err in _errors))
sys.exit(1)
except IndexError:
print("Usage: dhparser.py -cpu/mem FILENAME.ebnf")
sys.exit(1)
except FileNotFoundError:
print('File "%s" not found!' % sys.arg[2])
sys.exit(1)
except IOError as e:
print('Could not read file "%s": %s' %
(sys.argv[2], str(e)))
elif os.path.exists(sys.argv[1]) and os.path.isfile(sys.argv[1]):
_errors = compile_on_disk(sys.argv[1])
if _errors:
print('\n'.join(str(err) for err in _errors))
sys.exit(1)
else:
print('%s successfully compiled to %s' %
(sys.argv[1],
os.path.splitext(sys.argv[1])[0] + 'Parser.py'))
else:
create_project(sys.argv[1])
else:
print(
'Usage: \n'
' dhparser.py PROJECTNAME - to create a new project\n'
' dhparser.py FILENAME.ebnf - to produce a python-parser from an EBNF-grammar\n'
' dhparser.py --selftest - to run a self-test\n')
choice = input('\nWould you now like to ...\n'
' [1] create a new project\n'
' [2] compile an ebnf-grammar\n'
' [3] run a self-test\n'
' [q] to quit\n'
'Please chose 1, 2 or 3> ')
if choice.strip() == '1':
project_name = input('Please project name or path > ')
create_project(project_name)
elif choice.strip() == '2':
file_path = input('Please enter a file path for compilation > ')
if os.path.exists(file_path) and os.path.isfile(file_path):
_errors = compile_on_disk(file_path)
if _errors:
print('\n\n'.join(str(err) for err in _errors))
sys.exit(1)
else:
print('File %s not found! Aborting.' % file_path)
sys.exit(1)
elif choice.strip() == '3':
if LOGGING:
start_logging(LOGGING)
if not cpu_profile(selftest, 1):
print("Selftest FAILED :-(\n")
sys.exit(1)
print("Selftest SUCCEEDED :-)\n")
elif choice.strip().lower() not in {'q', 'quit', 'exit'}:
print('No valid choice. Goodbye!')