def main(inputname, verbose):
if inputname:
lexer = SujetALexer(FileStream(inputname))
else:
lexer = SujetALexer(StdinStream())
if not verbose:
lexer.removeErrorListener(ConsoleErrorListener.INSTANCE)
counter = CountErrorListener()
lexer._listeners.append(counter)
stream = CommonTokenStream(lexer)
stream.fill()
if counter.count > 0: # wrong token !
print("lexical error")
exit(1)
parser = SujetAParser(stream)
if not verbose:
parser.removeErrorListener(ConsoleErrorListener.INSTANCE)
parser._listeners.append(counter)
parser.start()
if counter.count > 0:
print("syntax error")
exit(2)
else:
print("ok")
exit(0)
def parse(input_: Union[str, FileStream], default_base: Optional[str]=None) -> Optional[Schema]:
"""
Parse the text in infile and return the resulting schema
:param input_: text or input stream to parse
:param default_base_: base URI for relative URI's in schema
:return: ShExJ Schema object. None if error.
"""
# Step 1: Tokenize the input stream
error_listener = ParseErrorListener()
if not isinstance(input_, FileStream):
input_ = InputStream(input_)
lexer = ShExDocLexer(input_)
lexer.addErrorListener(error_listener)
tokens = CommonTokenStream(lexer)
tokens.fill()
if error_listener.n_errors: # Lexer prints errors directly
return None
# Step 2: Generate the parse tree
parser = ShExDocParser(tokens)
parser.addErrorListener(error_listener)
parse_tree = parser.shExDoc()
if error_listener.n_errors:
print('\n'.join(error_listener.errors), file=sys.stderr)
return None
# Step 3: Transform the results the results
parser = ShexDocParser(default_base=default_base)
parser.visit(parse_tree)
return parser.context.schema
def parse(input_: Union[str, InputStream],
default_base: Optional[str] = None) -> Optional[Schema]:
"""
Parse the text in infile and return the resulting schema
:param input_: text or input stream to parse
:param default_base: base URI for relative URI's in schema
:return: ShExJ Schema object. None if error.
"""
# Step 1: Tokenize the input stream
error_listener = ParseErrorListener()
if not isinstance(input_, InputStream):
input_ = InputStream(input_)
lexer = ShExDocLexer(input_)
lexer.addErrorListener(error_listener)
tokens = CommonTokenStream(lexer)
tokens.fill()
if error_listener.n_errors: # Lexer prints errors directly
return None
# Step 2: Generate the parse tree
parser = ShExDocParser(tokens)
parser.addErrorListener(error_listener)
parse_tree = parser.shExDoc()
if error_listener.n_errors:
print('\n'.join(error_listener.errors), file=sys.stderr)
return None
# Step 3: Transform the results the results
parser = ShexDocParser(default_base=default_base)
parser.visit(parse_tree)
return parser.context.schema
def tokenize(string: str) -> Tuple[ASTBuilderVisitor, PyNestMLParser]:
lexer = PyNestMLLexer(InputStream(string))
# create a token stream
stream = CommonTokenStream(lexer)
stream.fill()
parser = PyNestMLParser(stream)
builder = ASTBuilderVisitor(stream.tokens)
return builder, parser
def create_token_stream(self, line):
istream = InputStream(line)
lexer = MetrinkFrontendLexer(istream)
stream = CommonTokenStream(lexer)
stream.fill()
return stream
def lex(i_stream):
"""Makes lexical analysis
Returns: stream of tokens
"""
lexer = Python3Lexer(i_stream)
t_stream = CommonTokenStream(lexer)
t_stream.fill()
return t_stream
def print_tokens(srcfile):
from antlr4 import FileStream, CommonTokenStream
from dynamod.parser.DynamodLexer import DynamodLexer
input = FileStream(srcfile)
lexer = DynamodLexer(input)
stream = CommonTokenStream(lexer)
stream.fill()
for token in stream.getTokens(0, 9999999):
print(str(token))
def split(self, path: str):
input = InputStream(path)
lexer = XPathLexer(input)
def recover(self, e):
raise e
lexer.recover = recover
lexer.removeErrorListeners()
lexer.addErrorListener(
ErrorListener()) # XPathErrorListener does no more
tokenStream = CommonTokenStream(lexer)
try:
tokenStream.fill()
except LexerNoViableAltException as e:
pos = lexer.column
msg = "Invalid tokens or characters at index %d in path '%s'" % (
pos, path)
raise Exception(msg, e)
tokens = iter(tokenStream.tokens)
elements = list()
for el in tokens:
invert = False
anywhere = False
# Check for path separators, if none assume root
if el.type in [XPathLexer.ROOT, XPathLexer.ANYWHERE]:
anywhere = el.type == XPathLexer.ANYWHERE
next_el = next(tokens, None)
if not next_el:
raise Exception('Missing element after %s' % el.getText())
else:
el = next_el
# Check for bangs
if el.type == XPathLexer.BANG:
invert = True
next_el = next(tokens, None)
if not next_el:
raise Exception('Missing element after %s' % el.getText())
else:
el = next_el
# Add searched element
if el.type in [
XPathLexer.TOKEN_REF, XPathLexer.RULE_REF,
XPathLexer.WILDCARD, XPathLexer.STRING
]:
element = self.getXPathElement(el, anywhere)
element.invert = invert
elements.append(element)
elif el.type == Token.EOF:
break
else:
raise Exception("Unknown path element %s" %
lexer.symbolicNames[el.type])
return elements
def split(self, path: str):
input = InputStream(path)
lexer = XPathLexer(input)
def recover(self, e):
raise e
lexer.recover = recover
lexer.removeErrorListeners()
lexer.addErrorListener(
ErrorListener()) # XPathErrorListener does no more
tokenStream = CommonTokenStream(lexer)
try:
tokenStream.fill()
except LexerNoViableAltException as e:
pos = lexer.getColumn()
msg = "Invalid tokens or characters at index " + str(
pos) + " in path '" + path + "'"
raise Exception(msg, e)
tokens = tokenStream.getTokens()
elements = list()
n = len(tokens)
i = 0
while i < n:
el = tokens[i]
next = None
if el.type in [XPathLexer.ROOT, XPathLexer.ANYWHERE]:
anywhere = el.type == XPathLexer.ANYWHERE
i += 1
next = tokens[i]
invert = next.type == XPathLexer.BANG
if invert:
i += 1
next = tokens[i]
pathElement = self.getXPathElement(next, anywhere)
pathElement.invert = invert
elements.append(pathElement)
i += 1
elif el.type in [
XPathLexer.TOKEN_REF, XPathLexer.RULE_REF,
XPathLexer.WILDCARD
]:
elements.append(self.getXPathElement(el, False))
i += 1
elif el.type == Token.EOF:
break
else:
raise Exception("Unknown path element " + str(el))
return elements
def get_stream(Lexer, file_path, strip_eof=True):
inp = FileStream(file_path)
lexer = Lexer(inp)
stream = CommonTokenStream(lexer)
stream.fill()
tokens = stream.tokens
if strip_eof:
new_stream = [t.type for t in tokens[:-1]]
else:
new_stream = [t.type for t in tokens]
return new_stream, tokens
def main():
# InputStream reads characters (from stdin in our case)
input_stream = InputStream(sys.stdin.read())
# The generated lexer groups characters into Tokens ...
lexer = Example1(input_stream)
# ... and the stream of Tokens is managed by the TokenStream.
stream = CommonTokenStream(lexer)
# Display the token stream
stream.fill() # needed to get stream.tokens (otherwise lazily filled-in)
for t in stream.tokens:
print(t)
print("Finished")
def split(self, path: str):
input = InputStream(path)
lexer = XPathLexer(input)
def recover(self, e):
raise e
lexer.recover = recover
lexer.removeErrorListeners()
lexer.addErrorListener(ErrorListener()) # XPathErrorListener does no more
tokenStream = CommonTokenStream(lexer)
try:
tokenStream.fill()
except LexerNoViableAltException as e:
pos = lexer.getColumn()
msg = "Invalid tokens or characters at index " + str(pos) + " in path '" + path + "'"
raise Exception(msg, e)
tokens = tokenStream.getTokens()
elements = list()
n = len(tokens)
i = 0
while i < n:
el = tokens[i]
next = None
if el.type in [XPathLexer.ROOT, XPathLexer.ANYWHERE]:
anywhere = el.type == XPathLexer.ANYWHERE
i += 1
next = tokens[i]
invert = next.type == XPathLexer.BANG
if invert:
i += 1
next = tokens[i]
pathElement = self.getXPathElement(next, anywhere)
pathElement.invert = invert
elements.append(pathElement)
i += 1
elif el.type in [XPathLexer.TOKEN_REF, XPathLexer.RULE_REF, XPathLexer.WILDCARD]:
elements.append(self.getXPathElement(el, False))
i += 1
elif el.type == Token.EOF:
break
else:
raise Exception("Unknown path element " + str(el))
return elements
def parse(text: str, production_rule: str,
listener) -> Optional[jsgParserVisitor]:
"""
Parse text fragment according to supplied production rule and evaluate with listener class.
Example: parse("{1,*}", "ebnfSuffix", JSGEbnf)
"""
error_listener = ParseErrorListener()
lexer = jsgLexer(InputStream(text))
lexer.addErrorListener(error_listener)
tokens = CommonTokenStream(lexer)
tokens.fill()
if error_listener.n_errors:
return None
parser = jsgParser(tokens)
parser.addErrorListener(error_listener)
base_node = getattr(parser, production_rule)()
listener_module = listener(JSGDocContext())
listener_module.visit(base_node)
return listener_module if not error_listener.n_errors else None
def parse(input_: Union[str, FileStream], source: str, emit_header: bool=True) -> Optional[str]:
"""Parse the text in infile and save the results in outfile
:param input_: string or stream to parse
:param source: source name for python file header
:param emit_header: True means include header in python file
:return: python text if successful
"""
# Step 1: Tokenize the input stream
error_listener = ParseErrorListener()
if not isinstance(input_, FileStream):
input_ = InputStream(input_)
lexer = jsgLexer(input_)
lexer.addErrorListener(error_listener)
tokens = CommonTokenStream(lexer)
tokens.fill()
if error_listener.n_errors:
return None
# Step 2: Generate the parse tree
parser = jsgParser(tokens)
parser.addErrorListener(error_listener)
parse_tree = parser.doc()
if error_listener.n_errors:
return None
# Step 3: Transform the results the results
parser = JSGDocParser()
parser.visit(parse_tree)
if parser.undefined_tokens():
for tkn in parser.undefined_tokens():
print("Undefined token: " + tkn)
return None
return parser.as_python(source, emit_header=emit_header)
import sys
import argparse
from antlr4 import FileStream, InputStream, CommonTokenStream
from SimpleJavaLexer import SimpleJavaLexer
from SimpleJavaParser import SimpleJavaParser
from SimpleJavaErrorListener import VerboseListener
if __name__ == '__main__':
p = argparse.ArgumentParser()
p.add_argument('-f', '--file', default=None)
p.add_argument('-t', '--tokens', action="store_true")
args = p.parse_args()
if args.file:
input_stream = FileStream(args.file, encoding='utf8')
else:
input_stream = InputStream(sys.stdin.read())
lexer = SimpleJavaLexer(input_stream)
token_stream = CommonTokenStream(lexer)
token_stream.fill()
if args.tokens:
for tok in token_stream.tokens:
print(tok)
parser = SimpleJavaParser(token_stream)
parser.removeErrorListeners()
parser.addErrorListener(VerboseListener())
parser.prog()
tree_representation += (" " *
indent) + f"TOKEN='{tree.getText()}'\n"
else:
tree_representation += (
" " * indent) + f"{rule_names[tree.getRuleIndex()]}\n"
for child in tree.children:
generate_indented_tree(child, rule_names, indent + 1)
if __name__ == "__main__":
input_data = FileStream(sys.argv[1])
tokens_print = []
lexer = HelloLexer(input_data)
stream = CommonTokenStream(lexer)
stream.fill()
for token in stream.tokens:
if token.type != Token.EOF:
token_attr = None
token_type = token.type
token_line = token.line
token_text = token.text
if token_type == lexer.ASSIGNMENT:
token_attr = "ASSIGNMENT"
if token_type == lexer.BEGIN:
token_attr = "BEGIN"
if token_type == lexer.BOOL:
token_attr = "BOOL"
if token_type == lexer.CLOSE_PARENTHESIS:
def getTokensFromText(input_string_to_parse, tLexer):
input_stream = InputStream(input_string_to_parse)
lexer = tLexer(input_stream) #lexer object is created
stream = CommonTokenStream(lexer)
stream.fill()
return stream.tokens
def parse_model(cls, file_path=None):
"""
Parses a handed over model and returns the meta_model representation of it.
:param file_path: the path to the file which shall be parsed.
:type file_path: str
:return: a new ASTNESTMLCompilationUnit object.
:rtype: ASTNestMLCompilationUnit
"""
try:
input_file = FileStream(file_path)
except IOError:
code, message = Messages.get_input_path_not_found(path=file_path)
Logger.log_message(node=None, code=None, message=message,
error_position=None, log_level=LoggingLevel.ERROR)
return
code, message = Messages.get_start_processing_file(file_path)
Logger.log_message(node=None, code=code, message=message, error_position=None, log_level=LoggingLevel.INFO)
# create a lexer and hand over the input
lexer = PyNestMLLexer()
lexer.removeErrorListeners()
lexer.addErrorListener(ConsoleErrorListener())
lexerErrorListener = NestMLErrorListener()
lexer.addErrorListener(lexerErrorListener)
# lexer._errHandler = BailErrorStrategy() # N.B. uncomment this line and the next to halt immediately on lexer errors
# lexer._errHandler.reset(lexer)
lexer.inputStream = input_file
# create a token stream
stream = CommonTokenStream(lexer)
stream.fill()
if lexerErrorListener._error_occurred:
code, message = Messages.get_lexer_error()
Logger.log_message(node=None, code=None, message=message,
error_position=None, log_level=LoggingLevel.ERROR)
return
# parse the file
parser = PyNestMLParser(None)
parser.removeErrorListeners()
parser.addErrorListener(ConsoleErrorListener())
parserErrorListener = NestMLErrorListener()
parser.addErrorListener(parserErrorListener)
# parser._errHandler = BailErrorStrategy() # N.B. uncomment this line and the next to halt immediately on parse errors
# parser._errHandler.reset(parser)
parser.setTokenStream(stream)
compilation_unit = parser.nestMLCompilationUnit()
if parserErrorListener._error_occurred:
code, message = Messages.get_parser_error()
Logger.log_message(node=None, code=None, message=message,
error_position=None, log_level=LoggingLevel.ERROR)
return
# create a new visitor and return the new AST
ast_builder_visitor = ASTBuilderVisitor(stream.tokens)
ast = ast_builder_visitor.visit(compilation_unit)
# create and update the corresponding symbol tables
SymbolTable.initialize_symbol_table(ast.get_source_position())
for neuron in ast.get_neuron_list():
neuron.accept(ASTSymbolTableVisitor())
SymbolTable.add_neuron_scope(neuron.get_name(), neuron.get_scope())
# store source paths
for neuron in ast.get_neuron_list():
neuron.file_path = file_path
ast.file_path = file_path
return ast
