def build_parse_pipeline(self, file: TextIOBase, makefile: Makefile) -> Iterator[Target]:
file_lines = FileLineIterator.make(file)
sub_conditions = [DatabaseSectionFilter.make(), FileSectionFilter.make(), InformationalCommentFilter.make()]
filtered_lines = ConditionFilter.make(file_lines, CompositeCondition.make(sub_conditions))
file_section_chars = LineToCharIterator.make(filtered_lines)
char_stream_1 = None
with StringIO() as strbuff:
if file_section_chars.is_at_start:
file_section_chars.move_to_next()
while file_section_chars.has_current_item:
strbuff.write(file_section_chars.current_item)
file_section_chars.move_to_next()
char_stream_1 = InputStream(cast(StringIO, strbuff).getvalue())
paragraph_lexer = TargetParagraphLexer(char_stream_1)
paragraph_tokens = TokenSourceToIteratorAdapter.make(paragraph_lexer)
paragraph_chars = TokenToCharIterator.make(paragraph_tokens)
char_stream_2 = None
with StringIO() as strbuff:
if paragraph_chars.is_at_start:
paragraph_chars.move_to_next()
while paragraph_chars.has_current_item:
strbuff.write(paragraph_chars.current_item)
paragraph_chars.move_to_next()
char_stream_2 = InputStream(cast(StringIO, strbuff).getvalue())
makefile_rule_lexer = MakefileRuleLexer(char_stream_2)
token_stream_1 = CommonTokenStream(makefile_rule_lexer)
makefile_rule_parser = MakefileRuleParser(token_stream_1)
makefile_target_iterator = MakefileRuleParserToIteratorAdapter.make(makefile_rule_parser)
makefile_target_iterator.makefile = makefile
return makefile_target_iterator
def __init__(self, path=None, stream=None, text=None):
chars = None
self.path = path
if stream is not None:
bytes = stream.read()
data = codecs.decode(bytes)
chars = InputStream(data)
stream.close()
elif text is not None:
chars = InputStream(text)
if chars is not None:
lexer = MIndentingLexer(chars)
tokens = CommonTokenStream(lexer)
super().__init__(tokens)
def __init__(self, path=None, stream=None, text=None):
chars = None
self.path = path
if stream is not None:
bytes = stream.read()
data = bytes if isinstance(bytes, unicode) else codecs.decode(
bytes, "utf-8")
chars = InputStream(data)
stream.close()
elif text is not None:
chars = InputStream(text)
if chars is not None:
lexer = MIndentingLexer(chars)
tokens = CommonTokenStream(lexer)
super(MCleverParser, self).__init__(tokens)
def main(argv):
global mode, build_parse_tree, token_stream
input_file_name = "default_input_file"
if len(argv) > 1:
input_file_name = argv[1]
if len(argv) > 2:
mode = argv[2].lower()
if len(argv) > 3:
build_parse_tree = argv[3].lower() == "true"
data = ""
with open("default_input_file", "r") as file:
data = file.read().replace("\n", "")
code_stream = InputStream(data)
lexer = __TemplateGrammarName__Lexer(code_stream)
token_stream = CommonTokenStream(lexer)
__TemplateGrammarName_____RuntimeName__()
benchmark(True, warm_up_count)
time = benchmark(False, iteration_count)
with open("__TemplateGrammarName_____RuntimeName__.benchmark",
"w") as result_file:
result_file.write(str(time))
def parse(self):
if self.spec is None:
raise STLParseException('STL specification if empty')
# Parse the STL spec - ANTLR4 magic
entire_spec = self.modular_spec + self.spec
input_stream = InputStream(entire_spec)
lexer = StlLexer(input_stream)
stream = CommonTokenStream(lexer)
parser = StlParser(stream)
parser._listeners = [STLParserErrorListener()]
ctx = parser.specification_file()
# Create the visitor for the actual spec nodes
visitor = STLSpecificationParser(self)
self.top = visitor.visitSpecification_file(ctx)
# print('Hello')
# print(self.unit)
# print('sampling period unit: ' + str(self.sampling_period_unit))
# print(self.U[self.unit])
# print(self.U[self.sampling_period_unit])
self.normalize = float(self.U[self.unit]) / float(
self.U[self.sampling_period_unit])
def tokenize(self, pattern: str):
# split pattern into chunks: sea (raw input) and islands (<ID>, <expr>)
chunks = self.split(pattern)
# create token stream from text and tags
tokens = list()
for chunk in chunks:
if isinstance(chunk, TagChunk):
# add special rule token or conjure up new token from name
if chunk.tag[0].isupper():
ttype = self.parser.getTokenType(chunk.tag)
if ttype == Token.INVALID_TYPE:
raise Exception("Unknown token " + str(chunk.tag) +
" in pattern: " + pattern)
tokens.append(TokenTagToken(chunk.tag, ttype, chunk.label))
elif chunk.tag[0].islower():
ruleIndex = self.parser.getRuleIndex(chunk.tag)
if ruleIndex == -1:
raise Exception("Unknown rule " + str(chunk.tag) +
" in pattern: " + pattern)
ruleImaginaryTokenType = self.parser.getATNWithBypassAlts(
).ruleToTokenType[ruleIndex]
tokens.append(
RuleTagToken(chunk.tag, ruleImaginaryTokenType,
chunk.label))
else:
raise Exception("invalid tag: " + str(chunk.tag) +
" in pattern: " + pattern)
else:
self.lexer.setInputStream(InputStream(chunk.text))
t = self.lexer.nextToken()
while t.type != Token.EOF:
tokens.append(t)
t = self.lexer.nextToken()
return tokens
def parse(actions, spdOutput, two_stageParsing):
tokens = CommonTokenStream(SPDGrammarLexer(InputStream(spdOutput)))
parser = SPDGrammarParser(tokens)
visitor = SPDGrammarVisitorImplementation(actions)
if not two_stageParsing:
visitor.visit(parser.json())
return visitor._errors
parser._interp.predictionMode = PredictionMode.SLL
parser.removeErrorListeners()
parser._errHandler = BailErrorStrategy()
try:
visitor.visit(parser.json())
except RuntimeError as exception:
if isinstance(exception, RecognitionException):
tokens.seek(0)
parser.addErrorListener(ConsoleErrorListener.INSTANCE)
parser._errHandler = DefaultErrorStrategy()
parser._interp.predictionMode = PredictionMode.LL
visitor.visit(parser.json())
return visitor._errors
def morph(cls, src, slicing=False):
#filename = "Test4.sparql"
#inputstream = antlr4.InputStream.InputStream(src)
inputstream = InputStream(src)
lexer = SparqlLexer(inputstream)
stream = antlr4.CommonTokenStream(lexer)
parser = SparqlParser(stream)
#tree = parser.StartRule()
tree = parser.query()
#fmind = Fmind(gname)
#fnode = fmind.make_right(u"root")
#toFmind(fnode, tree)
#fmind.unfold_all()
#fmind.dump_to_file("l.mm")
#tree = parser.prologue()
morpher = MorpherContext3(slicing=slicing)
listener = MySparqlParserListener(morpher)
walker = antlr4.ParseTreeWalker()
walker.walk(listener, tree)
#logging.info("Output:%s", sys.argv[1])
#print "# ", sys.argv[1]
return morpher.get_result()
def measure(db_path, graph, grammar_path, algorithm):
grammar = ""
start = None
with open(grammar_path, "r") as f:
for line in f:
lhs, rhs = line.strip().split(" ", 1)
if start is None:
start = lhs
grammar += f"{lhs} = {rhs};\n"
script = f"""
connect "{db_path}";
{grammar}
select a, b from "{graph}" where path(a, b, {start}) using "{algorithm}";
"""
print(script)
t1 = time.time()
parsed = parse_query(InputStream(script))
executor = Executor(FileDatabase())
executor.execute_many(parsed)
t2 = time.time()
return t2 - t1
def main(argv):
data = generate_data()
code_stream = InputStream(data)
lexer = LeftRecursionGrammarLexer(code_stream)
global mode, tokens_stream
tokens_stream = CommonTokenStream(lexer)
mode = "SLL"
sll_left_recursion = benchmark_average("left_recursion_test", "SLL")
sll_not_left_recursion = benchmark_average("not_left_recursion_test",
"SLL")
mode = "LL"
ll_left_recursion = benchmark_average("left_recursion_test", "LL")
ll_not_left_recursion = benchmark_average("not_left_recursion_test", "LL")
print("")
print("SLL:")
print(f"left recursion: {sll_left_recursion} us")
print(f"not left recursion: {sll_not_left_recursion} us")
print(f"ratio: {sll_not_left_recursion / sll_left_recursion:0.2f}")
print("")
print("LL:")
print(f"left recursion: {ll_left_recursion} us")
print(f"not left recursion: {ll_not_left_recursion} us")
print(f"ratio: {ll_not_left_recursion / ll_left_recursion:0.2f}")
def evaluateDTS(varTs, line):
istream = InputStream(line)
lexer = ExprLexer(istream)
token_stream = CommonTokenStream(lexer)
parser = ExprParser(token_stream)
parser.varTs = varTs
parser.expression()
def load_ast_from_str(data):
f = InputStream(data)
lexer = openflowLexer(f)
lexer.addErrorListener(MyErrorListener())
stream = CommonTokenStream(lexer)
parser = openflowParser(stream)
parser.addErrorListener(MyErrorListener())
tree = parser.openflow_dump_text()
return tree
def newTokenStreamFromResource(self, resourceName):
input_ = None #ClassLoader.getSystemClassLoader().getResourceAsStream(resourceName)
self.assertIsNotNone(input_)
try:
stream = InputStream(input)
return ONamingLexer(stream)
except Exception as e:
self.fail(e.text)
return None
def eval_ra_expr(database, ra_str):
input_stream = InputStream(ra_str)
lexer = RelationalAlgebraLexer(input_stream)
token_stream = CommonTokenStream(lexer)
parser = RelationalAlgebraParser(token_stream)
parse_tree = parser.relStmt()
visitor = RelationalAlgebraEvaluator(database)
return visitor.visit(parse_tree)
def testInsertAfterLastIndex(self):
input = InputStream('abc')
lexer = TestLexer(input)
stream = CommonTokenStream(lexer=lexer)
stream.fill()
rewriter = TokenStreamRewriter(tokens=stream)
rewriter.insertAfter(10, 'x')
self.assertEquals(rewriter.getDefaultText(), 'abcx')
def testInsertBeforeIndexZero(self):
input = InputStream('abc')
lexer = TestLexer(input)
stream = CommonTokenStream(lexer=lexer)
stream.fill()
rewriter = TokenStreamRewriter(tokens=stream)
rewriter.insertBeforeIndex(0, '0')
self.assertEquals(rewriter.getDefaultText(), '0abc')
def testReplaceMiddleIndex(self):
input = InputStream('abc')
lexer = TestLexer(input)
stream = CommonTokenStream(lexer=lexer)
stream.fill()
rewriter = TokenStreamRewriter(tokens=stream)
rewriter.replaceIndex(1, 'x')
self.assertEquals(rewriter.getDefaultText(), 'axc')
def testReplaceSubsetThenFetch(self):
input = InputStream('abcccba')
lexer = TestLexer(input)
stream = CommonTokenStream(lexer=lexer)
stream.fill()
rewriter = TokenStreamRewriter(tokens=stream)
rewriter.replaceRange(2, 4, 'xyz')
self.assertEquals('abxyzba', rewriter.getDefaultText())
def testReplaceAll(self):
input = InputStream('abcccba')
lexer = TestLexer(input)
stream = CommonTokenStream(lexer=lexer)
stream.fill()
rewriter = TokenStreamRewriter(tokens=stream)
rewriter.replaceRange(0, 6, 'x')
self.assertEquals('x', rewriter.getDefaultText())
def _parse(input):
input_stream = InputStream(input)
lexer = PromiseLexer(input_stream)
token_stream = CommonTokenStream(lexer)
parser = PromiseParser(token_stream)
parser._listeners = [SyntaxErrorListener()]
tree = parser.prog()
visitor = PromiseSyntaxVisitor()
visitor.visit(tree)
return json.dumps(visitor.get_result())
def main():
nom = sys.argv[1]
print("Llegint el fitxer " + nom)
inputStream = InputStream(open('cl/input/' + nom).read().strip())
tokenStream = LexerPlusTokenizer(inputStream)
tree = Parser(tokenStream)
visitor = EnquestesVisitor()
nodes, arestes = visitor.visit(tree)
nom, graf = crearGraf(nodes, arestes)
pickleDump(graf, nom)
def getLexerFromString(inputText):
input_stream = InputStream(inputText)
lexer = QLGrammarLexer(input_stream)
token_stream = CommonTokenStream(lexer)
parser = QLGrammarParser(token_stream)
parser._listeners = [MyErrorListener()]
tree = parser.form()
tree_str = tree.toStringTree(recog=parser)
return str(tree_str)
def main(argv):
fileName = '../../Text'
if len(argv) > 0:
fileName = argv[1]
code = open(fileName, 'r').read()
codeStream = InputStream(code)
lexer = __TemplateGrammarName__Lexer(codeStream)
tokens = lexer.getAllTokens()
def parse(cls, statment):
stream = InputStream(statment)
lexer = MySQLLexer(stream);
stream = CommonTokenStream(lexer)
parser = MySQLParser(stream)
tree = parser.stat()
visitor = Visitor(cls.dynamodb)
if isinstance(visitor, ParseTreeVisitor):
parsed_stat = visitor.visit(tree)
return parsed_stat
def split(self, path: str):
input = InputStream(path)
lexer = XPathLexer(input)
def recover(self, e):
raise e
lexer.recover = recover
lexer.removeErrorListeners()
lexer.addErrorListener(
ErrorListener()) # XPathErrorListener does no more
tokenStream = CommonTokenStream(lexer)
try:
tokenStream.fill()
except LexerNoViableAltException as e:
pos = lexer.column
msg = "Invalid tokens or characters at index %d in path '%s'" % (
pos, path)
raise Exception(msg, e)
tokens = iter(tokenStream.tokens)
elements = list()
for el in tokens:
invert = False
anywhere = False
# Check for path separators, if none assume root
if el.type in [XPathLexer.ROOT, XPathLexer.ANYWHERE]:
anywhere = el.type == XPathLexer.ANYWHERE
next_el = next(tokens, None)
if not next_el:
raise Exception('Missing element after %s' % el.getText())
else:
el = next_el
# Check for bangs
if el.type == XPathLexer.BANG:
invert = True
next_el = next(tokens, None)
if not next_el:
raise Exception('Missing element after %s' % el.getText())
else:
el = next_el
# Add searched element
if el.type in [
XPathLexer.TOKEN_REF, XPathLexer.RULE_REF,
XPathLexer.WILDCARD, XPathLexer.STRING
]:
element = self.getXPathElement(el, anywhere)
element.invert = invert
elements.append(element)
elif el.type == Token.EOF:
break
else:
raise Exception("Unknown path element %s" %
lexer.symbolicNames[el.type])
return elements
def testInsertBeforeTokenThenDeleteThatToken(self):
input = InputStream('abc')
lexer = TestLexer(input)
stream = CommonTokenStream(lexer=lexer)
stream.fill()
rewriter = TokenStreamRewriter(tokens=stream)
rewriter.insertBeforeIndex(1, 'foo')
rewriter.replaceRange(1, 2, 'foo')
self.assertEquals('afoofoo', rewriter.getDefaultText())
def testLeaveAloneDisjointInsert2(self):
input = InputStream('abcc')
lexer = TestLexer(input)
stream = CommonTokenStream(lexer=lexer)
stream.fill()
rewriter = TokenStreamRewriter(tokens=stream)
rewriter.replaceRange(2, 3, 'foo')
rewriter.insertBeforeIndex(1, 'x')
self.assertEquals('axbfoo', rewriter.getDefaultText())
def testDropIdenticalReplace(self):
input = InputStream('abcc')
lexer = TestLexer(input)
stream = CommonTokenStream(lexer=lexer)
stream.fill()
rewriter = TokenStreamRewriter(tokens=stream)
rewriter.replaceRange(1, 2, 'foo')
rewriter.replaceRange(1, 2, 'foo')
self.assertEquals('afooc', rewriter.getDefaultText())
def testOverlappingReplace4(self):
input = InputStream('abcc')
lexer = TestLexer(input)
stream = CommonTokenStream(lexer=lexer)
stream.fill()
rewriter = TokenStreamRewriter(tokens=stream)
rewriter.replaceRange(1, 2, 'foo')
rewriter.replaceRange(1, 3, 'bar')
self.assertEquals('abar', rewriter.getDefaultText())
def testCombineInsertOnLeftWithDelete(self):
input = InputStream('abc')
lexer = TestLexer(input)
stream = CommonTokenStream(lexer=lexer)
stream.fill()
rewriter = TokenStreamRewriter(tokens=stream)
rewriter.delete('default', 0, 2)
rewriter.insertBeforeIndex(0, 'z')
self.assertEquals('z', rewriter.getDefaultText())
