def test9th(self):
exprs = compile_exprs([ r"""
~("a", error "literal 'a' should not appear" | any);
""" ])
assert len(exprs) == 1
inputText = r'b,c,d'
seq = [ 'code' ]; seq.extend(split_to_strings_iter(inputText))
seq = exprs[0].parse(seq)
self.assertEqual(seq, [ 'code', 'b', ',', 'c', ',', 'd' ])
inputText = r'a,b,c'
seq = [ 'code' ]; seq.extend(split_to_strings_iter(inputText))
self.assertRaises(pyrem_torq.expression.InterpretErrorByErrorExpr, exprs[0].parse, seq)
def test5th(self):
atoz = 'r"^[a-z]"'
exprStrs = [
r'~(req(%(atoz)s), ((op_logical_and <- "and") | (op_logical_or <- "or")), req^(%(atoz)s));'
% {
'atoz': atoz
}
]
exprs = compile_exprs(exprStrs)
inputText = r'if (x and y or z) printf("hello\n");'
inputText = inputText.decode(sys.getfilesystemencoding())
seq = ['code']
seq.extend(split_to_strings_iter(inputText))
for expr in exprs:
newSeq = expr.parse(seq)
self.assertTrue(newSeq, None)
seq = newSeq
foundAnd, foundOr = False, False
for item in seq:
if isinstance(item, list):
if item[0:1] == ['op_logical_and']: foundAnd = True
if item[0:1] == ['op_logical_or']: foundOr = True
self.assertTrue(foundAnd)
self.assertTrue(foundOr)
print "result seq=", "\n".join(pyrem_torq.treeseq.seq_pretty(seq))
def test3rd(self):
exprStrs = [
r'~(eol <- "\t" | "\f" | "\v" | "\r" | "\n");',
r'~(comment <- "/", "*", *(+"*", (req^("/"), any) | req^("*"), any), +"*", "/");',
r'~(comment <- "/", "/", *(req^(eol), any));',
]
exprs = compile_exprs(exprStrs)
inputText = """
#include <stdio.h> // import printf()
int main(int argc, char *argv[])
{
/************************
** the arguments argc/**
** argv are not used. **
************************/
printf("hello, world.\n");
return 0;
}
"""[1:-1]
seq = ['code']
seq.extend(split_to_strings_iter(inputText))
for expr in exprs:
posDelta, outSeq, dropSeq = expr.match(seq, 1)
self.assertEqual(1 + posDelta, len(seq))
seq = [seq[0]] + outSeq
print "result seq=", "\n".join(pyrem_torq.treeseq.seq_pretty(seq))
def test3rd(self):
exprStrs = [
r'~(eol <- "\t" | "\f" | "\v" | "\r" | "\n");',
r'~(comment <- "/", "*", *(+"*", (req^("/"), any) | req^("*"), any), +"*", "/");',
r'~(comment <- "/", "/", *(req^(eol), any));',
]
exprs = compile_exprs(exprStrs)
inputText = """
#include <stdio.h> // import printf()
int main(int argc, char *argv[])
{
/************************
** the arguments argc/**
** argv are not used. **
************************/
printf("hello, world.\n");
return 0;
}
"""[1:-1]
seq = [ 'code' ]; seq.extend(split_to_strings_iter(inputText))
for expr in exprs:
posDelta, outSeq, dropSeq = expr.match(seq, 1)
self.assertEqual(1 + posDelta, len(seq))
seq = [ seq[0] ] + outSeq
print "result seq=", "\n".join(pyrem_torq.treeseq.seq_pretty(seq))
def test1st(self):
exprStrs = [
r'~((v <- +(r"^\d" | ".")) | (null <- +(" " | "\t")));',
r'~(v <- (null <- "("), +(@0 | req^("(" | ")"), any), (null <- ")"));',
r"""
?(v <- (u_op <- "+" | "-"), (v :: ~@0)),
*(
(v :: ~@0), *("+" | "-")
| (v <- (u_op <- "+" | "-"), (v :: ~@0))
| any
);
""",
r'~((v <- (v :: ~@0), +((b_op <- "**"), (v :: ~@0))) | (v :: ~@0));',
r'~((v <- (v :: ~@0), +((b_op <- "*" | "/"), (v :: ~@0))) | (v :: ~@0));',
r'~((v <- (v :: ~@0), +((b_op <- "+" | "-"), (v :: ~@0))) | (v :: ~@0));',
]
exprs = compile_exprs(exprStrs)
seq = [ 'code' ]; seq.extend(split_to_strings_iter("+1.0 + 2 * ((3 - 4) / -.5) ** 6"))
for exprIndex, expr in enumerate(exprs):
print "exprIndex=", exprIndex, "cur seq=", "\n".join(pyrem_torq.treeseq.seq_pretty(seq))
newSeq = expr.parse(seq)
self.assertTrue(newSeq, None)
seq = newSeq
print "result seq=", "\n".join(pyrem_torq.treeseq.seq_pretty(seq))
def test1st(self):
exprStrs = [
r'~((v <- +(r"^\d" | ".")) | (null <- +(" " | "\t")));',
r'~(v <- (null <- "("), +(@0 | req^("(" | ")"), any), (null <- ")"));',
r"""
?(v <- (u_op <- "+" | "-"), (v :: ~@0)),
*(
(v :: ~@0), *("+" | "-")
| (v <- (u_op <- "+" | "-"), (v :: ~@0))
| any
);
""",
r'~((v <- (v :: ~@0), +((b_op <- "**"), (v :: ~@0))) | (v :: ~@0));',
r'~((v <- (v :: ~@0), +((b_op <- "*" | "/"), (v :: ~@0))) | (v :: ~@0));',
r'~((v <- (v :: ~@0), +((b_op <- "+" | "-"), (v :: ~@0))) | (v :: ~@0));',
]
exprs = compile_exprs(exprStrs)
seq = ['code']
seq.extend(split_to_strings_iter("+1.0 + 2 * ((3 - 4) / -.5) ** 6"))
for exprIndex, expr in enumerate(exprs):
print "exprIndex=", exprIndex, "cur seq=", "\n".join(
pyrem_torq.treeseq.seq_pretty(seq))
newSeq = expr.parse(seq)
self.assertTrue(newSeq, None)
seq = newSeq
print "result seq=", "\n".join(pyrem_torq.treeseq.seq_pretty(seq))
def tkn(inputText, inputFilePath=None):
if inputText.startswith(bom):
inputText = inputText[len(bom):] # remove bom
seq = ['code']
seq.extend(split_to_strings_iter(inputText, textSplitPattern))
seq.append(['eof']) # insert eof node at the end of sequence
return seq
def test9th(self):
exprs = compile_exprs([
r"""
~("a", error "literal 'a' should not appear" | any);
"""
])
assert len(exprs) == 1
inputText = r'b,c,d'
seq = ['code']
seq.extend(split_to_strings_iter(inputText))
seq = exprs[0].parse(seq)
self.assertEqual(seq, ['code', 'b', ',', 'c', ',', 'd'])
inputText = r'a,b,c'
seq = ['code']
seq.extend(split_to_strings_iter(inputText))
self.assertRaises(pyrem_torq.expression.InterpretErrorByErrorExpr,
exprs[0].parse, seq)
def test6th(self):
searchWordLike = r'~(wordlike <- "_", *(r"^[a-zA-Z]" | r"\d" | "_") | r"^[a-zA-Z]", *(r"^[a-zA-Z]" | r"\d" | "_"));'
exprs = compile_exprs([ searchWordLike ])
assert len(exprs) == 1
inputText = r'argv[0];'
inputText = inputText.decode(sys.getfilesystemencoding())
seq = [ 'code' ]; seq.extend(split_to_strings_iter(inputText))
seq = exprs[0].parse(seq)
self.assertEqual(seq[1], [ 'wordlike', u'argv' ])
print "result seq=", "\n".join(pyrem_torq.treeseq.seq_pretty(seq))
def test6th(self):
searchWordLike = r'~(wordlike <- "_", *(r"^[a-zA-Z]" | r"\d" | "_") | r"^[a-zA-Z]", *(r"^[a-zA-Z]" | r"\d" | "_"));'
exprs = compile_exprs([searchWordLike])
assert len(exprs) == 1
inputText = r'argv[0];'
inputText = inputText.decode(sys.getfilesystemencoding())
seq = ['code']
seq.extend(split_to_strings_iter(inputText))
seq = exprs[0].parse(seq)
self.assertEqual(seq[1], ['wordlike', u'argv'])
print "result seq=", "\n".join(pyrem_torq.treeseq.seq_pretty(seq))
def test4th(self):
IN = pyrem_torq.expression.InsertNode.build
BtN = pyrem_torq.expression.BuildToNode.build
L = pyrem_torq.expression.Literal.build
A = pyrem_torq.expression.Any.build
Q = pyrem_torq.expression.Req.build
S = pyrem_torq.expression.Search.build
eolExpr = BtN('eol', L("\r\n") | L("\n") | L("\r"))
expr = S(eolExpr) + Q(pyrem_torq.expression.EndOfNode()) + IN('eof')
seq = [ 'code' ]; seq.extend(split_to_strings_iter("abc\n"))
posDelta, outSeq, dropSeq = expr.match(seq, 1)
self.assertEqual(1 + posDelta, 3)
self.assertEqual(outSeq, [ 'abc', [ 'eol', '\n' ], [ 'eof' ] ])
def test4th(self):
IN = pyrem_torq.expression.InsertNode.build
BtN = pyrem_torq.expression.BuildToNode.build
L = pyrem_torq.expression.Literal.build
A = pyrem_torq.expression.Any.build
Q = pyrem_torq.expression.Req.build
S = pyrem_torq.expression.Search.build
eolExpr = BtN('eol', L("\r\n") | L("\n") | L("\r"))
expr = S(eolExpr) + Q(pyrem_torq.expression.EndOfNode()) + IN('eof')
seq = ['code']
seq.extend(split_to_strings_iter("abc\n"))
posDelta, outSeq, dropSeq = expr.match(seq, 1)
self.assertEqual(1 + posDelta, 3)
self.assertEqual(outSeq, ['abc', ['eol', '\n'], ['eof']])
def test7th(self):
searchWordLike = r'~(word <- ("_", *("_" | r"^[a-zA-Z]" | r"^\d") | r"^[a-zA-Z]", *("_" | r"^[a-zA-Z]" | r"^\d")));'
searchIdMake = r'~(id <- <>word);'
exprs = compile_exprs([ searchWordLike, searchIdMake ])
assert len(exprs) == 2
inputText = r'argv[0];'
inputText = inputText.decode(sys.getfilesystemencoding())
seq = [ 'code' ]; seq.extend(split_to_strings_iter(inputText))
for expr in exprs:
newSeq = expr.parse(seq)
self.assertTrue(newSeq)
seq = newSeq
print "result seq=", "\n".join(pyrem_torq.treeseq.seq_pretty(seq))
def test7th(self):
searchWordLike = r'~(word <- ("_", *("_" | r"^[a-zA-Z]" | r"^\d") | r"^[a-zA-Z]", *("_" | r"^[a-zA-Z]" | r"^\d")));'
searchIdMake = r'~(id <- <>word);'
exprs = compile_exprs([searchWordLike, searchIdMake])
assert len(exprs) == 2
inputText = r'argv[0];'
inputText = inputText.decode(sys.getfilesystemencoding())
seq = ['code']
seq.extend(split_to_strings_iter(inputText))
for expr in exprs:
newSeq = expr.parse(seq)
self.assertTrue(newSeq)
seq = newSeq
print "result seq=", "\n".join(pyrem_torq.treeseq.seq_pretty(seq))
def test8th(self):
searchFloatingPointLitearal = r"""~(
l_float <- "0", (
ri"^x[a-f0-9]+p\d+$" | ri"^x[a-f0-9]+p$", ("-" | "+"), r"^\d"
| ri"^x[a-f0-9]+$", ".", *(ri"^[a-f0-9]+$" | r"^\d"), ?(ri"^[a-f0-9]*p\d+$" | ri"^[a-f0-9]*p$", ("-" | "+"), r"^\d")
), ?i"l"
);
"""
exprs = compile_exprs([ searchFloatingPointLitearal ])
assert len(exprs) == 1
pat = re.compile(r"\d+|[a-zA-Z_][a-zA-Z_0-9]*|[ \t]+|\r\n|.", re.DOTALL | re.IGNORECASE)
sampleFlotingPointLiterals = [ '0x012abc.def', '0xabc.012', '0xap10' ]
for inputText in sampleFlotingPointLiterals:
inputText = inputText.decode(sys.getfilesystemencoding())
seq = [ 'code' ]; seq.extend(split_to_strings_iter(inputText, pat))
seq = exprs[0].parse(seq)
self.assertEqual(seq[0], 'code')
self.assertEqual(seq[1][0], 'l_float')
self.assertEqual(u"".join(seq[1][1:]), inputText)
def test8th(self):
searchFloatingPointLitearal = r"""~(
l_float <- "0", (
ri"^x[a-f0-9]+p\d+$" | ri"^x[a-f0-9]+p$", ("-" | "+"), r"^\d"
| ri"^x[a-f0-9]+$", ".", *(ri"^[a-f0-9]+$" | r"^\d"), ?(ri"^[a-f0-9]*p\d+$" | ri"^[a-f0-9]*p$", ("-" | "+"), r"^\d")
), ?i"l"
);
"""
exprs = compile_exprs([searchFloatingPointLitearal])
assert len(exprs) == 1
pat = re.compile(r"\d+|[a-zA-Z_][a-zA-Z_0-9]*|[ \t]+|\r\n|.",
re.DOTALL | re.IGNORECASE)
sampleFlotingPointLiterals = ['0x012abc.def', '0xabc.012', '0xap10']
for inputText in sampleFlotingPointLiterals:
inputText = inputText.decode(sys.getfilesystemencoding())
seq = ['code']
seq.extend(split_to_strings_iter(inputText, pat))
seq = exprs[0].parse(seq)
self.assertEqual(seq[0], 'code')
self.assertEqual(seq[1][0], 'l_float')
self.assertEqual(u"".join(seq[1][1:]), inputText)
def test5th(self):
atoz = 'r"^[a-z]"'
exprStrs = [ r'~(req(%(atoz)s), ((op_logical_and <- "and") | (op_logical_or <- "or")), req^(%(atoz)s));' % { 'atoz': atoz } ]
exprs = compile_exprs(exprStrs)
inputText = r'if (x and y or z) printf("hello\n");'
inputText = inputText.decode(sys.getfilesystemencoding())
seq = [ 'code' ]; seq.extend(split_to_strings_iter(inputText))
for expr in exprs:
newSeq = expr.parse(seq)
self.assertTrue(newSeq, None)
seq = newSeq
foundAnd, foundOr = False, False
for item in seq:
if isinstance(item, list):
if item[0:1] == [ 'op_logical_and' ]: foundAnd = True
if item[0:1] == [ 'op_logical_or' ]: foundOr = True
self.assertTrue(foundAnd)
self.assertTrue(foundOr)
print "result seq=", "\n".join(pyrem_torq.treeseq.seq_pretty(seq))
def tkn(inputText, inputFilePath=None):
if inputText.startswith(bom): inputText = inputText[len(bom):] # remove bom
seq = [ 'code' ]; seq.extend(split_to_strings_iter(inputText, textSplitPattern))
seq.append([ 'eof' ]) # insert eof node at the end of sequence
return seq
def tkn(inputText, inputFilePath=None):
seq = ['code']
seq.extend(split_to_strings_iter(inputText, textSplitPattern))
seq.append(['eof']) # insert eof node at the end of sequence
return seq
def tkn(inputText, inputFilePath=None):
seq = [ 'code' ]; seq.extend(split_to_strings_iter(inputText, textSplitPattern))
seq.append([ 'eof' ]) # insert eof node at the end of sequence
return seq