def test_split_verbose3(self):
text = '''
// this code won't compile but the preprocessing still has to be done corrrectly
9a ** abc1
~-|=?==!=/* gj **/
'''
actual = spl_verbose([ParseableToken(text)], None)
expected = [
'\n',
OneLineCommentStart(),
ParseableToken("this"),
ParseableToken("code"),
ParseableToken("won"), "'",
ParseableToken("t"),
ParseableToken("compile"),
ParseableToken("but"),
ParseableToken("the"),
ParseableToken("preprocessing"),
ParseableToken("still"),
ParseableToken("has"),
ParseableToken("to"),
ParseableToken("be"),
ParseableToken("done"),
ParseableToken("corrrectly"), '\n',
ParseableToken("9a"), '**',
ParseableToken("abc1"), '\n', '~', '-', '|=', '?', '==', '!=',
MultilineCommentStart(),
ParseableToken("gj"), '*',
MultilineCommentEnd(), '\n'
]
self.assertEqual(expected, actual)
def test_split_verbose_log_statement(self):
text = '''
logger.info("The value is " + val);
'''
actual = spl_verbose([ParseableToken(text)], None)
expected = [
'\n',
ParseableToken("logger"),
'.',
ParseableToken("info"),
'(',
Quote(),
ParseableToken("The"),
ParseableToken("value"),
ParseableToken("is"),
Quote(),
'+',
ParseableToken('val'),
')',
';',
'\n',
]
self.assertEqual(expected, actual)
def split_log_text_to_keywords_and_identifiers(multitoken_list):
res = []
for multitoken in multitoken_list:
if multitoken not in [
placeholders['string_resource'], placeholders['var']
]:
res.extend(spl_verbose([multitoken]))
else:
res.extend([multitoken])
return res
def test_split_verbose2(self):
text = '''
float[] floats = {}; //floats were removed
BigAWESOMEString[] a2y = "abc".doSplit("\\"");
'''
actual = spl_verbose([ParseableToken(text)], None)
expected = [
'\n',
ParseableToken("float"),
'[',
']',
ParseableToken("floats"),
'=',
'{',
'}',
';',
OneLineCommentStart(),
ParseableToken("floats"),
ParseableToken("were"),
ParseableToken("removed"),
'\n',
ParseableToken("BigAWESOMEString"),
'[',
']',
ParseableToken("a2y"),
'=',
Quote(),
ParseableToken("abc"),
Quote(),
'.',
ParseableToken("doSplit"),
'(',
Quote(),
Backslash(),
Quote(),
Quote(),
')',
';',
'\n',
]
self.assertEqual(expected, actual)
def test_split_verbose1(self):
text = '''
long[] lovely_longs = {/* there should be some longs here*/};
int[] _my_favoRite_ints_ = {/* ints here*/};
'''
actual = spl_verbose([ParseableToken(text)], None)
expected = [
'\n',
ParseableToken("long"),
'[',
']',
ParseableToken("lovely_longs"),
'=',
'{',
MultilineCommentStart(),
ParseableToken("there"),
ParseableToken("should"),
ParseableToken("be"),
ParseableToken("some"),
ParseableToken("longs"),
ParseableToken("here"),
MultilineCommentEnd(),
"}",
';',
'\n',
ParseableToken("int"),
'[',
']',
ParseableToken("_my_favoRite_ints_"),
'=',
'{',
MultilineCommentStart(),
ParseableToken("ints"),
ParseableToken("here"),
MultilineCommentEnd(),
'}',
';',
'\n',
]
self.assertEqual(expected, actual)
def test_split_verbose4(self):
text = '''
a++a
b--b
c+=c
d-=d
e/=e
f*=f
g%=g
h$h
i<=i
j>=j
k@k
l^=l
m&=m
n#n
o>>o
p<<p
q&&q
r||r
+*!/><\t\n
{}[],.-:();&|\\'~%^
/*multi-line MyComment_
*//
_operations
'''
actual = spl_verbose([ParseableToken(text)], None)
expected = [
'\n',
ParseableToken('a'), '++',
ParseableToken('a'), '\n',
ParseableToken('b'), '--',
ParseableToken('b'), '\n',
ParseableToken('c'), '+=',
ParseableToken('c'), '\n',
ParseableToken('d'), '-=',
ParseableToken('d'), '\n',
ParseableToken('e'), '/=',
ParseableToken('e'), '\n',
ParseableToken('f'), '*=',
ParseableToken('f'), '\n',
ParseableToken('g'), '%=',
ParseableToken('g'), '\n',
ParseableToken('h'), '$',
ParseableToken('h'), '\n',
ParseableToken('i'), '<=',
ParseableToken('i'), '\n',
ParseableToken('j'), '>=',
ParseableToken('j'), '\n',
ParseableToken('k'), '@',
ParseableToken('k'), '\n',
ParseableToken('l'), '^=',
ParseableToken('l'), '\n',
ParseableToken('m'), '&=',
ParseableToken('m'), '\n',
ParseableToken('n'), '#',
ParseableToken('n'), '\n',
ParseableToken('o'), '>>',
ParseableToken('o'), '\n',
ParseableToken('p'), '<<',
ParseableToken('p'), '\n',
ParseableToken('q'), '&&',
ParseableToken('q'), '\n',
ParseableToken('r'), '||',
ParseableToken('r'), '\n', '+', '*', '!', '/', '>', '<',
Tab(), '\n', '\n', '{', '}', '[', ']', ',', '.', '-', ':', '(',
')', ';', '&', '|',
Backslash(), "'", '~', '%', '^', '\n',
MultilineCommentStart(),
ParseableToken("multi"), '-',
ParseableToken("line"),
ParseableToken("MyComment_"), '\n',
MultilineCommentEnd(), '/', '\n',
ParseableToken("_operations"), '\n'
]
self.assertEqual(expected, actual)