def mark(token_list, context):
new_tokens = []
suspected_log_tokens = []
state = Searching()
log_statement = LogStatement()
for token in token_list:
search_result = state.check(token)
if search_result == SearchResult.NOT_FOUND:
new_tokens.append(token)
elif search_result == SearchResult.IN_PROGRESS:
suspected_log_tokens.append(token)
state = state.action(log_statement, token)
elif search_result == SearchResult.FAILED:
state = Searching()
new_tokens.extend(suspected_log_tokens) # in case 'log statement' was found,
# but later wasn't marked as log statement
suspected_log_tokens = [] # TODO come up with unit-tests that will fail without this line
new_tokens.append(token)
elif search_result == SearchResult.BUILT:
new_tokens.append(log_statement)
log_statement = LogStatement()
suspected_log_tokens = []
state = Searching()
else:
raise AssertionError()
return new_tokens
def test_log_no_mark_logs(self):
prep_config = PrepConfig({
PrepParam.EN_ONLY: 1,
PrepParam.COM_STR: 0,
PrepParam.SPLIT: 1,
PrepParam.TABS_NEWLINES: 0,
PrepParam.MARK_LOGS: 0,
PrepParam.CAPS: 1
})
ngramSplittingConfig = NgramSplitConfig()
tokens = [
LogStatement(
SplitContainer.from_single_token('LOGGER'),
SplitContainer.from_single_token('Info'), INFO,
[StringLiteral([SplitContainer.from_single_token("Hi")])])
]
actual = to_repr(prep_config, tokens, ngramSplittingConfig)
expected = [
pl['capitals'], 'logger', '.', pl['capital'], 'info', '(', '"',
pl['capital'], 'hi', '"', ')', ';'
]
self.assertEqual(expected, actual)
def test_2_logs(self):
input = [NewLine(),
SplitContainer.from_single_token('log'), '.', SplitContainer.from_single_token('t'),
'(', StringLiteral([SplitContainer.from_single_token("Hi")]), ')', ';',
NewLine(),
SplitContainer.from_single_token('Logger'), '.', SplitContainer.from_single_token('SEVERE'),
'(', StringLiteral([SplitContainer.from_single_token("Hi")]), ')', ';', ]
actual = logs.mark(input, None)
expected = [NewLine(),
LogStatement(SplitContainer.from_single_token('log'),
SplitContainer.from_single_token('t'), TRACE,
[StringLiteral([SplitContainer.from_single_token("Hi")])]),
NewLine(), LogStatement(SplitContainer.from_single_token('Logger'),
SplitContainer.from_single_token('SEVERE'), FATAL,
[StringLiteral([SplitContainer.from_single_token("Hi")])])]
self.assertEqual(expected, actual)
def test_simple_log(self):
input = [NewLine(),
SplitContainer.from_single_token('log'),
'.', SplitContainer.from_single_token('info'),
'(',
StringLiteral([SplitContainer.from_single_token("Hi")]),
')', ';',
Number([HexStart(), '3', '4', 'a', '3', '5', 'E', L()])]
actual = logs.mark(input, None)
expected = [NewLine(),
LogStatement(SplitContainer.from_single_token('log'),
SplitContainer.from_single_token('info'), INFO,
[StringLiteral([SplitContainer.from_single_token("Hi")])]),
Number([HexStart(), '3', '4', 'a', '3', '5', 'E', L()])]
self.assertEqual(expected, actual)
def test_tabs_and_newlines_before_semicolon(self):
input = [NewLine(),
SplitContainer.from_single_token('log'),
'.', SplitContainer.from_single_token('d'),
'(',
StringLiteral([SplitContainer.from_single_token("Hi")]),
')', NewLine(), NewLine(), Tab(), Tab(), ';',
Number([HexStart(), '3', '4', 'a', '3', '5', 'E', L()])]
actual = logs.mark(input, None)
expected = [NewLine(),
LogStatement(SplitContainer.from_single_token('log'),
SplitContainer.from_single_token('d'), DEBUG,
[StringLiteral([SplitContainer.from_single_token("Hi")])],
[NewLine(), NewLine(), Tab(), Tab()]),
Number([HexStart(), '3', '4', 'a', '3', '5', 'E', L()])]
self.assertEqual(expected, actual)
"Сегодня": (
[SplitContainer([(NonEng(Word.from_("Сегодня")))])],
[placeholders['capital'], placeholders['non_eng']]
),
'"сегодня"': (
[StringLiteral([SplitContainer([(NonEng(Word.from_("сегодня")))])])],
['"', placeholders['non_eng'], '"']
),
'logger.info("Установлена licht4bild пользователем" + user.getNick()) ;': (
[LogStatement(SplitContainer.from_single_token('logger'), SplitContainer.from_single_token('info'), INFO,
[StringLiteral([
SplitContainer([NonEng(Word.from_('Установлена'))]),
SplitContainer([
NonEng(Word.from_('licht')),
Word.from_('4'),
NonEng(Word.from_('bild'))
]),
SplitContainer([NonEng(Word.from_('пользователем'))])
]), '+', SplitContainer.from_single_token('user'), '.',
SplitContainer([
Word.from_('get'),
Word.from_('Nick')
]), '(', ')'])],
['`L', '`info', 'logger', '.', 'info', '(', '"', '`C', '`E', '`w', '`E', '4', '`E', 'w`', '`E', '"',
'+', 'user', '.', '`w', 'get', '`C', 'ni', 'ck', 'w`', '(', ')', ')', ';', 'L`']
)
}
bpe_merges_cache = {
"players": ["play", "er", "s"],
"0.345e+4": ["0.", "3", "4", "5", "e+", "4"],
"modified": ["mod", "if", "ied"],
def action(self, log_statement: LogStatement, token):
log_statement.method_name = token
log_statement.level = get_log_level(str(token))
return MethodFound()