def main():
"""Module body."""
parser = OptionParser()
parser.add_option('-e', '--exec', action='store_true', dest='execute', default=False, help='Command is executed')
(options, args) = parser.parse_args()
if len(args) != 1 :
print "{0}: {1}".format(bold('Syntax'), r'./try_command.py command_identifier?param1=value1\¶m2=value2')
sys.exit(1)
command_id, command_params = CommandFactory.parse_url(args[0])
conf_loader = ConfLoader(dirname(realpath(__file__)) + '/conf')
factory = CommandFactory(conf_loader.load('commands'))
try:
command = factory.get_by_id(command_id)
command.insuflate(command_params)
except RxcCommandException as err:
print '{0}: {1}'.format(bold('Error'), err)
sys.exit(1)
if not options.execute:
print "{0}: {1}\n(use --exec to execute it.)".format(bold('Command'), command.to_exec)
sys.exit(0)
try:
result = command.run()
for key, value in result.items():
print '-' * 80
print "{0}\n{1}".format(bold(key), value)
print '-' * 80
sys.exit(0)
except RxcCommandException as err:
print '{0}: {1}'.format(bold('Error'), err)
sys.exit(1)
def test_path_stripping_in_test_failure_last_line(self):
f = io.StringIO()
r = TerminalReporter(watch_path='/path/to/watch',
build_path='/path/to/build',
terminal=Terminal(stream=f))
failures = [[
'core.ok',
[
'/path/to/watch/test/test_core.cc:12: Failure',
'Value of: 2',
'Expected: ok()',
'Which is: 42',
], [], FAILED
]]
r.report_failures(failures)
expected = [
'================================== FAILURES ==================================', # noqa
termstyle.bold(termstyle.red(
'__________________________________ core.ok ___________________________________' # noqa
)),
'/path/to/watch/test/test_core.cc:12: Failure',
'Value of: 2',
'Expected: ok()',
'Which is: 42',
termstyle.bold(termstyle.red(
'____________________________ test/test_core.cc:12 ____________________________', # noqa
)),
]
actual = f.getvalue().splitlines()
assert actual == expected
def desc(color):
desc = color(" %s %s" % (prefix, name))
if a:
desc += ' ' + color(termstyle.bold(' '.join(map(nice_repr,a))))
if kw:
desc += ' ' + ' '.join([color("%s=%s") % (k, termstyle.bold(repr(v))) for k,v in kw.items()])
return desc
def test_report_multiple_failed(self):
f = io.StringIO()
r = TerminalReporter(watch_path='/path',
build_path=None,
terminal=Terminal(stream=f))
results = {
'total_runtime': 2.09,
'total_passed': 0,
'total_failed': 2,
'failures': [
[
'fail1',
[
'/path/to/file:12: blah',
'results line 2',
'results line 3',
'results line 4',
], [], FAILED
],
[
'fail2',
[
'/path/to/file:102: blah',
'results line 2',
'results line 3',
'results line 4',
], [], FAILED
],
]
}
r.report_results(results)
expected = [
'================================== FAILURES ==================================', # noqa
termstyle.bold(termstyle.red(
'___________________________________ fail1 ____________________________________' # noqa
)),
'/path/to/file:12: blah',
'results line 2',
'results line 3',
'results line 4',
termstyle.bold(termstyle.red(
'_________________________________ to/file:12 _________________________________' # noqa
)),
termstyle.bold(termstyle.red(
'___________________________________ fail2 ____________________________________' # noqa
)),
'/path/to/file:102: blah',
'results line 2',
'results line 3',
'results line 4',
termstyle.bold(termstyle.red(
'________________________________ to/file:102 _________________________________' # noqa
)),
termstyle.bold(termstyle.red(
'===================== 2 failed, 0 passed in 2.09 seconds =====================' # noqa
)),
]
actual = f.getvalue().splitlines()
assert actual == expected
def test_wait_change(self):
f = io.StringIO()
r = TerminalReporter(
watch_path="watch_path",
build_path="build_path",
terminal=Terminal(stream=f),
timestamp=lambda: "timestamp",
)
r.wait_change()
assert (
f.getvalue()
== termstyle.bold(
"".ljust(28, "#") + " waiting for changes " + "".ljust(29, "#")
)
+ os.linesep
+ termstyle.bold("### Since: timestamp")
+ os.linesep
+ termstyle.bold("### Watching: watch_path")
+ os.linesep
+ termstyle.bold("### Build at: build_path")
+ os.linesep
+ termstyle.bold("### Using {}: {}".format(__progname__, __version__))
+ os.linesep
)
def test_writeln_decorator(self):
f = io.StringIO()
t = Terminal(stream=f)
t.writeln("hello", decorator=[bold])
assert f.getvalue() == bold("hello") + linesep
f = io.StringIO()
t = Terminal(stream=f)
t.writeln("hello", decorator=[bold, red])
assert f.getvalue() == red(bold("hello")) + linesep
def test_writeln_decorator(self):
f = io.StringIO()
t = Terminal(stream=f)
t.writeln('hello', decorator=[bold])
assert f.getvalue() == bold('hello') + linesep
f = io.StringIO()
t = Terminal(stream=f)
t.writeln('hello', decorator=[bold, red])
assert f.getvalue() == red(bold('hello')) + linesep
def desc(color):
desc = color(" %s %s" % (prefix, name))
if a:
desc += ' ' + color(
termstyle.bold(' '.join(map(nice_repr, a))))
if kw:
desc += ' ' + ' '.join([
color("%s=%s") % (k, termstyle.bold(repr(v)))
for k, v in kw.items()
])
return desc
def test_report_with_stdout_and_stderr_in_additional_output(self):
f = io.StringIO()
r = TerminalReporter(
watch_path="/path", build_path=None, terminal=Terminal(stream=f)
)
results = {
"total_runtime": 2.09,
"total_passed": 0,
"total_failed": 1,
"failures": [
[
"fail1",
[
"extra line 1",
"extra line 2",
"/path/to/file:12: blah",
"results line 1",
"results line 2",
"results line 3",
],
[],
FAILED,
],
],
}
r.report_results(results)
expected = [
"================================== FAILURES ==================================", # noqa
termstyle.bold(
termstyle.red(
"___________________________________ fail1 ____________________________________" # noqa
)
),
"/path/to/file:12: blah",
"results line 1",
"results line 2",
"results line 3",
"----------------------------- Additional output ------------------------------", # noqa
"extra line 1",
"extra line 2",
termstyle.bold(
termstyle.red(
"_________________________________ to/file:12 _________________________________" # noqa
)
),
termstyle.bold(
termstyle.red(
"===================== 1 failed, 0 passed in 2.09 seconds =====================" # noqa
)
),
]
actual = f.getvalue().splitlines()
assert actual == expected
def s(*a, **kw):
desc = "\t%s %s" % (prefix, ' '.join([
name,
termstyle.bold(' '.join(map(repr, a))), ' '.join(
["%s=%r" % (k, termstyle.bold(v)) for k, v in kw.items()])
]))
try:
ret = func(*a, **kw)
print >> sys.stderr, termstyle.green(desc)
return ret
except:
print >> sys.stderr, termstyle.red(desc)
import traceback
traceback.print_exc(file=sys.stderr)
raise
def test_detailed_verbosity(self):
results = [
'Running main() from gtest_main.cc',
'Note: Google Test filter = core.ok',
'[==========] Running 1 test from 1 test case.',
'[----------] Global test environment set-up.',
'[----------] 1 test from core',
'[ RUN ] core.ok',
'[ OK ] core.ok (0 ms)',
'[----------] 1 test from core (0 ms total)',
'',
'[----------] Global test environment tear-down',
'[==========] 1 test from 1 test case ran. (0 ms total)',
'[ PASSED ] 1 test.',
]
f = io.StringIO()
gtest = GTest('/test/test_core.cc', 'test_core',
term=Terminal(f, verbosity=1))
for line in results:
gtest(sys.stdout, line)
import termstyle
expected_results = results[:2]
for l in results[2:]:
if l.startswith('['):
expected_results.append('{}{}'.format(
termstyle.green(termstyle.bold(l[:13])),
l[13:]
))
else:
expected_results.append(l)
assert f.getvalue() == os.linesep.join(expected_results) + os.linesep
def begin_new_run(self, current_time):
print "\n" * 10
subprocess.call('clear')
msg = "# Running tests at %s " % (time.strftime("%H:%M:%S", current_time))
print >> sys.stderr, termstyle.inverted(termstyle.bold(msg))
print >> sys.stderr, ""
def process(self, event):
if isinstance(event, TestRun):
print "\n" * 10
subprocess.call('clear')
msg = "# Running tests at %s " % (time.strftime("%H:%M:%S"), )
print >> sys.stderr, termstyle.inverted(termstyle.bold(msg))
print >> sys.stderr, ""
def test_report_with_stdout_and_stderr_in_additional_output(self):
f = io.StringIO()
r = TerminalReporter(watch_path='/path',
build_path=None,
terminal=Terminal(stream=f))
results = {
'total_runtime': 2.09,
'total_passed': 0,
'total_failed': 1,
'failures': [
[
'fail1',
[
'extra line 1',
'extra line 2',
'/path/to/file:12: blah',
'results line 1',
'results line 2',
'results line 3',
], [], FAILED
],
]
}
r.report_results(results)
expected = [
'================================== FAILURES ==================================', # noqa
termstyle.bold(termstyle.red(
'___________________________________ fail1 ____________________________________' # noqa
)),
'/path/to/file:12: blah',
'results line 1',
'results line 2',
'results line 3',
'----------------------------- Additional output ------------------------------', # noqa
'extra line 1',
'extra line 2',
termstyle.bold(termstyle.red(
'_________________________________ to/file:12 _________________________________' # noqa
)),
termstyle.bold(termstyle.red(
'===================== 1 failed, 0 passed in 2.09 seconds =====================' # noqa
)),
]
actual = f.getvalue().splitlines()
assert actual == expected
def process(self, event):
if isinstance(event, TestRun):
print "\n" * 10
subprocess.call('clear')
msg = "# Running tests at %s " % (time.strftime("%H:%M:%S"),)
print >> sys.stderr, termstyle.inverted(termstyle.bold(msg))
print >> sys.stderr, ""
def test_report_build_path(self):
f = io.StringIO()
r = TerminalReporter(watch_path='watch_path',
build_path='build_path',
terminal=Terminal(stream=f))
r.report_build_path()
assert f.getvalue() == (termstyle.bold('### Building: build_path')
+ os.linesep)
def test_wait_change(self):
f = io.StringIO()
r = TerminalReporter(watch_path='watch_path',
build_path='build_path',
terminal=Terminal(stream=f),
timestamp=lambda: 'timestamp')
r.wait_change()
assert f.getvalue() == termstyle.bold(
''.ljust(28, '#') +
' waiting for changes ' +
''.ljust(29, '#')
) + os.linesep + termstyle.bold(
'### Since: timestamp'
) + os.linesep + termstyle.bold(
'### Watching: watch_path'
) + os.linesep + termstyle.bold(
'### Build at: build_path'
) + os.linesep
def test_session_start(self):
f = io.StringIO()
r = TerminalReporter(watch_path=None,
build_path=None,
terminal=Terminal(stream=f))
r.session_start('test')
assert f.getvalue() == (termstyle.bold(''.ljust(28, '=')
+ ' test session starts '
+ ''.ljust(29, '='))
+ os.linesep)
def test_path_stripping_in_test_failure_last_line(self):
f = io.StringIO()
r = TerminalReporter(
watch_path="/path/to/watch",
build_path="/path/to/build",
terminal=Terminal(stream=f),
)
failures = [
[
"core.ok",
[
"/path/to/watch/test/test_core.cc:12: Failure",
"Value of: 2",
"Expected: ok()",
"Which is: 42",
],
[],
FAILED,
]
]
r.report_failures(failures)
expected = [
"================================== FAILURES ==================================", # noqa
termstyle.bold(
termstyle.red(
"__________________________________ core.ok ___________________________________" # noqa
)
),
"/path/to/watch/test/test_core.cc:12: Failure",
"Value of: 2",
"Expected: ok()",
"Which is: 42",
termstyle.bold(
termstyle.red(
"____________________________ test/test_core.cc:12 ____________________________", # noqa
)
),
]
actual = f.getvalue().splitlines()
assert actual == expected
def _fmt_message_sqlalchemy_line(self, line, match, color):
logger = match.group(1)
level = match.group(2)
verb = match.group(3)
rest = match.group(4) if len(match.groups()) > 3 else ''
if verb in ['SELECT']:
head = color("%s: %s:" % (logger, level))
self._in_multiline_sql = verb
self._in_multiline_sql_color = termstyle.cyan
return "%s %s %s" % (head, termstyle.bold(termstyle.cyan(verb)), termstyle.cyan(rest))
elif verb in ['BEGIN', 'SAVEPOINT', 'RELEASE', 'ROLLBACK', 'COMMIT']:
head = color("%s: %s:" % (logger, level))
self._in_multiline_sql = verb
self._in_multiline_sql_color = termstyle.green
return "%s %s %s" % (head, termstyle.bold(termstyle.green(verb)), termstyle.green(rest))
elif verb in ['INSERT', 'UPDATE', 'DELETE', 'TRUNCATE']:
head = color("%s: %s:" % (logger, level))
self._in_multiline_sql = verb
self._in_multiline_sql_color = termstyle.red
return "%s %s %s" % (head, termstyle.bold(termstyle.red(verb)), termstyle.red(rest))
return color(line)
def __init__(self, termcolor=None):
"""
termcolor - If None, attempt to autodetect whether we are in a terminal
and turn on terminal colors if we think we are. If True, force
terminal colors on. If False, force terminal colors off.
"""
if termcolor == None:
termstyle.auto()
self.termcolor = bool(termstyle.bold(""))
else:
self.termcolor = termcolor
self._restoreColor()
def __init__(self, termcolor=None):
"""
termcolor - If None, attempt to autodetect whether we are in a terminal
and turn on terminal colors if we think we are. If True, force
terminal colors on. If False, force terminal colors off.
"""
if termcolor is None:
termstyle.auto()
self.termcolor = bool(termstyle.bold(""))
else:
self.termcolor = termcolor
self._restoreColor()
def test_report_build_path(self):
f = io.StringIO()
r = TerminalReporter(
watch_path="watch_path",
build_path="build_path",
terminal=Terminal(stream=f),
)
r.report_build_path()
assert f.getvalue() == (
termstyle.bold("### Building: build_path") + os.linesep
)
def test_session_start(self):
f = io.StringIO()
r = TerminalReporter(
watch_path=None, build_path=None, terminal=Terminal(stream=f)
)
r.session_start("test")
assert f.getvalue() == (
termstyle.bold(
"".ljust(28, "=") + " test session starts " + "".ljust(29, "=")
)
+ os.linesep
)
def _relative_path(self, path): """ If path is a child of the current working directory, the relative path is returned surrounded by bold xterm escape sequences. If path is not a child of the working directory, path is returned """ try: here = os.path.abspath(os.path.realpath(os.getcwd())) fullpath = os.path.abspath(os.path.realpath(path)) except OSError: return path if fullpath.startswith(here): return termstyle.bold(fullpath[len(here)+1:]) return path
def _fmt_message_sqlalchemy_multi_line(self, line, color):
param = re.match('(sqlalchemy.engine.base.Engine): (\w+): ({.*})', line)
if param:
if param.group(3) == '{}':
return None
else:
return color("%s: %s: " % (param.group(1), param.group(2))) + termstyle.bold(self._in_multiline_sql_color(param.group(3)))
elif not re.match('[\w\.]+: [A-Z]*:', line):
return self._in_multiline_sql_color(line)
else:
self._in_multiline_sql = False
self._in_multiline_sql_color = termstyle.default
return color(line)
def _relative_path(self, path):
"""
If path is a child of the current working directory, the relative
path is returned surrounded by bold xterm escape sequences.
If path is not a child of the working directory, path is returned
"""
try:
here = os.path.abspath(os.path.realpath(os.getcwd()))
fullpath = os.path.abspath(os.path.realpath(path))
except OSError:
return path
if fullpath.startswith(here):
return termstyle.bold(fullpath[len(here) + 1:])
return path
def _format_exception_message(self, exception_type, exception_instance, message_color):
"""Returns a colorized formatted exception message."""
orig_message_lines = to_unicode(exception_instance).splitlines()
if len(orig_message_lines) == 0:
return ''
exception_message = orig_message_lines[0]
message_lines = [message_color(' ', termstyle.bold(message_color(exception_type.__name__)), ": ") + message_color(exception_message)]
for line in orig_message_lines[1:]:
match = re.match('^---.* begin captured stdout.*----$', line)
if match:
message_color = termstyle.magenta
message_lines.append('')
line = ' ' + line
message_lines.append(message_color(line))
return '\n'.join(message_lines)
def test_report_all_passed(self):
f = io.StringIO()
r = TerminalReporter(watch_path=None,
build_path=None,
terminal=Terminal(stream=f))
results = {
'total_runtime': 2.09,
'total_passed': 1,
'total_failed': 0,
'failures': []
}
r.report_results(results)
assert f.getvalue() == (
termstyle.bold(termstyle.green(''.ljust(26, '=')
+ ' 1 passed in 2.09 seconds '
+ ''.ljust(26, '=')))
+ os.linesep)
def _report_test(self, report_num, type_, test, err):
"""report the results of a single (failing or errored) test"""
self._line(termstyle.black)
self._out("%s) " % (report_num))
if type_ == failure:
color = termstyle.red
self._outln(color('FAIL: %s' % (self._format_test_name(test), )))
else:
color = termstyle.yellow
self._outln(color('ERROR: %s' % (self._format_test_name(test), )))
exc_type, exc_instance, exc_trace = err
self._outln()
self._outln(self._fmt_traceback(exc_trace))
self._out(color(' ', termstyle.bold(color(exc_type.__name__)), ": "))
self._outln(self._fmt_message(exc_instance, color))
self._outln()
def _report_test(self, report_num, type_, test, err):
"""report the results of a single (failing or errored) test"""
self._line(termstyle.black)
self._out("%s) " % (report_num))
if type_ == failure:
color = termstyle.red
self._outln(color('FAIL: %s' % (self._format_test_name(test),)))
else:
color = termstyle.yellow
self._outln(color('ERROR: %s' % (self._format_test_name(test),)))
exc_type, exc_instance, exc_trace = err
self._outln()
self._outln(self._fmt_traceback(exc_trace))
self._out(color(' ', termstyle.bold(color(exc_type.__name__)), ": "))
self._outln(self._fmt_message(exc_instance, color))
self._outln()
def __init__(self, termcolor=None, html=False):
"""
termcolor - If None, attempt to autodetect whether we are in a terminal
and turn on terminal colors if we think we are. If True, force
terminal colors on. If False, force terminal colors off. This
value is ignored if html is True.
html - If true, enables HTML output and causes termcolor to be ignored.
"""
self.html = html
if html:
termcolor = False
if termcolor == None:
termstyle.auto()
self.termcolor = bool(termstyle.bold(""))
else:
self.termcolor = termcolor
self._restoreColor()
def test_report_all_passed(self):
f = io.StringIO()
r = TerminalReporter(
watch_path=None, build_path=None, terminal=Terminal(stream=f)
)
results = {
"total_runtime": 2.09,
"total_passed": 1,
"total_failed": 0,
"failures": [],
}
r.report_results(results)
assert f.getvalue() == (
termstyle.bold(
termstyle.green(
"".ljust(26, "=") + " 1 passed in 2.09 seconds " + "".ljust(26, "=")
)
)
+ os.linesep
)
def _file_line(self, tb):
"""formats the file / lineno / function line of a traceback element"""
prefix = "file://"
prefix = ""
f = tb.tb_frame
if '__unittest' in f.f_globals:
# this is the magical flag that prevents unittest internal
# code from junking up the stacktrace
return None
filename = f.f_code.co_filename
lineno = tb.tb_lineno
linecache.checkcache(filename)
function_name = f.f_code.co_name
line_contents = linecache.getline(filename, lineno,
f.f_globals).strip()
return " %s line %s in %s\n %s" % (
termstyle.blue(prefix, self._relative_path(filename)),
termstyle.bold(termstyle.cyan(lineno)),
termstyle.cyan(function_name), line_contents)
def _format_traceback_line(self, tb):
"""
Formats the file / lineno / function line of a traceback element.
Returns None is the line is not relevent to the user i.e. inside the test runner.
"""
if self._is_relevant_tb_level(tb):
return None
f = tb.tb_frame
filename = f.f_code.co_filename
lineno = tb.tb_lineno
linecache.checkcache(filename)
function_name = f.f_code.co_name
line_contents = linecache.getline(filename, lineno, f.f_globals).strip()
return " %s line %s in %s\n %s" % (
termstyle.blue(self._relative_path(filename) if self.use_relative_path else filename),
termstyle.bold(termstyle.cyan(lineno)),
termstyle.cyan(function_name),
line_contents
)
def _file_line(self, tb): """formats the file / lineno / function line of a traceback element""" prefix = "file://" prefix = "" f = tb.tb_frame if '__unittest' in f.f_globals: # this is the magical flag that prevents unittest internal # code from junking up the stacktrace return None filename = f.f_code.co_filename lineno = tb.tb_lineno linecache.checkcache(filename) function_name = f.f_code.co_name line_contents = linecache.getline(filename, lineno, f.f_globals).strip() return " %s line %s in %s\n %s" % ( termstyle.blue(prefix, self._relative_path(filename)), termstyle.bold(termstyle.cyan(lineno)), termstyle.cyan(function_name), line_contents)
def bold(self, text):
self._restoreColor()
return termstyle.bold(text)
def test_report_multiple_failed(self):
f = io.StringIO()
r = TerminalReporter(
watch_path="/path", build_path=None, terminal=Terminal(stream=f)
)
results = {
"total_runtime": 2.09,
"total_passed": 0,
"total_failed": 2,
"failures": [
[
"fail1",
[
"/path/to/file:12: blah",
"results line 2",
"results line 3",
"results line 4",
],
[],
FAILED,
],
[
"fail2",
[
"/path/to/file:102: blah",
"results line 2",
"results line 3",
"results line 4",
],
[],
FAILED,
],
],
}
r.report_results(results)
expected = [
"================================== FAILURES ==================================", # noqa
termstyle.bold(
termstyle.red(
"___________________________________ fail1 ____________________________________" # noqa
)
),
"/path/to/file:12: blah",
"results line 2",
"results line 3",
"results line 4",
termstyle.bold(
termstyle.red(
"_________________________________ to/file:12 _________________________________" # noqa
)
),
termstyle.bold(
termstyle.red(
"___________________________________ fail2 ____________________________________" # noqa
)
),
"/path/to/file:102: blah",
"results line 2",
"results line 3",
"results line 4",
termstyle.bold(
termstyle.red(
"________________________________ to/file:102 _________________________________" # noqa
)
),
termstyle.bold(
termstyle.red(
"===================== 2 failed, 0 passed in 2.09 seconds =====================" # noqa
)
),
]
actual = f.getvalue().splitlines()
assert actual == expected
def main():
parser = argparse.ArgumentParser(
prog=APPNAME,
formatter_class=argparse.RawDescriptionHelpFormatter,
usage=USAGE,
description=DESCRIPTION,
epilog=EPILOG,
)
parser.add_argument('botid',
metavar='botid',
nargs='?',
default=None,
help='botid to inspect dumps of')
args = parser.parse_args()
if args.botid is None:
filenames = glob.glob(os.path.join(DEFAULT_LOGGING_PATH, '*.dump'))
if not len(filenames):
print(green('Nothing to recover from, no dump files found!'))
exit(0)
filenames = [(fname, fname[len(DEFAULT_LOGGING_PATH):-5])
for fname in sorted(filenames)]
length = max([len(value[1]) for value in filenames])
print(
bold("{c:>3}: {s:{l}} {i}".format(c='id',
s='name (bot id)',
i='content',
l=length)))
for count, (fname, shortname) in enumerate(filenames):
info = dump_info(fname)
print("{c:3}: {s:{l}} {i}".format(c=count,
s=shortname,
i=info,
l=length))
botid = input(inverted('Which dump file to process (id or name)? '))
botid = botid.strip()
if botid == 'q' or not botid:
exit(0)
try:
fname, botid = filenames[int(botid)]
except ValueError:
fname = os.path.join(DEFAULT_LOGGING_PATH, botid) + '.dump'
else:
botid = args.botid
fname = os.path.join(DEFAULT_LOGGING_PATH, botid) + '.dump'
if not os.path.isfile(fname):
print(bold('Given file does not exist: {}'.format(fname)))
exit(1)
while True:
info = dump_info(fname)
print('Processing {}: {}'.format(bold(botid), info))
try:
with io.open(fname, 'rt') as handle:
content = json.load(handle)
meta = load_meta(content)
except ValueError:
available_opts = [item[0] for item in ACTIONS.values() if item[2]]
print(
bold('Could not load file:') + '\n{}\nRestricted actions.'
''.format(traceback.format_exc()))
else:
available_opts = [item[0] for item in ACTIONS.values()]
for count, line in enumerate(meta):
print('{:3}: {} {}'.format(count, *line))
answer = input(inverted(', '.join(available_opts) + '? ')).split()
if not answer:
continue
if any([answer[0] == char for char in AVAILABLE_IDS]):
ids = [int(item) for item in answer[1].split(',')]
queue_name = None
if answer[0] == 'a':
# recover all -> recover all by ids
answer[0] = 'r'
ids = range(len(meta))
if len(answer) > 1:
queue_name = answer[1]
if answer[0] == 'q':
break
elif answer[0] == 'e':
# Delete entries
for entry in ids:
del content[meta[entry][0]]
save_file(fname, content)
elif answer[0] == 'r':
# recover entries
for key, entry in [
item for (count, item) in enumerate(content.items())
if count in ids
]:
if type(entry['message']) is dict:
if '__type' in entry['message']:
msg = json.dumps(entry['message'])
# backwards compat: dumps had no type info
elif '-parser' in entry['bot_id']:
msg = message.Report(entry['message']).serialize()
else:
msg = message.Event(entry['message']).serialize()
elif issubclass(type(entry['message']),
(six.binary_type, six.text_type)):
msg = entry['message']
elif entry['message'] is None:
print(bold('No message here, deleting directly.'))
del content[key]
save_file(fname, content)
continue
else:
print(
bold('Unhandable type of message: {!r}'
''.format(type(entry['message']))))
continue
print(entry['source_queue'])
default = utils.load_configuration(DEFAULTS_CONF_FILE)
runtime = utils.load_configuration(RUNTIME_CONF_FILE)
params = utils.load_parameters(default, runtime)
pipe = pipeline.PipelineFactory.create(params)
if queue_name is None:
if len(answer) == 2:
queue_name = answer[2]
else:
queue_name = entry['source_queue']
try:
pipe.set_queues(queue_name, 'destination')
pipe.connect()
pipe.send(msg)
except exceptions.PipelineError:
print(
red('Could not reinject into queue {}: {}'
''.format(queue_name, traceback.format_exc())))
else:
del content[key]
save_file(fname, content)
elif answer[0] == 'd':
# delete dumpfile
os.remove(fname)
print('Deleted file {}'.format(fname))
break
elif answer[0] == 's':
# Show entries by id
for count, (key, value) in enumerate(content.items()):
if count not in ids:
continue
print('=' * 100, '\nShowing id {} {}\n'.format(count, key),
'-' * 50)
if isinstance(value['message'],
(six.binary_type, six.text_type)):
value['message'] = json.loads(value['message'])
if ('raw' in value['message']
and len(value['message']['raw']) > 1000):
value['message']['raw'] = value['message'][
'raw'][:1000] + '...[truncated]'
value['traceback'] = value['traceback'].splitlines()
pprint.pprint(value)
def main():
parser = argparse.ArgumentParser(
prog=APPNAME,
formatter_class=argparse.RawDescriptionHelpFormatter,
usage=USAGE,
description=DESCRIPTION,
epilog=EPILOG,
)
parser.add_argument('botid', metavar='botid', nargs='?',
default=None, help='botid to inspect dumps of')
args = parser.parse_args()
# Try to get log_level from defaults_configuration, else use default
try:
log_level = utils.load_configuration(DEFAULTS_CONF_FILE)['logging_level']
except Exception:
log_level = DEFAULT_LOGGING_LEVEL
try:
logger = utils.log('intelmqdump', log_level=log_level)
except (FileNotFoundError, PermissionError) as exc:
logger = utils.log('intelmqdump', log_level=log_level, log_path=False)
logger.error('Not logging to file: %s', exc)
ctl = intelmqctl.IntelMQController()
readline.parse_and_bind("tab: complete")
readline.set_completer_delims('')
pipeline_config = utils.load_configuration(PIPELINE_CONF_FILE)
pipeline_pipes = {}
for bot, pipes in pipeline_config.items():
pipeline_pipes[pipes.get('source-queue', '')] = bot
if args.botid is None:
filenames = glob.glob(os.path.join(DEFAULT_LOGGING_PATH, '*.dump'))
if not len(filenames):
print(green('Nothing to recover from, no dump files found!'))
sys.exit(0)
filenames = [(fname, fname[len(DEFAULT_LOGGING_PATH):-5])
for fname in sorted(filenames)]
length = max([len(value[1]) for value in filenames])
print(bold("{c:>3}: {s:{length}} {i}".format(c='id', s='name (bot id)',
i='content',
length=length)))
for count, (fname, shortname) in enumerate(filenames):
info = dump_info(fname)
print("{c:3}: {s:{length}} {i}".format(c=count, s=shortname, i=info,
length=length))
try:
bot_completer = Completer(possible_values=[f[1] for f in filenames])
readline.set_completer(bot_completer.complete)
botid = input(inverted('Which dump file to process (id or name)?') +
' ')
except EOFError:
sys.exit(0)
else:
botid = botid.strip()
if botid == 'q' or not botid:
exit(0)
try:
fname, botid = filenames[int(botid)]
except ValueError:
fname = os.path.join(DEFAULT_LOGGING_PATH, botid) + '.dump'
else:
botid = args.botid
fname = os.path.join(DEFAULT_LOGGING_PATH, botid) + '.dump'
if not os.path.isfile(fname):
print(bold('Given file does not exist: {}'.format(fname)))
exit(1)
answer = None
delete_file = False
while True:
with open(fname, 'r+') as handle:
try:
fcntl.flock(handle, fcntl.LOCK_EX | fcntl.LOCK_NB)
except BlockingIOError:
print(red('Dump file is currently locked. Stopping.'))
break
info = dump_info(fname, file_descriptor=handle)
handle.seek(0)
available_answers = ACTIONS.keys()
print('Processing {}: {}'.format(bold(botid), info))
if info.startswith(str(red)):
available_opts = [item[0] for item in ACTIONS.values() if item[2]]
available_answers = [k for k, v in ACTIONS.items() if v[2]]
print('Restricted actions.')
else:
# don't display list after 'show' and 'recover' command
if not (answer and isinstance(answer, list) and answer[0] in ['s', 'r']):
content = json.load(handle)
handle.seek(0)
content = OrderedDict(sorted(content.items(), key=lambda t: t[0])) # sort by key here, #1280
meta = load_meta(content)
available_opts = [item[0] for item in ACTIONS.values()]
for count, line in enumerate(meta):
print('{:3}: {} {}'.format(count, *line))
# Determine bot status
try:
bot_status = ctl.bot_status(botid)
if bot_status[1] == 'running':
print(red('This bot is currently running, the dump file is now locked and '
'the bot can\'t write it.'))
except KeyError:
bot_status = 'error'
print(red('Attention: This bot is not defined!'))
available_opts = [item[0] for item in ACTIONS.values() if item[2]]
available_answers = [k for k, v in ACTIONS.items() if v[2]]
print('Restricted actions.')
try:
possible_answers = list(available_answers)
for id_action in ['r', 'a']:
if id_action in possible_answers:
possible_answers[possible_answers.index(id_action)] = id_action + ' '
action_completer = Completer(possible_answers, queues=pipeline_pipes.keys())
readline.set_completer(action_completer.complete)
answer = input(inverted(', '.join(available_opts) + '?') + ' ').split()
except EOFError:
break
else:
if not answer:
continue
if len(answer) == 0 or answer[0] not in available_answers:
print('Action not allowed.')
continue
if any([answer[0] == char for char in AVAILABLE_IDS]) and len(answer) > 1:
ids = [int(item) for item in answer[1].split(',')]
else:
ids = []
queue_name = None
if answer[0] == 'a':
# recover all -> recover all by ids
answer[0] = 'r'
ids = range(len(meta))
if len(answer) > 1:
queue_name = answer[1]
if answer[0] == 'q':
break
elif answer[0] == 'e':
# Delete entries
for entry in ids:
del content[meta[entry][0]]
save_file(handle, content)
elif answer[0] == 'r':
# recover entries
default = utils.load_configuration(DEFAULTS_CONF_FILE)
runtime = utils.load_configuration(RUNTIME_CONF_FILE)
params = utils.load_parameters(default, runtime)
pipe = pipeline.PipelineFactory.create(params, logger)
try:
for i, (key, entry) in enumerate([item for (count, item)
in enumerate(content.items()) if count in ids]):
if entry['message']:
msg = copy.copy(entry['message']) # otherwise the message field gets converted
if isinstance(msg, dict):
msg = json.dumps(msg)
else:
print('No message here, deleting entry.')
del content[key]
continue
if queue_name is None:
if len(answer) == 3:
queue_name = answer[2]
else:
queue_name = entry['source_queue']
if queue_name in pipeline_pipes:
if runtime[pipeline_pipes[queue_name]]['group'] == 'Parser' and json.loads(msg)['__type'] == 'Event':
print('Event converted to Report automatically.')
msg = message.Report(message.MessageFactory.unserialize(msg)).serialize()
try:
pipe.set_queues(queue_name, 'destination')
pipe.connect()
pipe.send(msg)
except exceptions.PipelineError:
print(red('Could not reinject into queue {}: {}'
''.format(queue_name, traceback.format_exc())))
else:
del content[key]
print(green('Recovered dump {}.'.format(i)))
finally:
save_file(handle, content)
if not content:
delete_file = True
print('Deleting empty file {}'.format(fname))
break
elif answer[0] == 'd':
# delete dumpfile
delete_file = True
print('Deleting empty file {}'.format(fname))
break
elif answer[0] == 's':
# Show entries by id
for count, (key, orig_value) in enumerate(content.items()):
value = copy.copy(orig_value) # otherwise the raw field gets truncated
if count not in ids:
continue
print('=' * 100, '\nShowing id {} {}\n'.format(count, key),
'-' * 50)
if isinstance(value['message'], (bytes, str)):
value['message'] = json.loads(value['message'])
if ('raw' in value['message'] and
len(value['message']['raw']) > 1000):
value['message']['raw'] = value['message'][
'raw'][:1000] + '...[truncated]'
if type(value['traceback']) is not list:
value['traceback'] = value['traceback'].splitlines()
pprint.pprint(value)
if delete_file:
os.remove(fname)
def main():
parser = argparse.ArgumentParser(
prog=APPNAME,
formatter_class=argparse.RawDescriptionHelpFormatter,
usage=USAGE,
description=DESCRIPTION,
epilog=EPILOG,
)
parser.add_argument('botid', metavar='botid', nargs='?',
default=None, help='botid to inspect dumps of')
args = parser.parse_args()
ctl = intelmqctl.IntelMQController()
if args.botid is None:
filenames = glob.glob(os.path.join(DEFAULT_LOGGING_PATH, '*.dump'))
if not len(filenames):
print(green('Nothing to recover from, no dump files found!'))
exit(0)
filenames = [(fname, fname[len(DEFAULT_LOGGING_PATH):-5])
for fname in sorted(filenames)]
length = max([len(value[1]) for value in filenames])
print(bold("{c:>3}: {s:{l}} {i}".format(c='id', s='name (bot id)',
i='content', l=length)))
for count, (fname, shortname) in enumerate(filenames):
info = dump_info(fname)
print("{c:3}: {s:{l}} {i}".format(c=count, s=shortname, i=info,
l=length))
try:
botid = input(inverted('Which dump file to process (id or name)?') +
' ')
except EOFError:
exit(0)
else:
botid = botid.strip()
if botid == 'q' or not botid:
exit(0)
try:
fname, botid = filenames[int(botid)]
except ValueError:
fname = os.path.join(DEFAULT_LOGGING_PATH, botid) + '.dump'
else:
botid = args.botid
fname = os.path.join(DEFAULT_LOGGING_PATH, botid) + '.dump'
if not os.path.isfile(fname):
print(bold('Given file does not exist: {}'.format(fname)))
exit(1)
answer = None
while True:
info = dump_info(fname)
available_answers = ACTIONS.keys()
print('Processing {}: {}'.format(bold(botid), info))
if info.startswith(str(red)):
available_opts = [item[0] for item in ACTIONS.values() if item[2]]
available_answers = [k for k, v in ACTIONS.items() if v[2]]
print('Restricted actions.')
else:
# don't display list after 'show' and 'recover' command
if not (answer and isinstance(answer, list) and answer[0] in ['s', 'r']):
with open(fname, 'rt') as handle:
content = json.load(handle)
meta = load_meta(content)
available_opts = [item[0] for item in ACTIONS.values()]
for count, line in enumerate(meta):
print('{:3}: {} {}'.format(count, *line))
# Determine bot status
bot_status = ctl.bot_status(botid)
if bot_status == 'running':
print(red('Attention: This bot is currently running!'))
elif bot_status == 'error':
print(red('Attention: This bot is not defined!'))
try:
answer = input(inverted(', '.join(available_opts) + '?') + ' ').split()
except EOFError:
break
else:
if not answer:
continue
if len(answer) == 0 or answer[0] not in available_answers:
print('Action not allowed.')
continue
if any([answer[0] == char for char in AVAILABLE_IDS]) and len(answer) > 1:
ids = [int(item) for item in answer[1].split(',')]
else:
ids = []
queue_name = None
if answer[0] == 'a':
# recover all -> recover all by ids
answer[0] = 'r'
ids = range(len(meta))
if len(answer) > 1:
queue_name = answer[1]
if answer[0] == 'q':
break
elif answer[0] == 'e':
# Delete entries
for entry in ids:
del content[meta[entry][0]]
save_file(fname, content)
elif answer[0] == 'r':
if bot_status == 'running':
# See https://github.com/certtools/intelmq/issues/574
print(red('Recovery for running bots not possible.'))
continue
# recover entries
default = utils.load_configuration(DEFAULTS_CONF_FILE)
runtime = utils.load_configuration(RUNTIME_CONF_FILE)
params = utils.load_parameters(default, runtime)
pipe = pipeline.PipelineFactory.create(params)
try:
for i, (key, entry) in enumerate([item for (count, item)
in enumerate(content.items()) if count in ids]):
if entry['message']:
msg = entry['message']
else:
print('No message here, deleting entry.')
del content[key]
continue
if queue_name is None:
if len(answer) == 3:
queue_name = answer[2]
else:
queue_name = entry['source_queue']
try:
pipe.set_queues(queue_name, 'destination')
pipe.connect()
pipe.send(msg)
except exceptions.PipelineError:
print(red('Could not reinject into queue {}: {}'
''.format(queue_name, traceback.format_exc())))
else:
del content[key]
print(green('Recovered dump {}.'.format(i)))
finally:
save_file(fname, content)
if not content:
os.remove(fname)
print('Deleted empty file {}'.format(fname))
break
elif answer[0] == 'd':
# delete dumpfile
os.remove(fname)
print('Deleted file {}'.format(fname))
break
elif answer[0] == 's':
# Show entries by id
for count, (key, value) in enumerate(content.items()):
if count not in ids:
continue
print('=' * 100, '\nShowing id {} {}\n'.format(count, key),
'-' * 50)
if isinstance(value['message'], (bytes, str)):
value['message'] = json.loads(value['message'])
if ('raw' in value['message'] and
len(value['message']['raw']) > 1000):
value['message']['raw'] = value['message'][
'raw'][:1000] + '...[truncated]'
if type(value['traceback']) is not list:
value['traceback'] = value['traceback'].splitlines()
pprint.pprint(value)
def main():
parser = argparse.ArgumentParser(
prog=APPNAME,
formatter_class=argparse.RawDescriptionHelpFormatter,
usage=USAGE,
description=DESCRIPTION,
epilog=EPILOG,
)
parser.add_argument('botid',
metavar='botid',
nargs='?',
default=None,
help='botid to inspect dumps of')
args = parser.parse_args()
# Try to get log_level from defaults_configuration, else use default
try:
log_level = utils.load_configuration(
DEFAULTS_CONF_FILE)['logging_level']
except Exception:
log_level = DEFAULT_LOGGING_LEVEL
try:
logger = utils.log('intelmqdump', log_level=log_level)
except (FileNotFoundError, PermissionError) as exc:
logger = utils.log('intelmqdump', log_level=log_level, log_path=False)
logger.error('Not logging to file: %s', exc)
ctl = intelmqctl.IntelMQController()
readline.parse_and_bind("tab: complete")
readline.set_completer_delims('')
pipeline_config = utils.load_configuration(PIPELINE_CONF_FILE)
pipeline_pipes = {}
for bot, pipes in pipeline_config.items():
pipeline_pipes[pipes.get('source-queue', '')] = bot
if args.botid is None:
filenames = glob.glob(os.path.join(DEFAULT_LOGGING_PATH, '*.dump'))
if not len(filenames):
print(green('Nothing to recover from, no dump files found!'))
sys.exit(0)
filenames = [(fname, fname[len(DEFAULT_LOGGING_PATH):-5])
for fname in sorted(filenames)]
length = max([len(value[1]) for value in filenames])
print(
bold("{c:>3}: {s:{length}} {i}".format(c='id',
s='name (bot id)',
i='content',
length=length)))
for count, (fname, shortname) in enumerate(filenames):
info = dump_info(fname)
print("{c:3}: {s:{length}} {i}".format(c=count,
s=shortname,
i=info,
length=length))
try:
bot_completer = Completer(
possible_values=[f[1] for f in filenames])
readline.set_completer(bot_completer.complete)
botid = input(
inverted('Which dump file to process (id or name)?') + ' ')
except EOFError:
sys.exit(0)
else:
botid = botid.strip()
if botid == 'q' or not botid:
exit(0)
try:
fname, botid = filenames[int(botid)]
except ValueError:
fname = os.path.join(DEFAULT_LOGGING_PATH, botid) + '.dump'
else:
botid = args.botid
fname = os.path.join(DEFAULT_LOGGING_PATH, botid) + '.dump'
if not os.path.isfile(fname):
print(bold('Given file does not exist: {}'.format(fname)))
exit(1)
answer = None
delete_file = False
while True:
with open(fname, 'r+') as handle:
try:
fcntl.flock(handle, fcntl.LOCK_EX | fcntl.LOCK_NB)
except BlockingIOError:
print(red('Dump file is currently locked. Stopping.'))
break
info = dump_info(fname, file_descriptor=handle)
handle.seek(0)
available_answers = ACTIONS.keys()
print('Processing {}: {}'.format(bold(botid), info))
if info.startswith(str(red)):
available_opts = [
item[0] for item in ACTIONS.values() if item[2]
]
available_answers = [k for k, v in ACTIONS.items() if v[2]]
print('Restricted actions.')
else:
# don't display list after 'show', 'recover' & edit commands
if not (answer and isinstance(answer, list)
and answer[0] in ['s', 'r', 'v']):
content = json.load(handle)
handle.seek(0)
content = OrderedDict(
sorted(content.items(),
key=lambda t: t[0])) # sort by key here, #1280
meta = load_meta(content)
available_opts = [item[0] for item in ACTIONS.values()]
for count, line in enumerate(meta):
print('{:3}: {} {}'.format(count, *line))
# Determine bot status
try:
bot_status = ctl.bot_status(botid)
if bot_status[1] == 'running':
print(
red('This bot is currently running, the dump file is now locked and '
'the bot can\'t write it.'))
except KeyError:
bot_status = 'error'
print(red('Attention: This bot is not defined!'))
available_opts = [
item[0] for item in ACTIONS.values() if item[2]
]
available_answers = [k for k, v in ACTIONS.items() if v[2]]
print('Restricted actions.')
try:
possible_answers = list(available_answers)
for id_action in ['r', 'a']:
if id_action in possible_answers:
possible_answers[possible_answers.index(
id_action)] = id_action + ' '
action_completer = Completer(possible_answers,
queues=pipeline_pipes.keys())
readline.set_completer(action_completer.complete)
answer = input(
inverted(', '.join(available_opts) + '?') + ' ').split()
except EOFError:
break
else:
if not answer:
continue
if len(answer) == 0 or answer[0] not in available_answers:
print('Action not allowed.')
continue
if any([answer[0] == char
for char in AVAILABLE_IDS]) and len(answer) > 1:
ids = [int(item) for item in answer[1].split(',')]
else:
ids = []
queue_name = None
if answer[0] == 'a':
# recover all -> recover all by ids
answer[0] = 'r'
ids = range(len(meta))
if len(answer) > 1:
queue_name = answer[1]
if answer[0] == 'q':
break
elif answer[0] == 'e':
# Delete entries
for entry in ids:
del content[meta[entry][0]]
save_file(handle, content)
elif answer[0] == 'r':
# recover entries
default = utils.load_configuration(DEFAULTS_CONF_FILE)
runtime = utils.load_configuration(RUNTIME_CONF_FILE)
params = utils.load_parameters(default, runtime)
pipe = pipeline.PipelineFactory.create(params, logger)
try:
for i, (key, entry) in enumerate([
item
for (count, item) in enumerate(content.items())
if count in ids
]):
if entry['message']:
msg = copy.copy(
entry['message']
) # otherwise the message field gets converted
if isinstance(msg, dict):
msg = json.dumps(msg)
else:
print('No message here, deleting entry.')
del content[key]
continue
if queue_name is None:
if len(answer) == 3:
queue_name = answer[2]
else:
queue_name = entry['source_queue']
if queue_name in pipeline_pipes:
if runtime[pipeline_pipes[queue_name]][
'group'] == 'Parser' and json.loads(
msg)['__type'] == 'Event':
print(
'Event converted to Report automatically.')
msg = message.Report(
message.MessageFactory.unserialize(
msg)).serialize()
try:
pipe.set_queues(queue_name, 'destination')
pipe.connect()
pipe.send(msg)
except exceptions.PipelineError:
print(
red('Could not reinject into queue {}: {}'
''.format(queue_name,
traceback.format_exc())))
else:
del content[key]
print(green('Recovered dump {}.'.format(i)))
finally:
save_file(handle, content)
if not content:
delete_file = True
print('Deleting empty file {}'.format(fname))
break
elif answer[0] == 'd':
# delete dumpfile
delete_file = True
print('Deleting empty file {}'.format(fname))
break
elif answer[0] == 's':
# Show entries by id
for count, (key, orig_value) in enumerate(content.items()):
value = copy.copy(
orig_value) # otherwise the raw field gets truncated
if count not in ids:
continue
print('=' * 100, '\nShowing id {} {}\n'.format(count, key),
'-' * 50)
if value.get('message_type') == 'base64':
if len(value['message']) > 1000:
value['message'] = value[
'message'][:1000] + '...[truncated]'
else:
if isinstance(value['message'], (bytes, str)):
value['message'] = json.loads(value['message'])
if ('raw' in value['message']
and len(value['message']['raw']) > 1000):
value['message']['raw'] = value['message'][
'raw'][:1000] + '...[truncated]'
if type(value['traceback']) is not list:
value['traceback'] = value['traceback'].splitlines()
pprint.pprint(value)
elif answer[0] == 'v':
# edit given id
if not ids:
print(red('Edit mode needs an id'))
continue
for entry in ids:
if content[meta[entry][0]].get('message_type') == 'base64':
with tempfile.NamedTemporaryFile(
mode='w+b', suffix='.txt') as tmphandle:
filename = tmphandle.name
tmphandle.write(
base64.b64decode(
content[meta[entry][0]]['message']))
tmphandle.flush()
proc = subprocess.call(
['sensible-editor', filename])
if proc != 0:
print(red('Calling editor failed.'))
else:
tmphandle.seek(0)
new_content = tmphandle.read()
try:
new_content = new_content.decode()
except UnicodeDecodeError as exc:
print(
red("Could not write the new message because of the following error:"
))
print(
red(
exceptions.DecodingError(
exception=exc)))
else:
del content[meta[entry][0]]['message_type']
content[meta[entry]
[0]]['message'] = new_content
save_file(handle, content)
else:
with tempfile.NamedTemporaryFile(
mode='w+t', suffix='.json') as tmphandle:
filename = tmphandle.name
utils.write_configuration(
configuration_filepath=filename,
content=json.loads(
content[meta[entry][0]]['message']),
new=True,
backup=False)
proc = subprocess.call(
['sensible-editor', filename])
if proc != 0:
print(red('Calling editor failed.'))
else:
tmphandle.seek(0)
content[meta[entry]
[0]]['message'] = tmphandle.read()
save_file(handle, content)
if delete_file:
os.remove(fname)
def bold(self, text):
self._restoreColor()
return termstyle.bold(text)
def stopFactory(self):
print termstyle.magenta(termstyle.bold('stopping factory'))
return
def bold(self, text):
self._restoreColor()
if self.html:
return '<span style="color: rgb(255,255,255);">{}</span>'.format(text)
else:
return termstyle.bold(text)
def main():
parser = argparse.ArgumentParser(
prog=APPNAME,
formatter_class=argparse.RawDescriptionHelpFormatter,
usage=USAGE,
description=DESCRIPTION,
epilog=EPILOG,
)
parser.add_argument('botid',
metavar='botid',
nargs='?',
default=None,
help='botid to inspect dumps of')
args = parser.parse_args()
ctl = intelmqctl.IntelMQContoller()
if args.botid is None:
filenames = glob.glob(os.path.join(DEFAULT_LOGGING_PATH, '*.dump'))
if not len(filenames):
print(green('Nothing to recover from, no dump files found!'))
exit(0)
filenames = [(fname, fname[len(DEFAULT_LOGGING_PATH):-5])
for fname in sorted(filenames)]
length = max([len(value[1]) for value in filenames])
print(
bold("{c:>3}: {s:{l}} {i}".format(c='id',
s='name (bot id)',
i='content',
l=length)))
for count, (fname, shortname) in enumerate(filenames):
info = dump_info(fname)
print("{c:3}: {s:{l}} {i}".format(c=count,
s=shortname,
i=info,
l=length))
try:
botid = input(
inverted('Which dump file to process (id or name)?') + ' ')
except EOFError:
exit(0)
else:
botid = botid.strip()
if botid == 'q' or not botid:
exit(0)
try:
fname, botid = filenames[int(botid)]
except ValueError:
fname = os.path.join(DEFAULT_LOGGING_PATH, botid) + '.dump'
else:
botid = args.botid
fname = os.path.join(DEFAULT_LOGGING_PATH, botid) + '.dump'
if not os.path.isfile(fname):
print(bold('Given file does not exist: {}'.format(fname)))
exit(1)
answer = None
while True:
info = dump_info(fname)
print('Processing {}: {}'.format(bold(botid), info))
if info.startswith(str(red)):
available_opts = [item[0] for item in ACTIONS.values() if item[2]]
print('Restricted actions.')
else:
# don't display list after 'show' and 'recover' command
if not (answer and isinstance(answer, list)
and answer[0] in ['s', 'r']):
with open(fname, 'rt') as handle:
content = json.load(handle)
meta = load_meta(content)
available_opts = [item[0] for item in ACTIONS.values()]
for count, line in enumerate(meta):
print('{:3}: {} {}'.format(count, *line))
# Determine bot status
bot_status = ctl.bot_status(botid)
if bot_status == 'running':
print(red('Attention: This bot is currently running!'))
elif bot_status == 'error':
print(red('Attention: This bot is not defined!'))
try:
answer = input(inverted(', '.join(available_opts) + '?') +
' ').split()
except EOFError:
break
else:
if not answer:
continue
if any([answer[0] == char
for char in AVAILABLE_IDS]) and len(answer) > 1:
ids = [int(item) for item in answer[1].split(',')]
else:
ids = []
queue_name = None
if answer[0] == 'a':
# recover all -> recover all by ids
answer[0] = 'r'
ids = range(len(meta))
if len(answer) > 1:
queue_name = answer[1]
if answer[0] == 'q':
break
elif answer[0] == 'e':
# Delete entries
for entry in ids:
del content[meta[entry][0]]
save_file(fname, content)
elif answer[0] == 'r':
if bot_status == 'running':
# See https://github.com/certtools/intelmq/issues/574
print(red('Recovery for running bots not possible.'))
continue
# recover entries
default = utils.load_configuration(DEFAULTS_CONF_FILE)
runtime = utils.load_configuration(RUNTIME_CONF_FILE)
params = utils.load_parameters(default, runtime)
pipe = pipeline.PipelineFactory.create(params)
try:
for i, (key, entry) in enumerate([
item for (count, item) in enumerate(content.items())
if count in ids
]):
if entry['message']:
msg = entry['message']
else:
print('No message here, deleting entry.')
del content[key]
continue
if queue_name is None:
if len(answer) == 3:
queue_name = answer[2]
else:
queue_name = entry['source_queue']
try:
pipe.set_queues(queue_name, 'destination')
pipe.connect()
pipe.send(msg)
except exceptions.PipelineError:
print(
red('Could not reinject into queue {}: {}'
''.format(queue_name, traceback.format_exc())))
else:
del content[key]
print(green('Recovered dump {}.'.format(i)))
finally:
save_file(fname, content)
if not content:
os.remove(fname)
print('Deleted empty file {}'.format(fname))
break
elif answer[0] == 'd':
# delete dumpfile
os.remove(fname)
print('Deleted file {}'.format(fname))
break
elif answer[0] == 's':
# Show entries by id
for count, (key, value) in enumerate(content.items()):
if count not in ids:
continue
print('=' * 100, '\nShowing id {} {}\n'.format(count, key),
'-' * 50)
if isinstance(value['message'], (bytes, str)):
value['message'] = json.loads(value['message'])
if ('raw' in value['message']
and len(value['message']['raw']) > 1000):
value['message']['raw'] = value['message'][
'raw'][:1000] + '...[truncated]'
if type(value['traceback']) is not list:
value['traceback'] = value['traceback'].splitlines()
pprint.pprint(value)
def main():
parser = argparse.ArgumentParser(
prog=APPNAME,
formatter_class=argparse.RawDescriptionHelpFormatter,
usage=USAGE,
description=DESCRIPTION,
epilog=EPILOG,
)
parser.add_argument('botid', metavar='botid', nargs='?',
default=None, help='botid to inspect dumps of')
args = parser.parse_args()
if args.botid is None:
filenames = glob.glob(os.path.join(DEFAULT_LOGGING_PATH, '*.dump'))
if not len(filenames):
print(green('Nothing to recover from, no dump files found!'))
exit(0)
filenames = [(fname, fname[len(DEFAULT_LOGGING_PATH):-5])
for fname in sorted(filenames)]
length = max([len(value[1]) for value in filenames])
print(bold("{c:>3}: {s:{l}} {i}".format(c='id', s='name (bot id)',
i='content', l=length)))
for count, (fname, shortname) in enumerate(filenames):
info = dump_info(fname)
print("{c:3}: {s:{l}} {i}".format(c=count, s=shortname, i=info,
l=length))
botid = input(inverted('Which dump file to process (id or name)? '))
botid = botid.strip()
if botid == 'q' or not botid:
exit(0)
try:
fname, botid = filenames[int(botid)]
except ValueError:
fname = os.path.join(DEFAULT_LOGGING_PATH, botid) + '.dump'
else:
botid = args.botid
fname = os.path.join(DEFAULT_LOGGING_PATH, botid) + '.dump'
if not os.path.isfile(fname):
print(bold('Given file does not exist: {}'.format(fname)))
exit(1)
while True:
info = dump_info(fname)
print('Processing {}: {}'.format(bold(botid), info))
try:
with io.open(fname, 'rt') as handle:
content = json.load(handle)
meta = load_meta(content)
except ValueError:
available_opts = [item[0] for item in ACTIONS.values() if item[2]]
print(bold('Could not load file:') + '\n{}\nRestricted actions.'
''.format(traceback.format_exc()))
else:
available_opts = [item[0] for item in ACTIONS.values()]
for count, line in enumerate(meta):
print('{:3}: {} {}'.format(count, *line))
answer = input(inverted(', '.join(available_opts) + '? ')).split()
if not answer:
continue
if any([answer[0] == char for char in AVAILABLE_IDS]):
ids = [int(item) for item in answer[1].split(',')]
queue_name = None
if answer[0] == 'a':
# recover all -> recover all by ids
answer[0] = 'r'
ids = range(len(meta))
if len(answer) > 1:
queue_name = answer[1]
if answer[0] == 'q':
break
elif answer[0] == 'e':
# Delete entries
for entry in ids:
del content[meta[entry][0]]
save_file(fname, content)
elif answer[0] == 'r':
# recover entries
for key, entry in [item for (count, item)
in enumerate(content.items()) if count in ids]:
if type(entry['message']) is dict:
if '__type' in entry['message']:
msg = json.dumps(entry['message'])
# backwards compat: dumps had no type info
elif '-parser' in entry['bot_id']:
msg = message.Report(entry['message']).serialize()
else:
msg = message.Event(entry['message']).serialize()
elif issubclass(type(entry['message']), (six.binary_type,
six.text_type)):
msg = entry['message']
elif entry['message'] is None:
print(bold('No message here, deleting directly.'))
del content[key]
save_file(fname, content)
continue
else:
print(bold('Unhandable type of message: {!r}'
''.format(type(entry['message']))))
continue
print(entry['source_queue'])
default = utils.load_configuration(DEFAULTS_CONF_FILE)
runtime = utils.load_configuration(RUNTIME_CONF_FILE)
params = utils.load_parameters(default, runtime)
pipe = pipeline.PipelineFactory.create(params)
if queue_name is None:
if len(answer) == 2:
queue_name = answer[2]
else:
queue_name = entry['source_queue']
try:
pipe.set_queues(queue_name, 'destination')
pipe.connect()
pipe.send(msg)
except exceptions.PipelineError:
print(red('Could not reinject into queue {}: {}'
''.format(queue_name, traceback.format_exc())))
else:
del content[key]
save_file(fname, content)
elif answer[0] == 'd':
# delete dumpfile
os.remove(fname)
print('Deleted file {}'.format(fname))
break
elif answer[0] == 's':
# Show entries by id
for count, (key, value) in enumerate(content.items()):
if count not in ids:
continue
print('=' * 100, '\nShowing id {} {}\n'.format(count, key),
'-' * 50)
if isinstance(value['message'], (six.binary_type,
six.text_type)):
value['message'] = json.loads(value['message'])
if ('raw' in value['message'] and
len(value['message']['raw']) > 1000):
value['message']['raw'] = value['message'][
'raw'][:1000] + '...[truncated]'
value['traceback'] = value['traceback'].splitlines()
pprint.pprint(value)
