def __init__(self, bot_id):
self.__log_buffer = []
self.parameters = Parameters()
self.__error_retries_counter = 0
self.__source_pipeline = None
self.__destination_pipeline = None
self.logger = None
try:
version_info = sys.version.splitlines()[0].strip()
self.__log_buffer.append(
('info', '{} initialized with id {} and version '
'{} as process {}.'
''.format(self.__class__.__name__, bot_id, version_info,
os.getpid())))
self.__log_buffer.append(('debug', 'Library path: %r.' % __file__))
self.__load_defaults_configuration()
self.__load_system_configuration()
self.__check_bot_id(bot_id)
self.__bot_id = bot_id
if self.parameters.logging_handler == 'syslog':
syslog = self.parameters.logging_syslog
else:
syslog = False
self.logger = utils.log(self.__bot_id,
syslog=syslog,
log_path=self.parameters.logging_path,
log_level=self.parameters.logging_level)
except:
self.__log_buffer.append(('critical', traceback.format_exc()))
self.stop()
else:
for line in self.__log_buffer:
getattr(self.logger, line[0])(line[1])
try:
self.logger.info('Bot is starting.')
self.__load_runtime_configuration()
self.__load_pipeline_configuration()
self.__load_harmonization_configuration()
self.init()
self.__sighup = False
signal.signal(signal.SIGHUP, self.__handle_sighup_signal)
# system calls should not be interrupted, but restarted
signal.siginterrupt(signal.SIGHUP, False)
except Exception as exc:
if self.parameters.error_log_exception:
self.logger.exception('Bot initialization failed.')
else:
self.logger.error(utils.error_message_from_exc(exc))
self.logger.error('Bot initialization failed.')
self.stop()
raise
def __init__(self, bot_id):
self.parameters = Parameters()
self.current_message = None
self.last_message = None
self.message_counter = 0
self.error_retries_counter = 0
self.check_bot_id(bot_id)
self.bot_id = bot_id
self.load_system_configuration()
self.logger = utils.log(
self.parameters.logging_path,
self.bot_id,
self.parameters.logging_level
)
self.logger.info('Bot is starting')
self.load_defaults_configuration()
self.load_runtime_configuration()
self.load_pipeline_configuration()
self.load_harmonization_configuration()
self.init()
def __init__(self, bot_id):
self.__log_buffer = []
self.parameters = Parameters()
self.__error_retries_counter = 0
self.__source_pipeline = None
self.__destination_pipeline = None
self.logger = None
try:
version_info = sys.version.splitlines()[0].strip()
self.__log_buffer.append(('info',
'{} initialized with id {} and version '
'{} as process {}.'
''.format(self.__class__.__name__,
bot_id, version_info,
os.getpid())))
self.__log_buffer.append(('debug', 'Library path: %r.' % __file__))
self.__load_defaults_configuration()
self.__load_system_configuration()
self.__check_bot_id(bot_id)
self.__bot_id = bot_id
if self.parameters.logging_handler == 'syslog':
syslog = self.parameters.logging_syslog
else:
syslog = False
self.logger = utils.log(self.__bot_id, syslog=syslog,
log_path=self.parameters.logging_path,
log_level=self.parameters.logging_level)
except:
self.__log_buffer.append(('critical', traceback.format_exc()))
self.stop()
else:
for line in self.__log_buffer:
getattr(self.logger, line[0])(line[1])
try:
self.logger.info('Bot is starting.')
self.__load_runtime_configuration()
self.__load_pipeline_configuration()
self.__load_harmonization_configuration()
self.init()
self.__sighup = False
signal.signal(signal.SIGHUP, self.__handle_sighup_signal)
# system calls should not be interrupted, but restarted
signal.siginterrupt(signal.SIGHUP, False)
except Exception as exc:
if self.parameters.error_log_exception:
self.logger.exception('Bot initialization failed.')
else:
self.logger.error(utils.error_message_from_exc(exc))
self.logger.error('Bot initialization failed.')
self.stop()
raise
def __is_valid_value(self, key, value):
if key == '__type':
return (True, )
config = self.__get_type_config(key)
class_reference = getattr(intelmq.lib.harmonization, config['type'])
if not class_reference().is_valid(value):
logger = utils.log("edvard", log_level='DEBUG')
logger.info(config['type'])
logger.info(intelmq.lib.harmonization)
logger.info(config)
logger.info(class_reference())
logger.info(value)
return (False, 'is_valid returned False.')
if 'length' in config:
length = len(str(value))
if not length <= config['length']:
return (False,
'too long: {} > {}.'.format(length, config['length']))
if 'regex' in config:
if not re.search(config['regex'], str(value)):
return (False, 'regex did not match.')
if 'iregex' in config:
if not re.search(config['iregex'], str(value), re.IGNORECASE):
return (False, 'regex (case insensitive) did not match.')
return (True, )
def __init_logger(self):
"""
Initialize the logger.
"""
if self.parameters.logging_handler == 'syslog':
syslog = self.parameters.logging_syslog
else:
syslog = False
self.logger = utils.log(self.__bot_id_full, syslog=syslog,
log_path=self.parameters.logging_path,
log_level=self.parameters.logging_level)
def __init_logger(self):
"""
Initialize the logger.
"""
if self.parameters.logging_handler == 'syslog':
syslog = self.parameters.logging_syslog
else:
syslog = False
self.logger = utils.log(self.__bot_id_full, syslog=syslog,
log_path=self.parameters.logging_path,
log_level=self.parameters.logging_level)
def __init__(self, bot_id):
self.log_buffer = []
self.parameters = Parameters()
self.current_message = None
self.last_message = None
self.message_counter = 0
self.error_retries_counter = 0
self.source_pipeline = None
self.destination_pipeline = None
self.logger = None
try:
version_info = sys.version.splitlines()[0].strip()
self.log_buffer.append(
('info', '{} initialized with id {} and version {}.'
''.format(self.__class__.__name__, bot_id, version_info)))
self.check_bot_id(bot_id)
self.bot_id = bot_id
self.load_defaults_configuration()
self.load_system_configuration()
import os
self.log_buffer.append(('info', os.path.abspath(utils.__file__)))
self.log_buffer.append(('info', utils.log.__doc__))
self.log_buffer.append(('info', repr(sys.modules['intelmq'])))
if self.parameters.logging_handler == 'syslog':
syslog = self.parameters.logging_syslog
else:
syslog = False
self.logger = utils.log(self.bot_id,
log_level=self.parameters.logging_level)
# syslog=syslog)
except:
self.log_buffer.append(('critical', traceback.format_exc()))
self.stop()
else:
for line in self.log_buffer:
getattr(self.logger, line[0])(line[1])
try:
self.logger.info('Bot is starting.')
self.load_runtime_configuration()
self.load_pipeline_configuration()
self.load_harmonization_configuration()
self.init()
except:
self.logger.exception('Bot initialization failed.')
raise
def __init__(self, bot_id):
self.__log_buffer = []
self.parameters = Parameters()
self.__current_message = None
self.__message_counter = 0
self.__error_retries_counter = 0
self.__source_pipeline = None
self.__destination_pipeline = None
self.logger = None
try:
version_info = sys.version.splitlines()[0].strip()
self.__log_buffer.append(
(
"info",
"{} initialized with id {} and version "
"{} as process {}."
"".format(self.__class__.__name__, bot_id, version_info, os.getpid()),
)
)
self.__load_defaults_configuration()
self.__load_system_configuration()
self.__check_bot_id(bot_id)
self.__bot_id = bot_id
if self.parameters.logging_handler == "syslog":
syslog = self.parameters.logging_syslog
else:
syslog = False
self.logger = utils.log(self.__bot_id, syslog=syslog, log_level=self.parameters.logging_level)
except:
self.__log_buffer.append(("critical", traceback.format_exc()))
self.stop()
else:
for line in self.__log_buffer:
getattr(self.logger, line[0])(line[1])
try:
self.logger.info("Bot is starting.")
self.__load_runtime_configuration()
self.__load_pipeline_configuration()
self.__load_harmonization_configuration()
self.init()
signal.signal(signal.SIGHUP, self.__sighup)
except:
self.logger.exception("Bot initialization failed.")
raise
def __init__(self, bot_id):
self.__log_buffer = []
self.parameters = Parameters()
self.__current_message = None
self.__last_message = None
self.__message_counter = 0
self.__error_retries_counter = 0
self.__source_pipeline = None
self.__destination_pipeline = None
self.logger = None
try:
version_info = sys.version.splitlines()[0].strip()
self.__log_buffer.append(('info',
'{} initialized with id {} and version '
'{} as process {}.'
''.format(self.__class__.__name__,
bot_id, version_info,
os.getpid())))
self.__load_defaults_configuration()
self.__load_system_configuration()
self.__check_bot_id(bot_id)
self.__bot_id = bot_id
if self.parameters.logging_handler == 'syslog':
syslog = self.parameters.logging_syslog
else:
syslog = False
self.logger = utils.log(self.__bot_id, syslog=syslog,
log_level=self.parameters.logging_level)
except:
self.__log_buffer.append(('critical', traceback.format_exc()))
self.stop()
else:
for line in self.__log_buffer:
getattr(self.logger, line[0])(line[1])
try:
self.logger.info('Bot is starting.')
self.__load_runtime_configuration()
self.__load_pipeline_configuration()
self.__load_harmonization_configuration()
self.init()
signal.signal(signal.SIGHUP, self.__sighup)
except:
self.logger.exception('Bot initialization failed.')
raise
def __init__(self, bot_id):
self.log_buffer = []
self.parameters = Parameters()
self.current_message = None
self.last_message = None
self.message_counter = 0
self.error_retries_counter = 0
self.source_pipeline = None
self.destination_pipeline = None
self.logger = None
try:
version_info = sys.version.splitlines()[0].strip()
self.log_buffer.append(('info',
'{} initialized with id {} and version {}.'
''.format(self.__class__.__name__,
bot_id, version_info)))
self.check_bot_id(bot_id)
self.bot_id = bot_id
self.load_defaults_configuration()
self.load_system_configuration()
import os
self.log_buffer.append(('info', os.path.abspath(utils.__file__)))
self.log_buffer.append(('info', utils.log.__doc__))
self.log_buffer.append(('info', repr(sys.modules['intelmq'])))
if self.parameters.logging_handler == 'syslog':
syslog = self.parameters.logging_syslog
else:
syslog = False
self.logger = utils.log(self.bot_id,
log_level=self.parameters.logging_level)
# syslog=syslog)
except:
self.log_buffer.append(('critical', traceback.format_exc()))
self.stop()
else:
for line in self.log_buffer:
getattr(self.logger, line[0])(line[1])
try:
self.logger.info('Bot is starting.')
self.load_runtime_configuration()
self.load_pipeline_configuration()
self.load_harmonization_configuration()
self.init()
except:
self.logger.exception('Bot initialization failed.')
raise
def test_stream_logger(self):
stdout = io.StringIO()
stderr = io.StringIO()
with contextlib.redirect_stdout(stdout):
with contextlib.redirect_stderr(stderr):
logger = utils.log('test-bot', log_path=None)
logger.info(LINES['spare'][0])
logger.error(LINES['spare'][1])
logger.critical(LINES['spare'][2])
line_format = [line.format('test-bot') for line in LINES['short']]
self.assertEqual(stdout.getvalue(), line_format[0] + '\n')
self.assertEqual(
stderr.getvalue(), '\n'.join((termstyle.red(
line_format[1]), termstyle.red(line_format[2]))) + '\n')
def test_stream_logger(self):
"""Tests if a logger for a stream can be generated with log()."""
stream = io.StringIO()
with tempfile.NamedTemporaryFile() as handle:
filename = handle.name
name = os.path.split(filename)[-1]
logger = utils.log(name, log_path=tempfile.tempdir, stream=stream)
logger.info(LINES['spare'][0])
logger.error(LINES['spare'][1])
logger.critical(LINES['spare'][2])
stream_lines = stream.getvalue().splitlines()
line_format = [line.format(name) for line in LINES['short']]
self.assertSequenceEqual(line_format, stream_lines)
def test_stream_logger(self):
"""Tests if a logger for a stream can be generated with log()."""
stream = io.StringIO()
with tempfile.NamedTemporaryFile() as handle:
filename = handle.name
name = os.path.split(filename)[-1]
logger = utils.log(name, log_path=tempfile.tempdir, stream=stream)
logger.info(LINES['spare'][0])
logger.error(LINES['spare'][1])
logger.critical(LINES['spare'][2])
stream_lines = stream.getvalue().splitlines()
line_format = [line.format(name) for line in LINES['short']]
self.assertSequenceEqual(line_format, stream_lines)
def test_file_logger(self):
"""Tests if a logger for a file can be generated with log()."""
with tempfile.NamedTemporaryFile(suffix=".log", mode='w+') as handle:
filename = handle.name
name = os.path.splitext(os.path.split(filename)[-1])[0]
logger = utils.log(name, log_path=tempfile.tempdir,
stream=io.StringIO())
logger.info(termstyle.green(LINES['spare'][0]))
logger.error(LINES['spare'][1])
logger.critical(LINES['spare'][2])
handle.seek(0)
file_lines = handle.readlines()
line_format = [line.format(name) for line in LINES['long']]
for ind, line in enumerate(file_lines):
self.assertRegex(line.strip(), line_format[ind])
def test_file_logger(self):
"""Tests if a logger for a file can be generated with log()."""
with tempfile.NamedTemporaryFile(suffix=".log", mode='w+') as handle:
filename = handle.name
name = os.path.splitext(os.path.split(filename)[-1])[0]
logger = utils.log(name, log_path=tempfile.tempdir,
stream=io.StringIO())
logger.info(LINES['spare'][0])
logger.error(LINES['spare'][1])
logger.critical(LINES['spare'][2])
handle.seek(0)
file_lines = handle.readlines()
line_format = [line.format(name) for line in LINES['long']]
for ind, line in enumerate(file_lines):
self.assertRegex(line.strip(), line_format[ind])
def __init__(self, bot_id):
self.log_buffer = []
self.parameters = Parameters()
self.current_message = None
self.last_message = None
self.message_counter = 0
self.error_retries_counter = 0
self.source_pipeline = None
self.destination_pipeline = None
self.logger = None
try:
self.log_buffer.append(('debug',
'{} initialized with id {}.'
''.format(self.__class__.__name__,
bot_id)))
self.check_bot_id(bot_id)
self.bot_id = bot_id
self.load_defaults_configuration()
self.load_system_configuration()
self.logger = utils.log(self.bot_id,
log_level=self.parameters.logging_level)
except:
self.log_buffer.append(('critical', traceback.format_exc()))
self.stop()
else:
for line in self.log_buffer:
getattr(self.logger, line[0])(line[1])
try:
self.logger.info('Bot is starting.')
self.load_runtime_configuration()
self.load_pipeline_configuration()
self.load_harmonization_configuration()
self.init()
except:
self.logger.exception('Bot initialization failed.')
raise
def __init__(self, bot_id):
self.parameters = Parameters()
self.current_message = None
self.last_message = None
self.message_counter = 0
self.check_bot_id(bot_id)
self.bot_id = bot_id
self.load_system_configurations()
self.logger = log(self.parameters.logging_path, self.bot_id, self.parameters.logging_level)
self.logger.info('Bot is starting')
self.load_runtime_configurations()
self.src_queue, self.dest_queues = self.load_pipeline()
self.parameters.processing_interval = float(self.parameters.processing_interval)
self.init()
def test_file_logger(self):
"""Tests if a logger for a file can be generated with log()."""
with tempfile.NamedTemporaryFile() as handle:
filename = handle.name
name = os.path.split(filename)[-1]
logger = utils.log(name, log_path=tempfile.tempdir,
stream=io.StringIO())
logger.info(LINES['spare'][0])
logger.error(LINES['spare'][1])
logger.critical(LINES['spare'][2])
handle.seek(0)
file_lines = handle.readlines()
line_format = [line.format(name) for line in LINES['long']]
for ind, line in enumerate(file_lines):
try:
self.assertRegex(line, line_format[ind])
except AttributeError: # Py2
self.assertRegexpMatches(line, line_format[ind])
def __init__(self, bot_id):
self.parameters = Parameters()
self.current_message = None
self.last_message = None
self.message_counter = 0
self.check_bot_id(bot_id)
self.bot_id = bot_id
self.load_system_configurations()
self.logger = log(self.parameters.logging_path, self.bot_id, self.parameters.logging_level)
self.logger.info('Bot is starting')
self.load_runtime_configurations()
self.source_queue, self.destination_queues = self.load_pipeline()
self.parameters.rate_limit = float(self.parameters.rate_limit)
self.parameters.retry_delay = int(self.parameters.retry_delay)
self.init()
def __init__(self, bot_id):
self.parameters = Parameters()
self.current_message = None
self.last_message = None
self.message_counter = 0
self.check_bot_id(bot_id)
self.bot_id = bot_id
self.load_system_configurations()
self.logger = log(self.parameters.logging_path, self.bot_id, self.parameters.logging_level)
self.logger.info("Bot is starting")
self.load_runtime_configurations()
self.load_pipeline_configurations()
# == NEW CODE
self.parameters.threads_number = 1
# == NEW CODE
self.init()
def __init__(self):
global RETURN_TYPE
global logger
logger = utils.log('intelmqctl', log_level='DEBUG')
self.logger = logger
APPNAME = "intelmqctl"
VERSION = "0.0.0"
DESCRIPTION = """
description: intelmqctl is the tool to control intelmq system.
Outputs are logged to /opt/intelmq/var/log/intelmqctl"""
USAGE = '''
intelmqctl --bot [start|stop|restart|status] --id=cymru-expert
intelmqctl --botnet [start|stop|restart|status]
intelmqctl --list [bots|queues]'''
parser = argparse.ArgumentParser(prog=APPNAME,
usage=USAGE,
epilog=DESCRIPTION)
group = parser.add_mutually_exclusive_group()
group_list = group.add_mutually_exclusive_group()
parser.add_argument('-v',
'--version',
action='version',
version=VERSION)
parser.add_argument('--id',
'-i',
dest='bot_id',
default=None,
help='bot ID')
parser.add_argument('--type',
'-t',
choices=RETURN_TYPES,
default=RETURN_TYPES[0],
help='choose if it should return regular text or '
'other forms of output')
group_list.add_argument('--log',
'-l',
metavar='[log-level]:[number-of-lines]',
default=None,
help='''Reads the last lines from bot log, or
from system log if no bot ID was given.
Log level should be one of DEBUG, INFO, ERROR
or CRTICAL. Default is INFO.
Number of lines defaults to 10, -1 gives all.
Reading from system log is not implemented yet.
''')
group_list.add_argument('--bot',
'-b',
choices=['start', 'stop', 'restart', 'status'],
metavar='[start|stop|restart|status]',
default=None)
group_list.add_argument('--botnet',
'-n',
choices=['start', 'stop', 'restart', 'status'],
metavar='[start|stop|restart|status]',
default=None)
group_list.add_argument('--list',
'-s',
choices=['bots', 'queues'],
metavar='[bots|queues]',
default=None)
group_list.add_argument('--clear',
'-c',
metavar='queue',
default=None,
help='''Clears the given queue in broker''')
self.args = parser.parse_args()
if len(sys.argv) == 1:
parser.print_help()
RETURN_TYPE = self.args.type
with open(STARTUP_CONF_FILE, 'r') as fp:
self.startup = json.load(fp)
with open(SYSTEM_CONF_FILE, 'r') as fp:
self.system = json.load(fp)
if not os.path.exists(PIDDIR):
os.makedirs(PIDDIR)
# stolen functions from the bot file
# this will not work with various instances of REDIS
self.parameters = Parameters()
self.load_defaults_configuration()
self.load_system_configuration()
self.pipepline_configuration = utils.load_configuration(
PIPELINE_CONF_FILE)
self.runtime_configuration = utils.load_configuration(
RUNTIME_CONF_FILE)
self.startup_configuration = utils.load_configuration(
STARTUP_CONF_FILE)
def setup(self):
self.args = self.parser.parse_args()
if self.args.verbose:
self.verbose = True
if self.args.dry_run:
self.dryrun = True
if self.args.batch:
self.batch = True
if self.args.quiet:
self.quiet = True
if self.args.feed:
self.additional_where += """ AND "feed.name" = ANY(%s::VARCHAR[]) """
self.additional_params += ('{' + ','.join(self.args.feed) + '}', )
if self.args.skip_feed:
self.additional_where += """ AND "feed.name" != ANY(%s::VARCHAR[]) """
self.additional_params += ('{' + ','.join(self.args.skip_feed) + '}', )
if self.args.asn:
self.additional_where += """ AND "source.asn" = ANY(%s::INT[]) """
self.additional_params += ('{' + ','.join(map(str, self.args.asn)) + '}', )
if self.args.skip_asn:
self.additional_where += """ AND "source.asn" != ANY(%s::INT[]) """
self.additional_params += ('{' + ','.join(map(str, self.args.skip_asn)) + '}', )
if self.args.taxonomy:
self.additional_where += """ AND "classification.taxonomy" = ANY(%s::VARCHAR[]) """
self.additional_params += ('{' + ','.join(self.args.taxonomy) + '}', )
if self.args.skip_taxonomy:
self.additional_where += """ AND "classification.taxonomy" != ANY(%s::VARCHAR[]) """
self.additional_params += ('{' + ','.join(self.args.skip_taxonomy) + '}', )
if self.args.type:
self.additional_where += """ AND "classification.type" = ANY(%s::VARCHAR[]) """
self.additional_params += ('{' + ','.join(self.args.type) + '}', )
if self.args.skip_type:
self.additional_where += """ AND "classification.type" != ANY(%s::VARCHAR[]) """
self.additional_params += ('{' + ','.join(self.args.skip_type) + '}', )
if self.args.identifier:
self.additional_where += """ AND "classification.identifier" = ANY(%s::VARCHAR[]) """
self.additional_params += ('{' + ','.join(self.args.identifier) + '}', )
if self.args.skip_identifier:
self.additional_where += """ AND "classification.identifier" != ANY(%s::VARCHAR[]) """
self.additional_params += ('{' + ','.join(self.args.skip_identifier) + '}', )
with open('/etc/intelmq/intelmqcli.conf') as conf_handle:
self.config = json.load(conf_handle)
home = os.path.expanduser("~")
with open(os.path.expanduser(home + '/.intelmq/intelmqcli.conf')) as conf_handle:
user_config = json.load(conf_handle)
for key, value in user_config.items():
if key in self.config and isinstance(value, dict):
self.config[key].update(value)
else:
self.config[key] = value
if self.quiet:
stream = None
else:
stream = sys.stderr
self.logger = utils.log('intelmqcli', syslog='/dev/log',
log_level=self.config['log_level'].upper(),
stream=stream, log_format_stream='%(message)s')
self.rt = rt.Rt(self.config['rt']['uri'], self.config['rt']['user'],
self.config['rt']['password'])
def __init__(self, interactive=False, return_type="python", quiet=False):
"""
Initializes intelmqctl.
Parameters
==========
interactive : boolean
for cli-interface true, functions can exits, parameters are used
return_type : string
'python': no special treatment, can be used for use by other
python code
'text': user-friendly output for cli, default for interactive use
'json': machine-readable output for managers
quiet : boolean
False by default, can be activated for cronjobs etc.
"""
global RETURN_TYPE
RETURN_TYPE = return_type
global logger
global QUIET
QUIET = quiet
logger = utils.log('intelmqctl', log_level='DEBUG')
self.logger = logger
self.interactive = interactive
self.args = None
if os.geteuid() == 0:
logger.warning('Running intelmq as root is highly discouraged!')
APPNAME = "intelmqctl"
VERSION = pkg_resources.get_distribution("intelmq").version
DESCRIPTION = """
description: intelmqctl is the tool to control intelmq system.
Outputs are logged to /opt/intelmq/var/log/intelmqctl"""
USAGE = '''
intelmqctl [start|stop|restart|status|reload|run] bot-id
intelmqctl [start|stop|restart|status|reload]
intelmqctl list [bots|queues]
intelmqctl log bot-id [number-of-lines [log-level]]
intelmqctl clear queue-id
intelmqctl check
Starting a bot:
intelmqctl start bot-id
Stopping a bot:
intelmqctl stop bot-id
Restarting a bot:
intelmqctl restart bot-id
Get status of a bot:
intelmqctl status bot-id
Run a bot directly (blocking) for debugging purpose:
intelmqctl run bot-id
Starting the botnet (all bots):
intelmqctl start
etc.
Get a list of all configured bots:
intelmqctl list bots
Get a list of all queues:
intelmqctl list queues
Clear a queue:
intelmqctl clear queue-id
Get logs of a bot:
intelmqctl log bot-id [number-of-lines [log-level]]
Reads the last lines from bot log, or from system log if no bot ID was
given. Log level should be one of DEBUG, INFO, ERROR or CRITICAL.
Default is INFO. Number of lines defaults to 10, -1 gives all. Result
can be longer due to our logging format!'''
# stolen functions from the bot file
# this will not work with various instances of REDIS
self.parameters = Parameters()
self.load_defaults_configuration()
self.load_system_configuration()
try:
self.pipeline_configuration = utils.load_configuration(PIPELINE_CONF_FILE)
except ValueError as exc:
exit('Invalid syntax in %r: %s' % (PIPELINE_CONF_FILE, exc))
try:
self.runtime_configuration = utils.load_configuration(RUNTIME_CONF_FILE)
except ValueError as exc:
exit('Invalid syntax in %r: %s' % (RUNTIME_CONF_FILE, exc))
if os.path.exists(STARTUP_CONF_FILE):
self.logger.warning('Deprecated startup.conf file found, please migrate to runtime.conf soon.')
with open(STARTUP_CONF_FILE, 'r') as fp:
startup = json.load(fp)
for bot_id, bot_values in startup.items():
if 'parameters' in self.runtime_configuration[bot_id]:
self.logger.error('Mixed setup of new runtime.conf and old startup.conf'
' found. Ignoring startup.conf, please fix this!')
exit(1)
params = self.runtime_configuration[bot_id].copy()
self.runtime_configuration[bot_id].clear()
self.runtime_configuration[bot_id]['parameters'] = params
self.runtime_configuration[bot_id].update(bot_values)
try:
with open(RUNTIME_CONF_FILE + '.new', 'w') as fp:
json.dump(self.runtime_configuration, fp, indent=4, sort_keys=True,
separators=(',', ': '))
except PermissionError:
self.logger.info('Failed to write new configuration format to %r.'
'' % (RUNTIME_CONF_FILE + '.new'))
else:
self.logger.info('%r with new format written.' % (RUNTIME_CONF_FILE + '.new'))
self.bot_process_manager = BotProcessManager(
self.runtime_configuration
)
if self.interactive:
parser = argparse.ArgumentParser(
prog=APPNAME,
usage=USAGE,
epilog=DESCRIPTION
)
parser.add_argument('-v', '--version',
action='version', version=VERSION)
parser.add_argument('--type', '-t', choices=RETURN_TYPES,
default=RETURN_TYPES[0],
help='choose if it should return regular text '
'or other machine-readable')
parser.add_argument('action',
choices=['start', 'stop', 'restart', 'status',
'reload', 'run', 'list', 'clear',
'help', 'log', 'check'],
metavar='[start|stop|restart|status|reload|run'
'|list|clear|log|check]')
parser.add_argument('parameter', nargs='*')
parser.add_argument('--quiet', '-q', action='store_const',
help='Quiet mode, useful for reloads initiated'
'scripts like logrotate',
const=True)
self.parser = parser
self.args = parser.parse_args()
if self.args.action == 'help':
parser.print_help()
exit(0)
RETURN_TYPE = self.args.type
QUIET = self.args.quiet
def __init__(self, interactive=False, return_type="python", quiet=False):
"""
Initializes intelmqctl.
Parameters
==========
interactive : boolean
for cli-interface true, functions can exits, parameters are used
return_type : string
'python': no special treatment, can be used for use by other
python code
'text': user-friendly output for cli, default for interactive use
'json': machine-readable output for managers
quiet : boolean
False by default, can be activated for cronjobs etc.
"""
global RETURN_TYPE
RETURN_TYPE = return_type
global logger
global QUIET
QUIET = quiet
logger = utils.log('intelmqctl', log_level='DEBUG')
self.logger = logger
self.interactive = interactive
self.args = None
if os.geteuid() == 0:
logger.warning('Running intelmq as root is highly discouraged!')
APPNAME = "intelmqctl"
VERSION = pkg_resources.get_distribution("intelmq").version
DESCRIPTION = """
description: intelmqctl is the tool to control intelmq system.
Outputs are logged to /opt/intelmq/var/log/intelmqctl"""
USAGE = '''
intelmqctl [start|stop|restart|status|reload|run] bot-id
intelmqctl [start|stop|restart|status|reload]
intelmqctl list [bots|queues]
intelmqctl log bot-id [number-of-lines [log-level]]
intelmqctl clear queue-id
intelmqctl check
Starting a bot:
intelmqctl start bot-id
Stopping a bot:
intelmqctl stop bot-id
Restarting a bot:
intelmqctl restart bot-id
Get status of a bot:
intelmqctl status bot-id
Run a bot directly (blocking) for debugging purpose:
intelmqctl run bot-id
Starting the botnet (all bots):
intelmqctl start
etc.
Get a list of all configured bots:
intelmqctl list bots
Get a list of all queues:
intelmqctl list queues
Clear a queue:
intelmqctl clear queue-id
Get logs of a bot:
intelmqctl log bot-id [number-of-lines [log-level]]
Reads the last lines from bot log, or from system log if no bot ID was
given. Log level should be one of DEBUG, INFO, ERROR or CRITICAL.
Default is INFO. Number of lines defaults to 10, -1 gives all. Result
can be longer due to our logging format!'''
# stolen functions from the bot file
# this will not work with various instances of REDIS
self.parameters = Parameters()
self.load_defaults_configuration()
self.load_system_configuration()
self.pipepline_configuration = utils.load_configuration(
PIPELINE_CONF_FILE)
self.runtime_configuration = utils.load_configuration(
RUNTIME_CONF_FILE)
if os.path.exists(STARTUP_CONF_FILE):
self.logger.warning(
'Deprecated startup.conf file found, please migrate to runtime.conf soon.'
)
with open(STARTUP_CONF_FILE, 'r') as fp:
startup = json.load(fp)
for bot_id, bot_values in startup.items():
if 'parameters' in self.runtime_configuration[bot_id]:
self.logger.error(
'Mixed setup of new runtime.conf and old startup.conf'
' found. Ignoring startup.conf, please fix this!')
exit(1)
params = self.runtime_configuration[bot_id].copy()
self.runtime_configuration[bot_id].clear()
self.runtime_configuration[bot_id]['parameters'] = params
self.runtime_configuration[bot_id].update(bot_values)
try:
with open(RUNTIME_CONF_FILE + '.new', 'w') as fp:
json.dump(self.runtime_configuration,
fp,
indent=4,
sort_keys=True,
separators=(',', ': '))
except PermissionError:
self.logger.info(
'Failed to write new configuration format to %r.'
'' % (RUNTIME_CONF_FILE + '.new'))
else:
self.logger.info('%r with new format written.' %
(RUNTIME_CONF_FILE + '.new'))
if not os.path.exists(PIDDIR):
os.makedirs(PIDDIR)
if self.interactive:
parser = argparse.ArgumentParser(prog=APPNAME,
usage=USAGE,
epilog=DESCRIPTION)
parser.add_argument('-v',
'--version',
action='version',
version=VERSION)
parser.add_argument('--type',
'-t',
choices=RETURN_TYPES,
default=RETURN_TYPES[0],
help='choose if it should return regular text '
'or other machine-readable')
parser.add_argument('action',
choices=[
'start', 'stop', 'restart', 'status',
'reload', 'run', 'list', 'clear', 'help',
'log', 'check'
],
metavar='[start|stop|restart|status|reload|run'
'|list|clear|log|check]')
parser.add_argument('parameter', nargs='*')
parser.add_argument('--quiet',
'-q',
action='store_const',
help='Quiet mode, useful for reloads initiated'
'scripts like logrotate',
const=True)
self.parser = parser
self.args = parser.parse_args()
if self.args.action == 'help':
parser.print_help()
exit(0)
RETURN_TYPE = self.args.type
QUIET = self.args.quiet
def main():
parser = argparse.ArgumentParser(
prog=APPNAME,
formatter_class=argparse.RawDescriptionHelpFormatter,
usage=USAGE,
description=DESCRIPTION,
epilog=EPILOG,
)
parser.add_argument('botid', metavar='botid', nargs='?',
default=None, help='botid to inspect dumps of')
args = parser.parse_args()
# Try to get log_level from defaults_configuration, else use default
try:
log_level = utils.load_configuration(DEFAULTS_CONF_FILE)['logging_level']
except Exception:
log_level = DEFAULT_LOGGING_LEVEL
try:
logger = utils.log('intelmqdump', log_level=log_level)
except (FileNotFoundError, PermissionError) as exc:
logger = utils.log('intelmqdump', log_level=log_level, log_path=False)
logger.error('Not logging to file: %s', exc)
ctl = intelmqctl.IntelMQController()
readline.parse_and_bind("tab: complete")
readline.set_completer_delims('')
pipeline_config = utils.load_configuration(PIPELINE_CONF_FILE)
pipeline_pipes = {}
for bot, pipes in pipeline_config.items():
pipeline_pipes[pipes.get('source-queue', '')] = bot
if args.botid is None:
filenames = glob.glob(os.path.join(DEFAULT_LOGGING_PATH, '*.dump'))
if not len(filenames):
print(green('Nothing to recover from, no dump files found!'))
sys.exit(0)
filenames = [(fname, fname[len(DEFAULT_LOGGING_PATH):-5])
for fname in sorted(filenames)]
length = max([len(value[1]) for value in filenames])
print(bold("{c:>3}: {s:{length}} {i}".format(c='id', s='name (bot id)',
i='content',
length=length)))
for count, (fname, shortname) in enumerate(filenames):
info = dump_info(fname)
print("{c:3}: {s:{length}} {i}".format(c=count, s=shortname, i=info,
length=length))
try:
bot_completer = Completer(possible_values=[f[1] for f in filenames])
readline.set_completer(bot_completer.complete)
botid = input(inverted('Which dump file to process (id or name)?') +
' ')
except EOFError:
sys.exit(0)
else:
botid = botid.strip()
if botid == 'q' or not botid:
exit(0)
try:
fname, botid = filenames[int(botid)]
except ValueError:
fname = os.path.join(DEFAULT_LOGGING_PATH, botid) + '.dump'
else:
botid = args.botid
fname = os.path.join(DEFAULT_LOGGING_PATH, botid) + '.dump'
if not os.path.isfile(fname):
print(bold('Given file does not exist: {}'.format(fname)))
exit(1)
answer = None
delete_file = False
while True:
with open(fname, 'r+') as handle:
try:
fcntl.flock(handle, fcntl.LOCK_EX | fcntl.LOCK_NB)
except BlockingIOError:
print(red('Dump file is currently locked. Stopping.'))
break
info = dump_info(fname, file_descriptor=handle)
handle.seek(0)
available_answers = ACTIONS.keys()
print('Processing {}: {}'.format(bold(botid), info))
if info.startswith(str(red)):
available_opts = [item[0] for item in ACTIONS.values() if item[2]]
available_answers = [k for k, v in ACTIONS.items() if v[2]]
print('Restricted actions.')
else:
# don't display list after 'show' and 'recover' command
if not (answer and isinstance(answer, list) and answer[0] in ['s', 'r']):
content = json.load(handle)
handle.seek(0)
content = OrderedDict(sorted(content.items(), key=lambda t: t[0])) # sort by key here, #1280
meta = load_meta(content)
available_opts = [item[0] for item in ACTIONS.values()]
for count, line in enumerate(meta):
print('{:3}: {} {}'.format(count, *line))
# Determine bot status
try:
bot_status = ctl.bot_status(botid)
if bot_status[1] == 'running':
print(red('This bot is currently running, the dump file is now locked and '
'the bot can\'t write it.'))
except KeyError:
bot_status = 'error'
print(red('Attention: This bot is not defined!'))
available_opts = [item[0] for item in ACTIONS.values() if item[2]]
available_answers = [k for k, v in ACTIONS.items() if v[2]]
print('Restricted actions.')
try:
possible_answers = list(available_answers)
for id_action in ['r', 'a']:
if id_action in possible_answers:
possible_answers[possible_answers.index(id_action)] = id_action + ' '
action_completer = Completer(possible_answers, queues=pipeline_pipes.keys())
readline.set_completer(action_completer.complete)
answer = input(inverted(', '.join(available_opts) + '?') + ' ').split()
except EOFError:
break
else:
if not answer:
continue
if len(answer) == 0 or answer[0] not in available_answers:
print('Action not allowed.')
continue
if any([answer[0] == char for char in AVAILABLE_IDS]) and len(answer) > 1:
ids = [int(item) for item in answer[1].split(',')]
else:
ids = []
queue_name = None
if answer[0] == 'a':
# recover all -> recover all by ids
answer[0] = 'r'
ids = range(len(meta))
if len(answer) > 1:
queue_name = answer[1]
if answer[0] == 'q':
break
elif answer[0] == 'e':
# Delete entries
for entry in ids:
del content[meta[entry][0]]
save_file(handle, content)
elif answer[0] == 'r':
# recover entries
default = utils.load_configuration(DEFAULTS_CONF_FILE)
runtime = utils.load_configuration(RUNTIME_CONF_FILE)
params = utils.load_parameters(default, runtime)
pipe = pipeline.PipelineFactory.create(params, logger)
try:
for i, (key, entry) in enumerate([item for (count, item)
in enumerate(content.items()) if count in ids]):
if entry['message']:
msg = copy.copy(entry['message']) # otherwise the message field gets converted
if isinstance(msg, dict):
msg = json.dumps(msg)
else:
print('No message here, deleting entry.')
del content[key]
continue
if queue_name is None:
if len(answer) == 3:
queue_name = answer[2]
else:
queue_name = entry['source_queue']
if queue_name in pipeline_pipes:
if runtime[pipeline_pipes[queue_name]]['group'] == 'Parser' and json.loads(msg)['__type'] == 'Event':
print('Event converted to Report automatically.')
msg = message.Report(message.MessageFactory.unserialize(msg)).serialize()
try:
pipe.set_queues(queue_name, 'destination')
pipe.connect()
pipe.send(msg)
except exceptions.PipelineError:
print(red('Could not reinject into queue {}: {}'
''.format(queue_name, traceback.format_exc())))
else:
del content[key]
print(green('Recovered dump {}.'.format(i)))
finally:
save_file(handle, content)
if not content:
delete_file = True
print('Deleting empty file {}'.format(fname))
break
elif answer[0] == 'd':
# delete dumpfile
delete_file = True
print('Deleting empty file {}'.format(fname))
break
elif answer[0] == 's':
# Show entries by id
for count, (key, orig_value) in enumerate(content.items()):
value = copy.copy(orig_value) # otherwise the raw field gets truncated
if count not in ids:
continue
print('=' * 100, '\nShowing id {} {}\n'.format(count, key),
'-' * 50)
if isinstance(value['message'], (bytes, str)):
value['message'] = json.loads(value['message'])
if ('raw' in value['message'] and
len(value['message']['raw']) > 1000):
value['message']['raw'] = value['message'][
'raw'][:1000] + '...[truncated]'
if type(value['traceback']) is not list:
value['traceback'] = value['traceback'].splitlines()
pprint.pprint(value)
if delete_file:
os.remove(fname)
def __init__(self, interactive=False, return_type="python", quiet=False):
"""
Initializes intelmqctl.
Parameters
==========
interactive : boolean
for cli-interface true, functions can exits, parameters are used
return_type : string
'python': no special treatment, can be used for use by other
python code
'text': user-friendly output for cli, default for interactive use
'json': machine-readable output for managers
quiet : boolean
False by default, can be activated for cronjobs etc.
"""
global RETURN_TYPE
RETURN_TYPE = return_type
global logger
global QUIET
QUIET = quiet
logger = utils.log('intelmqctl', log_level='DEBUG')
self.logger = logger
self.interactive = interactive
if os.geteuid() == 0:
logger.warning('Running intelmq as root is highly discouraged!')
APPNAME = "intelmqctl"
VERSION = pkg_resources.get_distribution("intelmq").version
DESCRIPTION = """
description: intelmqctl is the tool to control intelmq system.
Outputs are logged to /opt/intelmq/var/log/intelmqctl"""
USAGE = '''
intelmqctl [start|stop|restart|status|reload|run] bot-id
intelmqctl [start|stop|restart|status|reload]
intelmqctl list [bots|queues]
intelmqctl log bot-id [number-of-lines [log-level]]
intelmqctl clear queue-id
Starting a bot:
intelmqctl start bot-id
Stopping a bot:
intelmqctl stop bot-id
Restarting a bot:
intelmqctl restart bot-id
Get status of a bot:
intelmqctl status bot-id
Run a bot directly (blocking) for debugging purpose:
intelmqctl run bot-id
Starting the botnet (all bots):
intelmqctl start
etc.
Get a list of all configured bots:
intelmqctl list bots
Get a list of all queues:
intelmqctl list queues
Clear a queue:
intelmqctl clear queue-id
Get logs of a bot:
intelmqctl log bot-id [number-of-lines [log-level]]
Reads the last lines from bot log, or from system log if no bot ID was
given. Log level should be one of DEBUG, INFO, ERROR or CRITICAL.
Default is INFO. Number of lines defaults to 10, -1 gives all. Result
can be longer due to our logging format!'''
with open(STARTUP_CONF_FILE, 'r') as fp:
self.startup = json.load(fp)
with open(SYSTEM_CONF_FILE, 'r') as fp:
self.system = json.load(fp)
if not os.path.exists(PIDDIR):
os.makedirs(PIDDIR)
# stolen functions from the bot file
# this will not work with various instances of REDIS
self.parameters = Parameters()
self.load_defaults_configuration()
self.load_system_configuration()
self.pipepline_configuration = utils.load_configuration(
PIPELINE_CONF_FILE)
self.runtime_configuration = utils.load_configuration(
RUNTIME_CONF_FILE)
self.startup_configuration = utils.load_configuration(
STARTUP_CONF_FILE)
if self.interactive:
parser = argparse.ArgumentParser(
prog=APPNAME,
usage=USAGE,
epilog=DESCRIPTION
)
parser.add_argument('-v', '--version',
action='version', version=VERSION)
parser.add_argument('--type', '-t', choices=RETURN_TYPES,
default=RETURN_TYPES[0],
help='choose if it should return regular text '
'or other machine-readable')
parser.add_argument('action',
choices=['start', 'stop', 'restart', 'status',
'reload', 'run', 'list', 'clear',
'help', 'log'],
metavar='[start|stop|restart|status|reload|run'
'|list|clear|log]')
parser.add_argument('parameter', nargs='*')
parser.add_argument('--quiet', '-q', action='store_const',
help='Quiet mode, useful for reloads initiated'
'scripts like logrotate',
const=True)
self.parser = parser
self.args = parser.parse_args()
if self.args.action == 'help':
parser.print_help()
exit(0)
RETURN_TYPE = self.args.type
QUIET = self.args.quiet
def prepare_bot(self, parameters={}, destination_queues=None):
"""
Reconfigures the bot with the changed attributes.
Parameters:
parameters: optional bot parameters for this run, as dict
destination_queues: optional definition of destination queues
default: {"_default": "{}-output".format(self.bot_id)}
"""
self.log_stream = io.StringIO()
src_name = "{}-input".format(self.bot_id)
if not destination_queues:
destination_queues = {"_default": "{}-output".format(self.bot_id)}
else:
destination_queues = {
queue_name:
"%s-%s-output" % (self.bot_id, queue_name.strip('_'))
for queue_name in destination_queues
}
config = BOT_CONFIG.copy()
config.update(self.sysconfig)
config.update(parameters)
config['destination_queues'] = destination_queues
self.mocked_config = mocked_config(
self.bot_id,
sysconfig=config,
group=self.bot_type.title(),
module=self.bot_reference.__module__,
)
self.logger = utils.log(self.bot_id,
log_path=False,
stream=self.log_stream,
log_format_stream=utils.LOG_FORMAT,
log_level=config['logging_level'])
self.logger_handlers_backup = self.logger.handlers
parameters = Parameters()
setattr(parameters, 'source_queue', src_name)
setattr(parameters, 'destination_queues', destination_queues)
with mock.patch('intelmq.lib.utils.load_configuration',
new=self.mocked_config):
with mock.patch('intelmq.lib.utils.log',
self.get_mocked_logger(self.logger)):
with mock.patch('intelmq.lib.utils.get_global_settings',
mocked_get_global_settings):
self.bot = self.bot_reference(self.bot_id)
self.bot._Bot__stats_cache = None
pipeline_args = {
key: getattr(self, key)
for key in dir(self)
if not inspect.ismethod(getattr(self, key)) and (key.startswith(
'source_pipeline_') or key.startswith('destination_pipeline'))
}
self.pipe = pipeline.Pythonlist(
logger=self.logger,
pipeline_args=pipeline_args,
load_balance=self.bot.load_balance,
is_multithreaded=self.bot.is_multithreaded)
self.pipe.set_queues(parameters.source_queue, "source")
self.pipe.set_queues(parameters.destination_queues, "destination")
self.prepare_source_queue()
def __init__(self,
interactive: bool = False,
return_type: str = "python",
quiet: bool = False):
"""
Initializes intelmqctl.
Parameters:
interactive: for cli-interface true, functions can exits, parameters are used
return_type: 'python': no special treatment, can be used for use by other
python code
'text': user-friendly output for cli, default for interactive use
'json': machine-readable output for managers
quiet: False by default, can be activated for cron jobs etc.
"""
global RETURN_TYPE
RETURN_TYPE = return_type
global logger
global QUIET
QUIET = quiet
try:
logger = utils.log('intelmqctl', log_level='DEBUG')
except (FileNotFoundError, PermissionError) as exc:
logger = utils.log('intelmqctl', log_level='DEBUG', log_path=False)
logger.error('Not logging to file: %s', exc)
self.logger = logger
self.interactive = interactive
if os.geteuid() == 0:
logger.warning('Running intelmqctl as root is highly discouraged!')
APPNAME = "intelmqctl"
try:
VERSION = pkg_resources.get_distribution("intelmq").version
except pkg_resources.DistributionNotFound: # pragma: no cover
# can only happen in interactive mode
self.logger.error(
'No valid IntelMQ installation found: DistributionNotFound')
exit(1)
DESCRIPTION = """
description: intelmqctl is the tool to control intelmq system.
Outputs are logged to /opt/intelmq/var/log/intelmqctl"""
EPILOG = '''
intelmqctl [start|stop|restart|status|reload] bot-id
intelmqctl [start|stop|restart|status|reload]
intelmqctl list [bots|queues]
intelmqctl log bot-id [number-of-lines [log-level]]
intelmqctl run bot-id message [get|pop|send]
intelmqctl run bot-id process [--msg|--dryrun]
intelmqctl run bot-id console
intelmqctl clear queue-id
intelmqctl check
Starting a bot:
intelmqctl start bot-id
Stopping a bot:
intelmqctl stop bot-id
Restarting a bot:
intelmqctl restart bot-id
Get status of a bot:
intelmqctl status bot-id
Run a bot directly for debugging purpose and temporarily leverage the logging level to DEBUG:
intelmqctl run bot-id
Get a pdb (or ipdb if installed) live console.
intelmqctl run bot-id console
See the message that waits in the input queue.
intelmqctl run bot-id message get
See additional help for further explanation.
intelmqctl run bot-id --help
Starting the botnet (all bots):
intelmqctl start
etc.
Get a list of all configured bots:
intelmqctl list bots
Get a list of all queues:
intelmqctl list queues
If -q is given, only queues with more than one item are listed.
Clear a queue:
intelmqctl clear queue-id
Get logs of a bot:
intelmqctl log bot-id number-of-lines log-level
Reads the last lines from bot log.
Log level should be one of DEBUG, INFO, ERROR or CRITICAL.
Default is INFO. Number of lines defaults to 10, -1 gives all. Result
can be longer due to our logging format!
Outputs are additionally logged to /opt/intelmq/var/log/intelmqctl'''
# stolen functions from the bot file
# this will not work with various instances of REDIS
self.parameters = Parameters()
self.load_defaults_configuration()
try:
self.pipeline_configuration = utils.load_configuration(
PIPELINE_CONF_FILE)
except ValueError as exc: # pragma: no cover
self.abort('Error loading %r: %s' % (PIPELINE_CONF_FILE, exc))
try:
self.runtime_configuration = utils.load_configuration(
RUNTIME_CONF_FILE)
except ValueError as exc: # pragma: no cover
self.abort('Error loading %r: %s' % (RUNTIME_CONF_FILE, exc))
process_manager = getattr(self.parameters, 'process_manager',
'intelmq')
if process_manager not in PROCESS_MANAGER:
self.abort(
'Invalid process manager given: %r, should be one of %r.'
'' % (process_manager, list(PROCESS_MANAGER.keys())))
self.bot_process_manager = PROCESS_MANAGER[process_manager](
self.runtime_configuration, logger, self)
if self.interactive:
parser = argparse.ArgumentParser(
prog=APPNAME,
description=DESCRIPTION,
epilog=EPILOG,
formatter_class=argparse.RawDescriptionHelpFormatter,
)
parser.add_argument('-v',
'--version',
action='version',
version=VERSION)
parser.add_argument('--type',
'-t',
choices=RETURN_TYPES,
default=RETURN_TYPES[0],
help='choose if it should return regular text '
'or other machine-readable')
parser.add_argument(
'--quiet',
'-q',
action='store_const',
help='Quiet mode, useful for reloads initiated '
'scripts like logrotate',
const=True)
subparsers = parser.add_subparsers(title='subcommands')
parser_list = subparsers.add_parser('list',
help='Listing bots or queues')
parser_list.add_argument('kind', choices=['bots', 'queues'])
parser_list.add_argument('--quiet',
'-q',
action='store_const',
help='Only list non-empty queues',
const=True)
parser_list.set_defaults(func=self.list)
subparsers.add_parser('check',
help='Check installation and configuration')
parser_clear = subparsers.add_parser('clear', help='Clear a queue')
parser_clear.add_argument('queue',
help='queue name',
choices=self.get_queues()[3])
parser_clear.set_defaults(func=self.clear_queue)
parser_log = subparsers.add_parser(
'log', help='Get last log lines of a bot')
parser_log.add_argument('bot_id', help='bot id')
parser_log.add_argument('number_of_lines',
help='number of lines',
default=10,
type=int,
nargs='?')
parser_log.add_argument('log_level',
help='logging level',
choices=LOG_LEVEL.keys(),
default='INFO',
nargs='?')
parser_log.set_defaults(func=self.read_bot_log)
parser_run = subparsers.add_parser('run',
help='Run a bot interactively')
parser_run.add_argument('bot_id',
choices=self.runtime_configuration.keys())
parser_run_subparsers = parser_run.add_subparsers(
title='run-subcommands')
parser_run_console = parser_run_subparsers.add_parser(
'console', help='Get a ipdb live console.')
parser_run_console.add_argument(
'console_type',
nargs='?',
help=
'You may specify which console should be run. Default is ipdb (if installed)'
' or pudb (if installed) or pdb but you may want to use another one.'
)
parser_run_console.set_defaults(run_subcommand="console")
parser_run_message = parser_run_subparsers.add_parser(
'message',
help='Debug bot\'s pipelines. Get the message in the'
' input pipeline, pop it (cut it) and display it, or'
' send the message directly to bot\'s output pipeline.')
parser_run_message.add_argument('message_action_kind',
choices=["get", "pop", "send"])
parser_run_message.add_argument(
'msg',
nargs='?',
help='If send was chosen, put here the message in JSON.')
parser_run_message.set_defaults(run_subcommand="message")
parser_run_process = parser_run_subparsers.add_parser(
'process', help='Single run of bot\'s process() method.')
parser_run_process.add_argument(
'--dryrun',
'-d',
action='store_true',
help='Never really pop the message from the input pipeline '
'nor send to output pipeline.')
parser_run_process.add_argument(
'--msg',
'-m',
help='Trick the bot to process this JSON '
'instead of the Message in its pipeline.')
parser_run_process.set_defaults(run_subcommand="process")
parser_run.set_defaults(func=self.bot_run)
parser_check = subparsers.add_parser(
'check', help='Check installation and configuration')
parser_check.set_defaults(func=self.check)
parser_help = subparsers.add_parser('help', help='Show the help')
parser_help.set_defaults(func=parser.print_help)
parser_start = subparsers.add_parser('start',
help='Start a bot or botnet')
parser_start.add_argument(
'bot_id', nargs='?', choices=self.runtime_configuration.keys())
parser_start.set_defaults(func=self.bot_start)
parser_stop = subparsers.add_parser('stop',
help='Stop a bot or botnet')
parser_stop.add_argument('bot_id',
nargs='?',
choices=self.runtime_configuration.keys())
parser_stop.set_defaults(func=self.bot_stop)
parser_restart = subparsers.add_parser(
'restart', help='Restart a bot or botnet')
parser_restart.add_argument(
'bot_id', nargs='?', choices=self.runtime_configuration.keys())
parser_restart.set_defaults(func=self.bot_restart)
parser_reload = subparsers.add_parser(
'reload', help='Reload a bot or botnet')
parser_reload.add_argument(
'bot_id', nargs='?', choices=self.runtime_configuration.keys())
parser_reload.set_defaults(func=self.bot_reload)
parser_status = subparsers.add_parser(
'status', help='Status of a bot or botnet')
parser_status.add_argument(
'bot_id', nargs='?', choices=self.runtime_configuration.keys())
parser_status.set_defaults(func=self.bot_status)
parser_status = subparsers.add_parser('enable',
help='Enable a bot')
parser_status.add_argument(
'bot_id', choices=self.runtime_configuration.keys())
parser_status.set_defaults(func=self.bot_enable)
parser_status = subparsers.add_parser('disable',
help='Disable a bot')
parser_status.add_argument(
'bot_id', choices=self.runtime_configuration.keys())
parser_status.set_defaults(func=self.bot_disable)
self.parser = parser
def setup(self):
self.args = self.parser.parse_args()
if self.args.verbose:
self.verbose = True
if self.args.dry_run:
self.dryrun = True
if self.args.batch:
self.batch = True
if self.args.quiet:
self.quiet = True
self.time_interval = '' .join(self.args.time_interval)
if self.args.feed:
self.additional_where += """ AND "feed.code" = ANY(%s::VARCHAR[]) """
self.additional_params += ('{' + ','.join(self.args.feed) + '}', )
if self.args.skip_feed:
self.additional_where += """ AND "feed.code" != ANY(%s::VARCHAR[]) """
self.additional_params += ('{' + ','.join(self.args.skip_feed) + '}', )
if self.args.asn:
self.additional_where += """ AND "source.asn" = ANY(%s::INT[]) """
self.additional_params += ('{' + ','.join(map(str, self.args.asn)) + '}', )
if self.args.skip_asn:
self.additional_where += """ AND "source.asn" != ANY(%s::INT[]) """
self.additional_params += ('{' + ','.join(map(str, self.args.skip_asn)) + '}', )
if self.args.taxonomy:
self.additional_where += """ AND "classification.taxonomy" = ANY(%s::VARCHAR[]) """
self.additional_params += ('{' + ','.join(self.args.taxonomy) + '}', )
if self.args.skip_taxonomy:
self.additional_where += """ AND "classification.taxonomy" != ANY(%s::VARCHAR[]) """
self.additional_params += ('{' + ','.join(self.args.skip_taxonomy) + '}', )
if self.args.type:
self.additional_where += """ AND "classification.type" = ANY(%s::VARCHAR[]) """
self.additional_params += ('{' + ','.join(self.args.type) + '}', )
if self.args.skip_type:
self.additional_where += """ AND "classification.type" != ANY(%s::VARCHAR[]) """
self.additional_params += ('{' + ','.join(self.args.skip_type) + '}', )
if self.args.identifier:
self.additional_where += """ AND "classification.identifier" = ANY(%s::VARCHAR[]) """
self.additional_params += ('{' + ','.join(self.args.identifier) + '}', )
if self.args.skip_identifier:
self.additional_where += """ AND "classification.identifier" != ANY(%s::VARCHAR[]) """
self.additional_params += ('{' + ','.join(self.args.skip_identifier) + '}', )
if self.args.ton:
self.additional_where += """ AND "time.observation" >= %s """
self.additional_params += (self.args.ton[0], )
with open(INTELMQCLI_CONF_FILE) as conf_handle:
self.config = json.load(conf_handle)
if self.quiet:
stream = None
else:
stream = sys.stderr
self.logger = utils.log('intelmqcli', syslog='/dev/log',
log_level=self.config['log_level'].upper(),
stream=stream, log_format_stream='%(message)s')
self.logger.info('Started %r at %s.',
' '.join(sys.argv),
datetime.datetime.now().isoformat())
self.rt = rt.Rt(self.config['rt']['uri'], self.config['rt']['user'],
self.config['rt']['password'])
def main():
parser = argparse.ArgumentParser(
prog=APPNAME,
formatter_class=argparse.RawDescriptionHelpFormatter,
usage=USAGE,
description=DESCRIPTION,
epilog=EPILOG,
)
parser.add_argument('botid',
metavar='botid',
nargs='?',
default=None,
help='botid to inspect dumps of')
args = parser.parse_args()
# Try to get log_level from defaults_configuration, else use default
try:
log_level = utils.load_configuration(
DEFAULTS_CONF_FILE)['logging_level']
except Exception:
log_level = DEFAULT_LOGGING_LEVEL
try:
logger = utils.log('intelmqdump', log_level=log_level)
except (FileNotFoundError, PermissionError) as exc:
logger = utils.log('intelmqdump', log_level=log_level, log_path=False)
logger.error('Not logging to file: %s', exc)
ctl = intelmqctl.IntelMQController()
readline.parse_and_bind("tab: complete")
readline.set_completer_delims('')
pipeline_config = utils.load_configuration(PIPELINE_CONF_FILE)
pipeline_pipes = {}
for bot, pipes in pipeline_config.items():
pipeline_pipes[pipes.get('source-queue', '')] = bot
if args.botid is None:
filenames = glob.glob(os.path.join(DEFAULT_LOGGING_PATH, '*.dump'))
if not len(filenames):
print(green('Nothing to recover from, no dump files found!'))
sys.exit(0)
filenames = [(fname, fname[len(DEFAULT_LOGGING_PATH):-5])
for fname in sorted(filenames)]
length = max([len(value[1]) for value in filenames])
print(
bold("{c:>3}: {s:{length}} {i}".format(c='id',
s='name (bot id)',
i='content',
length=length)))
for count, (fname, shortname) in enumerate(filenames):
info = dump_info(fname)
print("{c:3}: {s:{length}} {i}".format(c=count,
s=shortname,
i=info,
length=length))
try:
bot_completer = Completer(
possible_values=[f[1] for f in filenames])
readline.set_completer(bot_completer.complete)
botid = input(
inverted('Which dump file to process (id or name)?') + ' ')
except EOFError:
sys.exit(0)
else:
botid = botid.strip()
if botid == 'q' or not botid:
exit(0)
try:
fname, botid = filenames[int(botid)]
except ValueError:
fname = os.path.join(DEFAULT_LOGGING_PATH, botid) + '.dump'
else:
botid = args.botid
fname = os.path.join(DEFAULT_LOGGING_PATH, botid) + '.dump'
if not os.path.isfile(fname):
print(bold('Given file does not exist: {}'.format(fname)))
exit(1)
answer = None
delete_file = False
while True:
with open(fname, 'r+') as handle:
try:
fcntl.flock(handle, fcntl.LOCK_EX | fcntl.LOCK_NB)
except BlockingIOError:
print(red('Dump file is currently locked. Stopping.'))
break
info = dump_info(fname, file_descriptor=handle)
handle.seek(0)
available_answers = ACTIONS.keys()
print('Processing {}: {}'.format(bold(botid), info))
if info.startswith(str(red)):
available_opts = [
item[0] for item in ACTIONS.values() if item[2]
]
available_answers = [k for k, v in ACTIONS.items() if v[2]]
print('Restricted actions.')
else:
# don't display list after 'show', 'recover' & edit commands
if not (answer and isinstance(answer, list)
and answer[0] in ['s', 'r', 'v']):
content = json.load(handle)
handle.seek(0)
content = OrderedDict(
sorted(content.items(),
key=lambda t: t[0])) # sort by key here, #1280
meta = load_meta(content)
available_opts = [item[0] for item in ACTIONS.values()]
for count, line in enumerate(meta):
print('{:3}: {} {}'.format(count, *line))
# Determine bot status
try:
bot_status = ctl.bot_status(botid)
if bot_status[1] == 'running':
print(
red('This bot is currently running, the dump file is now locked and '
'the bot can\'t write it.'))
except KeyError:
bot_status = 'error'
print(red('Attention: This bot is not defined!'))
available_opts = [
item[0] for item in ACTIONS.values() if item[2]
]
available_answers = [k for k, v in ACTIONS.items() if v[2]]
print('Restricted actions.')
try:
possible_answers = list(available_answers)
for id_action in ['r', 'a']:
if id_action in possible_answers:
possible_answers[possible_answers.index(
id_action)] = id_action + ' '
action_completer = Completer(possible_answers,
queues=pipeline_pipes.keys())
readline.set_completer(action_completer.complete)
answer = input(
inverted(', '.join(available_opts) + '?') + ' ').split()
except EOFError:
break
else:
if not answer:
continue
if len(answer) == 0 or answer[0] not in available_answers:
print('Action not allowed.')
continue
if any([answer[0] == char
for char in AVAILABLE_IDS]) and len(answer) > 1:
ids = [int(item) for item in answer[1].split(',')]
else:
ids = []
queue_name = None
if answer[0] == 'a':
# recover all -> recover all by ids
answer[0] = 'r'
ids = range(len(meta))
if len(answer) > 1:
queue_name = answer[1]
if answer[0] == 'q':
break
elif answer[0] == 'e':
# Delete entries
for entry in ids:
del content[meta[entry][0]]
save_file(handle, content)
elif answer[0] == 'r':
# recover entries
default = utils.load_configuration(DEFAULTS_CONF_FILE)
runtime = utils.load_configuration(RUNTIME_CONF_FILE)
params = utils.load_parameters(default, runtime)
pipe = pipeline.PipelineFactory.create(params, logger)
try:
for i, (key, entry) in enumerate([
item
for (count, item) in enumerate(content.items())
if count in ids
]):
if entry['message']:
msg = copy.copy(
entry['message']
) # otherwise the message field gets converted
if isinstance(msg, dict):
msg = json.dumps(msg)
else:
print('No message here, deleting entry.')
del content[key]
continue
if queue_name is None:
if len(answer) == 3:
queue_name = answer[2]
else:
queue_name = entry['source_queue']
if queue_name in pipeline_pipes:
if runtime[pipeline_pipes[queue_name]][
'group'] == 'Parser' and json.loads(
msg)['__type'] == 'Event':
print(
'Event converted to Report automatically.')
msg = message.Report(
message.MessageFactory.unserialize(
msg)).serialize()
try:
pipe.set_queues(queue_name, 'destination')
pipe.connect()
pipe.send(msg)
except exceptions.PipelineError:
print(
red('Could not reinject into queue {}: {}'
''.format(queue_name,
traceback.format_exc())))
else:
del content[key]
print(green('Recovered dump {}.'.format(i)))
finally:
save_file(handle, content)
if not content:
delete_file = True
print('Deleting empty file {}'.format(fname))
break
elif answer[0] == 'd':
# delete dumpfile
delete_file = True
print('Deleting empty file {}'.format(fname))
break
elif answer[0] == 's':
# Show entries by id
for count, (key, orig_value) in enumerate(content.items()):
value = copy.copy(
orig_value) # otherwise the raw field gets truncated
if count not in ids:
continue
print('=' * 100, '\nShowing id {} {}\n'.format(count, key),
'-' * 50)
if value.get('message_type') == 'base64':
if len(value['message']) > 1000:
value['message'] = value[
'message'][:1000] + '...[truncated]'
else:
if isinstance(value['message'], (bytes, str)):
value['message'] = json.loads(value['message'])
if ('raw' in value['message']
and len(value['message']['raw']) > 1000):
value['message']['raw'] = value['message'][
'raw'][:1000] + '...[truncated]'
if type(value['traceback']) is not list:
value['traceback'] = value['traceback'].splitlines()
pprint.pprint(value)
elif answer[0] == 'v':
# edit given id
if not ids:
print(red('Edit mode needs an id'))
continue
for entry in ids:
if content[meta[entry][0]].get('message_type') == 'base64':
with tempfile.NamedTemporaryFile(
mode='w+b', suffix='.txt') as tmphandle:
filename = tmphandle.name
tmphandle.write(
base64.b64decode(
content[meta[entry][0]]['message']))
tmphandle.flush()
proc = subprocess.call(
['sensible-editor', filename])
if proc != 0:
print(red('Calling editor failed.'))
else:
tmphandle.seek(0)
new_content = tmphandle.read()
try:
new_content = new_content.decode()
except UnicodeDecodeError as exc:
print(
red("Could not write the new message because of the following error:"
))
print(
red(
exceptions.DecodingError(
exception=exc)))
else:
del content[meta[entry][0]]['message_type']
content[meta[entry]
[0]]['message'] = new_content
save_file(handle, content)
else:
with tempfile.NamedTemporaryFile(
mode='w+t', suffix='.json') as tmphandle:
filename = tmphandle.name
utils.write_configuration(
configuration_filepath=filename,
content=json.loads(
content[meta[entry][0]]['message']),
new=True,
backup=False)
proc = subprocess.call(
['sensible-editor', filename])
if proc != 0:
print(red('Calling editor failed.'))
else:
tmphandle.seek(0)
content[meta[entry]
[0]]['message'] = tmphandle.read()
save_file(handle, content)
if delete_file:
os.remove(fname)
def prepare_bot(self, parameters={}, destination_queues=None):
"""
Reconfigures the bot with the changed attributes.
Parameters:
parameters: optional bot parameters for this run, as dict
destination_queues: optional definition of destination queues
default: {"_default": "{}-output".format(self.bot_id)}
"""
self.log_stream = io.StringIO()
src_name = "{}-input".format(self.bot_id)
if not destination_queues:
destination_queues = {"_default": "{}-output".format(self.bot_id)}
else:
destination_queues = {queue_name: "%s-%s-output" % (self.bot_id,
queue_name.strip('_'))
for queue_name in destination_queues}
config = self.sysconfig.copy()
config.update(parameters)
self.mocked_config = mocked_config(self.bot_id,
src_name,
destination_queues,
sysconfig=config,
group=self.bot_type.title(),
module=self.bot_reference.__module__,
)
self.resulting_config = BOT_CONFIG.copy()
self.resulting_config.update(self.sysconfig)
self.resulting_config.update(parameters)
self.logger = utils.log(self.bot_id,
log_path=False, stream=self.log_stream,
log_format_stream=utils.LOG_FORMAT,
log_level=self.resulting_config['logging_level'])
self.logger_handlers_backup = self.logger.handlers
parameters = Parameters()
setattr(parameters, 'source_queue', src_name)
setattr(parameters, 'destination_queues', destination_queues)
with mock.patch('intelmq.lib.utils.load_configuration',
new=self.mocked_config):
with mock.patch('intelmq.lib.utils.log', self.get_mocked_logger(self.logger)):
self.bot = self.bot_reference(self.bot_id)
self.bot._Bot__stats_cache = None
self.pipe = pipeline.Pythonlist(parameters, logger=self.logger, bot=self.bot)
self.pipe.set_queues(parameters.source_queue, "source")
self.pipe.set_queues(parameters.destination_queues, "destination")
if self.input_message is not None:
if not isinstance(self.input_message, (list, tuple)):
self.input_message = [self.input_message]
self.input_queue = []
for msg in self.input_message:
if type(msg) is dict:
self.input_queue.append(json.dumps(msg))
elif issubclass(type(msg), message.Message):
self.input_queue.append(msg.serialize())
else:
self.input_queue.append(msg)
self.input_message = None
else:
if self.default_input_message: # None for collectors
self.input_queue = [self.default_input_message]
def __init__(self, interactive: bool=False, return_type: str="python", quiet: bool=False):
"""
Initializes intelmqctl.
Parameters:
interactive: for cli-interface true, functions can exits, parameters are used
return_type: 'python': no special treatment, can be used for use by other
python code
'text': user-friendly output for cli, default for interactive use
'json': machine-readable output for managers
quiet: False by default, can be activated for cronjobs etc.
"""
global RETURN_TYPE
RETURN_TYPE = return_type
global logger
global QUIET
QUIET = quiet
try:
logger = utils.log('intelmqctl', log_level='DEBUG')
except (FileNotFoundError, PermissionError) as exc:
logger = utils.log('intelmqctl', log_level='DEBUG', log_path=False)
logger.error('Not logging to file: %s' % exc)
self.logger = logger
self.interactive = interactive
if os.geteuid() == 0:
logger.warning('Running intelmqctl as root is highly discouraged!')
APPNAME = "intelmqctl"
try:
VERSION = pkg_resources.get_distribution("intelmq").version
except pkg_resources.DistributionNotFound: # pragma: no cover
# can only happen in interactive mode
self.logger.error('No valid IntelMQ installation found: DistributionNotFound')
exit(1)
DESCRIPTION = """
description: intelmqctl is the tool to control intelmq system.
Outputs are logged to /opt/intelmq/var/log/intelmqctl"""
EPILOG = '''
intelmqctl [start|stop|restart|status|reload|run] bot-id
intelmqctl [start|stop|restart|status|reload]
intelmqctl list [bots|queues]
intelmqctl log bot-id [number-of-lines [log-level]]
intelmqctl clear queue-id
intelmqctl check
Starting a bot:
intelmqctl start bot-id
Stopping a bot:
intelmqctl stop bot-id
Restarting a bot:
intelmqctl restart bot-id
Get status of a bot:
intelmqctl status bot-id
Run a bot directly (blocking) for debugging purpose:
intelmqctl run bot-id
Starting the botnet (all bots):
intelmqctl start
etc.
Get a list of all configured bots:
intelmqctl list bots
Get a list of all queues:
intelmqctl list queues
If -q is given, only queues with more than one item are listed.
Clear a queue:
intelmqctl clear queue-id
Get logs of a bot:
intelmqctl log bot-id number-of-lines log-level
Reads the last lines from bot log.
Log level should be one of DEBUG, INFO, ERROR or CRITICAL.
Default is INFO. Number of lines defaults to 10, -1 gives all. Result
can be longer due to our logging format!
Outputs are additionally logged to /opt/intelmq/var/log/intelmqctl'''
# stolen functions from the bot file
# this will not work with various instances of REDIS
self.parameters = Parameters()
self.load_defaults_configuration()
self.load_system_configuration()
try:
self.pipeline_configuration = utils.load_configuration(PIPELINE_CONF_FILE)
except ValueError as exc: # pragma: no cover
msg = 'Error loading %r: %s' % (PIPELINE_CONF_FILE, exc)
if interactive:
exit(msg)
else:
raise ValueError(msg)
try:
self.runtime_configuration = utils.load_configuration(RUNTIME_CONF_FILE)
except ValueError as exc: # pragma: no cover
msg = 'Error loading %r: %s' % (RUNTIME_CONF_FILE, exc)
if interactive:
exit(msg)
else:
raise ValueError(msg)
if os.path.exists(STARTUP_CONF_FILE):
self.logger.warning('Deprecated startup.conf file found, please migrate to runtime.conf soon.')
with open(STARTUP_CONF_FILE, 'r') as fp:
startup = json.load(fp)
for bot_id, bot_values in startup.items():
if 'parameters' in self.runtime_configuration[bot_id]: # pragma: no cover
msg = ('Mixed setup of new runtime.conf and old startup.conf'
' found. Ignoring startup.conf, please fix this!')
if interactive:
exit(msg)
else:
raise ValueError(msg)
params = self.runtime_configuration[bot_id].copy()
self.runtime_configuration[bot_id].clear()
self.runtime_configuration[bot_id]['parameters'] = params
self.runtime_configuration[bot_id].update(bot_values)
try:
with open(RUNTIME_CONF_FILE + '.new', 'w') as fp:
json.dump(self.runtime_configuration, fp, indent=4, sort_keys=True,
separators=(',', ': '))
except PermissionError: # pragma: no cover
self.logger.info('Failed to write new configuration format to %r.'
'' % (RUNTIME_CONF_FILE + '.new'))
else:
self.logger.info('%r with new format written.' % (RUNTIME_CONF_FILE + '.new'))
self.bot_process_manager = BotProcessManager(
self.runtime_configuration,
logger,
)
if self.interactive:
parser = argparse.ArgumentParser(
prog=APPNAME,
description=DESCRIPTION,
epilog=EPILOG,
formatter_class=argparse.RawDescriptionHelpFormatter,
)
parser.add_argument('-v', '--version',
action='version', version=VERSION)
parser.add_argument('--type', '-t', choices=RETURN_TYPES,
default=RETURN_TYPES[0],
help='choose if it should return regular text '
'or other machine-readable')
parser.add_argument('--quiet', '-q', action='store_const',
help='Quiet mode, useful for reloads initiated '
'scripts like logrotate',
const=True)
subparsers = parser.add_subparsers(title='subcommands')
parser_list = subparsers.add_parser('list', help='Listing bots or queues')
parser_list.add_argument('kind', choices=['bots', 'queues'])
parser_list.add_argument('--quiet', '-q', action='store_const',
help='Only list non-empty queues',
const=True)
parser_list.set_defaults(func=self.list)
subparsers.add_parser('check', help='Check installation and configuration')
parser_clear = subparsers.add_parser('clear', help='Clear a queue')
parser_clear.add_argument('queue', help='queue name',
choices=self.get_queues()[3])
parser_clear.set_defaults(func=self.clear_queue)
parser_log = subparsers.add_parser('log', help='Get last log lines of a bot')
parser_log.add_argument('bot_id', help='bot id')
parser_log.add_argument('number_of_lines', help='number of lines',
default=10, type=int, nargs='?')
parser_log.add_argument('log_level', help='logging level',
choices=LOG_LEVEL.keys(), default='INFO', nargs='?')
parser_log.set_defaults(func=self.read_bot_log)
parser_run = subparsers.add_parser('run', help='Run a bot interactively')
parser_run.add_argument('bot_id',
choices=self.runtime_configuration.keys())
parser_run.set_defaults(func=self.bot_run)
parser_check = subparsers.add_parser('check',
help='Check installation and configuration')
parser_check.set_defaults(func=self.check)
parser_help = subparsers.add_parser('help',
help='Show the help')
parser_help.set_defaults(func=parser.print_help)
parser_start = subparsers.add_parser('start', help='Start a bot or botnet')
parser_start.add_argument('bot_id', nargs='?',
choices=self.runtime_configuration.keys())
parser_start.set_defaults(func=self.bot_start)
parser_stop = subparsers.add_parser('stop', help='Stop a bot or botnet')
parser_stop.add_argument('bot_id', nargs='?',
choices=self.runtime_configuration.keys())
parser_stop.set_defaults(func=self.bot_stop)
parser_restart = subparsers.add_parser('restart', help='Restart a bot or botnet')
parser_restart.add_argument('bot_id', nargs='?',
choices=self.runtime_configuration.keys())
parser_restart.set_defaults(func=self.bot_restart)
parser_reload = subparsers.add_parser('reload', help='Reload a bot or botnet')
parser_reload.add_argument('bot_id', nargs='?',
choices=self.runtime_configuration.keys())
parser_reload.set_defaults(func=self.bot_reload)
parser_status = subparsers.add_parser('status', help='Status of a bot or botnet')
parser_status.add_argument('bot_id', nargs='?',
choices=self.runtime_configuration.keys())
parser_status.set_defaults(func=self.bot_status)
self.parser = parser
def __init__(self,
interactive: bool = False,
return_type: str = "python",
quiet: bool = False):
"""
Initializes intelmqctl.
Parameters:
interactive: for cli-interface true, functions can exits, parameters are used
return_type: 'python': no special treatment, can be used for use by other
python code
'text': user-friendly output for cli, default for interactive use
'json': machine-readable output for managers
quiet: False by default, can be activated for cronjobs etc.
"""
global RETURN_TYPE
RETURN_TYPE = return_type
global logger
global QUIET
QUIET = quiet
try:
logger = utils.log('intelmqctl', log_level='DEBUG')
except (FileNotFoundError, PermissionError) as exc:
logger = utils.log('intelmqctl', log_level='DEBUG', log_path=False)
logger.error('Not logging to file: %s' % exc)
self.logger = logger
self.interactive = interactive
if os.geteuid() == 0:
logger.warning('Running intelmqctl as root is highly discouraged!')
APPNAME = "intelmqctl"
try:
VERSION = pkg_resources.get_distribution("intelmq").version
except pkg_resources.DistributionNotFound: # pragma: no cover
# can only happen in interactive mode
self.logger.error(
'No valid IntelMQ installation found: DistributionNotFound')
exit(1)
DESCRIPTION = """
description: intelmqctl is the tool to control intelmq system.
Outputs are logged to /opt/intelmq/var/log/intelmqctl"""
EPILOG = '''
intelmqctl [start|stop|restart|status|reload|run] bot-id
intelmqctl [start|stop|restart|status|reload]
intelmqctl list [bots|queues]
intelmqctl log bot-id [number-of-lines [log-level]]
intelmqctl clear queue-id
intelmqctl check
Starting a bot:
intelmqctl start bot-id
Stopping a bot:
intelmqctl stop bot-id
Restarting a bot:
intelmqctl restart bot-id
Get status of a bot:
intelmqctl status bot-id
Run a bot directly (blocking) for debugging purpose:
intelmqctl run bot-id
Starting the botnet (all bots):
intelmqctl start
etc.
Get a list of all configured bots:
intelmqctl list bots
Get a list of all queues:
intelmqctl list queues
If -q is given, only queues with more than one item are listed.
Clear a queue:
intelmqctl clear queue-id
Get logs of a bot:
intelmqctl log bot-id number-of-lines log-level
Reads the last lines from bot log.
Log level should be one of DEBUG, INFO, ERROR or CRITICAL.
Default is INFO. Number of lines defaults to 10, -1 gives all. Result
can be longer due to our logging format!
Outputs are additionally logged to /opt/intelmq/var/log/intelmqctl'''
# stolen functions from the bot file
# this will not work with various instances of REDIS
self.parameters = Parameters()
self.load_defaults_configuration()
self.load_system_configuration()
try:
self.pipeline_configuration = utils.load_configuration(
PIPELINE_CONF_FILE)
except ValueError as exc: # pragma: no cover
self.abort('Error loading %r: %s' % (PIPELINE_CONF_FILE, exc))
try:
self.runtime_configuration = utils.load_configuration(
RUNTIME_CONF_FILE)
except ValueError as exc: # pragma: no cover
self.abort('Error loading %r: %s' % (RUNTIME_CONF_FILE, exc))
if os.path.exists(STARTUP_CONF_FILE):
self.logger.warning(
'Deprecated startup.conf file found, please migrate to runtime.conf soon.'
)
with open(STARTUP_CONF_FILE, 'r') as fp:
startup = json.load(fp)
for bot_id, bot_values in startup.items():
if 'parameters' in self.runtime_configuration[
bot_id]: # pragma: no cover
self.abort(
'Mixed setup of new runtime.conf and old startup.conf'
' found. Ignoring startup.conf, please fix this!')
params = self.runtime_configuration[bot_id].copy()
self.runtime_configuration[bot_id].clear()
self.runtime_configuration[bot_id]['parameters'] = params
self.runtime_configuration[bot_id].update(bot_values)
if self.write_updated_runtime_config(filename=RUNTIME_CONF_FILE +
'.new'):
self.logger.info('%r with new format written.' %
(RUNTIME_CONF_FILE + '.new'))
process_manager = getattr(self.parameters, 'process_manager',
'intelmq')
if process_manager not in PROCESS_MANAGER:
self.abort(
'Invalid process manager given: %r, should be one of %r.'
'' % (process_manager, list(PROCESS_MANAGER.keys())))
self.bot_process_manager = PROCESS_MANAGER[process_manager](
self.runtime_configuration, logger, self)
if self.interactive:
parser = argparse.ArgumentParser(
prog=APPNAME,
description=DESCRIPTION,
epilog=EPILOG,
formatter_class=argparse.RawDescriptionHelpFormatter,
)
parser.add_argument('-v',
'--version',
action='version',
version=VERSION)
parser.add_argument('--type',
'-t',
choices=RETURN_TYPES,
default=RETURN_TYPES[0],
help='choose if it should return regular text '
'or other machine-readable')
parser.add_argument(
'--quiet',
'-q',
action='store_const',
help='Quiet mode, useful for reloads initiated '
'scripts like logrotate',
const=True)
subparsers = parser.add_subparsers(title='subcommands')
parser_list = subparsers.add_parser('list',
help='Listing bots or queues')
parser_list.add_argument('kind', choices=['bots', 'queues'])
parser_list.add_argument('--quiet',
'-q',
action='store_const',
help='Only list non-empty queues',
const=True)
parser_list.set_defaults(func=self.list)
subparsers.add_parser('check',
help='Check installation and configuration')
parser_clear = subparsers.add_parser('clear', help='Clear a queue')
parser_clear.add_argument('queue',
help='queue name',
choices=self.get_queues()[3])
parser_clear.set_defaults(func=self.clear_queue)
parser_log = subparsers.add_parser(
'log', help='Get last log lines of a bot')
parser_log.add_argument('bot_id', help='bot id')
parser_log.add_argument('number_of_lines',
help='number of lines',
default=10,
type=int,
nargs='?')
parser_log.add_argument('log_level',
help='logging level',
choices=LOG_LEVEL.keys(),
default='INFO',
nargs='?')
parser_log.set_defaults(func=self.read_bot_log)
parser_run = subparsers.add_parser('run',
help='Run a bot interactively')
parser_run.add_argument('bot_id',
choices=self.runtime_configuration.keys())
parser_run.set_defaults(func=self.bot_run)
parser_check = subparsers.add_parser(
'check', help='Check installation and configuration')
parser_check.set_defaults(func=self.check)
parser_help = subparsers.add_parser('help', help='Show the help')
parser_help.set_defaults(func=parser.print_help)
parser_start = subparsers.add_parser('start',
help='Start a bot or botnet')
parser_start.add_argument(
'bot_id', nargs='?', choices=self.runtime_configuration.keys())
parser_start.set_defaults(func=self.bot_start)
parser_stop = subparsers.add_parser('stop',
help='Stop a bot or botnet')
parser_stop.add_argument('bot_id',
nargs='?',
choices=self.runtime_configuration.keys())
parser_stop.set_defaults(func=self.bot_stop)
parser_restart = subparsers.add_parser(
'restart', help='Restart a bot or botnet')
parser_restart.add_argument(
'bot_id', nargs='?', choices=self.runtime_configuration.keys())
parser_restart.set_defaults(func=self.bot_restart)
parser_reload = subparsers.add_parser(
'reload', help='Reload a bot or botnet')
parser_reload.add_argument(
'bot_id', nargs='?', choices=self.runtime_configuration.keys())
parser_reload.set_defaults(func=self.bot_reload)
parser_status = subparsers.add_parser(
'status', help='Status of a bot or botnet')
parser_status.add_argument(
'bot_id', nargs='?', choices=self.runtime_configuration.keys())
parser_status.set_defaults(func=self.bot_status)
parser_status = subparsers.add_parser('enable',
help='Enable a bot')
parser_status.add_argument(
'bot_id', choices=self.runtime_configuration.keys())
parser_status.set_defaults(func=self.bot_enable)
parser_status = subparsers.add_parser('disable',
help='Disable a bot')
parser_status.add_argument(
'bot_id', choices=self.runtime_configuration.keys())
parser_status.set_defaults(func=self.bot_disable)
self.parser = parser
def __init__(self):
global RETURN_TYPE
global logger
logger = utils.log('intelmqctl', log_level='DEBUG')
self.logger = logger
if os.geteuid() == 0:
logger.warning('Running intelmq as root is highly discouraged!')
APPNAME = "intelmqctl"
VERSION = pkg_resources.get_distribution("intelmq").version
DESCRIPTION = """
description: intelmqctl is the tool to control intelmq system.
Outputs are logged to /opt/intelmq/var/log/intelmqctl"""
USAGE = '''
intelmqctl --bot [start|stop|restart|status] --id=cymru-expert
intelmqctl --botnet [start|stop|restart|status]
intelmqctl --list [bots|queues]'''
parser = argparse.ArgumentParser(
prog=APPNAME,
usage=USAGE,
epilog=DESCRIPTION
)
group = parser.add_mutually_exclusive_group()
group_list = group.add_mutually_exclusive_group()
parser.add_argument('-v', '--version',
action='version', version=VERSION)
parser.add_argument('--id', '-i',
dest='bot_id', default=None, help='bot ID')
parser.add_argument('--type', '-t', choices=RETURN_TYPES,
default=RETURN_TYPES[0],
help='choose if it should return regular text or '
'other forms of output')
group_list.add_argument('--log', '-l',
metavar='[log-level]:[number-of-lines]',
default=None,
help='''Reads the last lines from bot log, or
from system log if no bot ID was given.
Log level should be one of DEBUG, INFO, ERROR
or CRTICAL. Default is INFO.
Number of lines defaults to 10, -1 gives all.
Reading from system log is not implemented yet.
''')
group_list.add_argument('--bot', '-b',
choices=['start', 'stop', 'restart', 'status'],
metavar='[start|stop|restart|status]',
default=None)
group_list.add_argument('--botnet', '-n',
choices=['start', 'stop', 'restart', 'status'],
metavar='[start|stop|restart|status]',
default=None)
group_list.add_argument('--list', '-s',
choices=['bots', 'queues'],
metavar='[bots|queues]',
default=None)
group_list.add_argument('--clear', '-c', metavar='queue', default=None,
help='''Clears the given queue in broker''')
self.args = parser.parse_args()
if len(sys.argv) == 1:
parser.print_help()
RETURN_TYPE = self.args.type
with open(STARTUP_CONF_FILE, 'r') as fp:
self.startup = json.load(fp)
with open(SYSTEM_CONF_FILE, 'r') as fp:
self.system = json.load(fp)
if not os.path.exists(PIDDIR):
os.makedirs(PIDDIR)
# stolen functions from the bot file
# this will not work with various instances of REDIS
self.parameters = Parameters()
self.load_defaults_configuration()
self.load_system_configuration()
self.pipepline_configuration = utils.load_configuration(
PIPELINE_CONF_FILE)
self.runtime_configuration = utils.load_configuration(
RUNTIME_CONF_FILE)
self.startup_configuration = utils.load_configuration(
STARTUP_CONF_FILE)
def __init__(self):
global RETURN_TYPE
global logger
global QUIET
logger = utils.log('intelmqctl', log_level='DEBUG')
self.logger = logger
if os.geteuid() == 0:
logger.warning('Running intelmq as root is highly discouraged!')
APPNAME = "intelmqctl"
VERSION = pkg_resources.get_distribution("intelmq").version
DESCRIPTION = """
description: intelmqctl is the tool to control intelmq system.
Outputs are logged to /opt/intelmq/var/log/intelmqctl"""
USAGE = '''
intelmqctl [start|stop|restart|status|reload|run] bot-id
intelmqctl [start|stop|restart|status|reload]
intelmqctl list [bots|queues]
intelmqctl log bot-id [number-of-lines [log-level]]
intelmqctl clear queue-id
Starting a bot:
intelmqctl start bot-id
Stopping a bot:
intelmqctl stop bot-id
Restarting a bot:
intelmqctl restart bot-id
Get status of a bot:
intelmqctl status bot-id
Run a bot directly (blocking) for debugging purpose:
intelmqctl run bot-id
Starting the botnet (all bots):
intelmqctl start
etc.
Get a list of all configured bots:
intelmqctl list bots
Get a list of all queues:
intelmqctl list queues
Clear a queue:
intelmqctl clear queue-id
Get logs of a bot:
intelmqctl log bot-id [number-of-lines [log-level]]
Reads the last lines from bot log, or from system log if no bot ID was
given. Log level should be one of DEBUG, INFO, ERROR or CRITICAL.
Default is INFO. Number of lines defaults to 10, -1 gives all. Result
can be longer due to our logging format!'''
parser = argparse.ArgumentParser(prog=APPNAME,
usage=USAGE,
epilog=DESCRIPTION)
parser.add_argument('-v',
'--version',
action='version',
version=VERSION)
parser.add_argument('--type',
'-t',
choices=RETURN_TYPES,
default=RETURN_TYPES[0],
help='choose if it should return regular text or '
'other machine-readable')
parser.add_argument('action',
choices=[
'start', 'stop', 'restart', 'status', 'reload',
'run', 'list', 'clear', 'help', 'log'
],
metavar='[start|stop|restart|status|reload|run|'
'list|clear|log]')
parser.add_argument('parameter', nargs='*')
parser.add_argument('--quiet',
'-q',
action='store_const',
const=True,
help='Quiet mode, useful for reloads initiated'
'scripts like logrotate')
self.parser = parser
self.args = parser.parse_args()
if self.args.action == 'help':
parser.print_help()
exit(0)
RETURN_TYPE = self.args.type
QUIET = self.args.quiet
with open(STARTUP_CONF_FILE, 'r') as fp:
self.startup = json.load(fp)
with open(SYSTEM_CONF_FILE, 'r') as fp:
self.system = json.load(fp)
if not os.path.exists(PIDDIR):
os.makedirs(PIDDIR)
# stolen functions from the bot file
# this will not work with various instances of REDIS
self.parameters = Parameters()
self.load_defaults_configuration()
self.load_system_configuration()
self.pipepline_configuration = utils.load_configuration(
PIPELINE_CONF_FILE)
self.runtime_configuration = utils.load_configuration(
RUNTIME_CONF_FILE)
self.startup_configuration = utils.load_configuration(
STARTUP_CONF_FILE)
def __init__(self, interactive: bool = False, return_type: str = "python", quiet: bool = False):
"""
Initializes intelmqctl.
Parameters:
interactive: for cli-interface true, functions can exits, parameters are used
return_type: 'python': no special treatment, can be used for use by other
python code
'text': user-friendly output for cli, default for interactive use
'json': machine-readable output for managers
quiet: False by default, can be activated for cron jobs etc.
"""
global RETURN_TYPE
RETURN_TYPE = return_type
global logger
global QUIET
QUIET = quiet
try:
logger = utils.log('intelmqctl', log_level='DEBUG')
except (FileNotFoundError, PermissionError) as exc:
logger = utils.log('intelmqctl', log_level='DEBUG', log_path=False)
logger.error('Not logging to file: %s', exc)
self.logger = logger
self.interactive = interactive
if os.geteuid() == 0:
logger.warning('Running intelmqctl as root is highly discouraged!')
APPNAME = "intelmqctl"
try:
VERSION = pkg_resources.get_distribution("intelmq").version
except pkg_resources.DistributionNotFound: # pragma: no cover
# can only happen in interactive mode
self.logger.error('No valid IntelMQ installation found: DistributionNotFound')
sys.exit(1)
DESCRIPTION = """
description: intelmqctl is the tool to control intelmq system.
Outputs are logged to /opt/intelmq/var/log/intelmqctl"""
EPILOG = '''
intelmqctl [start|stop|restart|status|reload] --group [collectors|parsers|experts|outputs]
intelmqctl [start|stop|restart|status|reload] bot-id
intelmqctl [start|stop|restart|status|reload]
intelmqctl list [bots|queues|queues-and-status]
intelmqctl log bot-id [number-of-lines [log-level]]
intelmqctl run bot-id message [get|pop|send]
intelmqctl run bot-id process [--msg|--dryrun]
intelmqctl run bot-id console
intelmqctl clear queue-id
intelmqctl check
Starting a bot:
intelmqctl start bot-id
Stopping a bot:
intelmqctl stop bot-id
Reloading a bot:
intelmqctl reload bot-id
Restarting a bot:
intelmqctl restart bot-id
Get status of a bot:
intelmqctl status bot-id
Run a bot directly for debugging purpose and temporarily leverage the logging level to DEBUG:
intelmqctl run bot-id
Get a pdb (or ipdb if installed) live console.
intelmqctl run bot-id console
See the message that waits in the input queue.
intelmqctl run bot-id message get
See additional help for further explanation.
intelmqctl run bot-id --help
Starting the botnet (all bots):
intelmqctl start
etc.
Starting a group of bots:
intelmqctl start --group experts
etc.
Get a list of all configured bots:
intelmqctl list bots
If -q is given, only the IDs of enabled bots are listed line by line.
Get a list of all queues:
intelmqctl list queues
If -q is given, only queues with more than one item are listed.
Get a list of all queues and status of the bots:
intelmqctl list queues-and-status
Clear a queue:
intelmqctl clear queue-id
Get logs of a bot:
intelmqctl log bot-id number-of-lines log-level
Reads the last lines from bot log.
Log level should be one of DEBUG, INFO, ERROR or CRITICAL.
Default is INFO. Number of lines defaults to 10, -1 gives all. Result
can be longer due to our logging format!
Outputs are additionally logged to /opt/intelmq/var/log/intelmqctl'''
# stolen functions from the bot file
# this will not work with various instances of REDIS
self.parameters = Parameters()
self.load_defaults_configuration()
try:
self.pipeline_configuration = utils.load_configuration(PIPELINE_CONF_FILE)
except ValueError as exc: # pragma: no cover
self.abort('Error loading %r: %s' % (PIPELINE_CONF_FILE, exc))
try:
self.runtime_configuration = utils.load_configuration(RUNTIME_CONF_FILE)
except ValueError as exc: # pragma: no cover
self.abort('Error loading %r: %s' % (RUNTIME_CONF_FILE, exc))
process_manager = getattr(self.parameters, 'process_manager', 'intelmq')
if process_manager not in PROCESS_MANAGER:
self.abort('Invalid process manager given: %r, should be one of %r.'
'' % (process_manager, list(PROCESS_MANAGER.keys())))
self.bot_process_manager = PROCESS_MANAGER[process_manager](
self.runtime_configuration,
logger,
self
)
if self.interactive:
parser = argparse.ArgumentParser(
prog=APPNAME,
description=DESCRIPTION,
epilog=EPILOG,
formatter_class=argparse.RawDescriptionHelpFormatter,
)
parser.add_argument('-v', '--version',
action='version', version=VERSION)
parser.add_argument('--type', '-t', choices=RETURN_TYPES,
default=RETURN_TYPES[0],
help='choose if it should return regular text '
'or other machine-readable')
parser.add_argument('--quiet', '-q', action='store_const',
help='Quiet mode, useful for reloads initiated '
'scripts like logrotate',
const=True)
subparsers = parser.add_subparsers(title='subcommands')
parser_list = subparsers.add_parser('list', help='Listing bots or queues')
parser_list.add_argument('kind', choices=['bots', 'queues', 'queues-and-status'])
parser_list.add_argument('--quiet', '-q', action='store_const',
help='Only list non-empty queues '
'or the IDs of enabled bots.',
const=True)
parser_list.set_defaults(func=self.list)
parser_clear = subparsers.add_parser('clear', help='Clear a queue')
parser_clear.add_argument('queue', help='queue name',
choices=self.get_queues()[3])
parser_clear.set_defaults(func=self.clear_queue)
parser_log = subparsers.add_parser('log', help='Get last log lines of a bot')
parser_log.add_argument('bot_id', help='bot id')
parser_log.add_argument('number_of_lines', help='number of lines',
default=10, type=int, nargs='?')
parser_log.add_argument('log_level', help='logging level',
choices=LOG_LEVEL.keys(), default='INFO', nargs='?')
parser_log.set_defaults(func=self.read_bot_log)
parser_run = subparsers.add_parser('run', help='Run a bot interactively')
parser_run.add_argument('bot_id',
choices=self.runtime_configuration.keys())
parser_run.add_argument('--loglevel', '-l',
nargs='?', default=None,
choices=LOG_LEVEL.keys())
parser_run_subparsers = parser_run.add_subparsers(title='run-subcommands')
parser_run_console = parser_run_subparsers.add_parser('console', help='Get a ipdb live console.')
parser_run_console.add_argument('console_type', nargs='?',
help='You may specify which console should be run. Default is ipdb (if installed)'
' or pudb (if installed) or pdb but you may want to use another one.')
parser_run_console.set_defaults(run_subcommand="console")
parser_run_message = parser_run_subparsers.add_parser('message',
help='Debug bot\'s pipelines. Get the message in the'
' input pipeline, pop it (cut it) and display it, or'
' send the message directly to bot\'s output pipeline.')
parser_run_message.add_argument('message_action_kind', choices=["get", "pop", "send"])
parser_run_message.add_argument('msg', nargs='?', help='If send was chosen, put here the message in JSON.')
parser_run_message.set_defaults(run_subcommand="message")
parser_run_process = parser_run_subparsers.add_parser('process', help='Single run of bot\'s process() method.')
parser_run_process.add_argument('--show-sent', '-s', action='store_true',
help='If message is sent through, displays it.')
parser_run_process.add_argument('--dryrun', '-d', action='store_true',
help='Never really pop the message from the input pipeline '
'nor send to output pipeline.')
parser_run_process.add_argument('--msg', '-m',
help='Trick the bot to process this JSON '
'instead of the Message in its pipeline.')
parser_run_process.set_defaults(run_subcommand="process")
parser_run.set_defaults(func=self.bot_run)
parser_check = subparsers.add_parser('check',
help='Check installation and configuration')
parser_check.add_argument('--quiet', '-q', action='store_const',
help='Only print warnings and errors.',
const=True)
parser_check.add_argument('--no-connections', '-C', action='store_const',
help='Do not test the connections to services like redis.',
const=True)
parser_check.set_defaults(func=self.check)
parser_help = subparsers.add_parser('help',
help='Show the help')
parser_help.set_defaults(func=parser.print_help)
parser_start = subparsers.add_parser('start', help='Start a bot or botnet')
parser_start.add_argument('bot_id', nargs='?',
choices=self.runtime_configuration.keys())
parser_start.add_argument('--group', help='Start a group of bots',
choices=BOT_GROUP.keys())
parser_start.set_defaults(func=self.bot_start)
parser_stop = subparsers.add_parser('stop', help='Stop a bot or botnet')
parser_stop.add_argument('bot_id', nargs='?',
choices=self.runtime_configuration.keys())
parser_stop.add_argument('--group', help='Stop a group of bots',
choices=BOT_GROUP.keys())
parser_stop.set_defaults(func=self.bot_stop)
parser_restart = subparsers.add_parser('restart', help='Restart a bot or botnet')
parser_restart.add_argument('bot_id', nargs='?',
choices=self.runtime_configuration.keys())
parser_restart.add_argument('--group', help='Restart a group of bots',
choices=BOT_GROUP.keys())
parser_restart.set_defaults(func=self.bot_restart)
parser_reload = subparsers.add_parser('reload', help='Reload a bot or botnet')
parser_reload.add_argument('bot_id', nargs='?',
choices=self.runtime_configuration.keys())
parser_reload.add_argument('--group', help='Reload a group of bots',
choices=BOT_GROUP.keys())
parser_reload.set_defaults(func=self.bot_reload)
parser_status = subparsers.add_parser('status', help='Status of a bot or botnet')
parser_status.add_argument('bot_id', nargs='?',
choices=self.runtime_configuration.keys())
parser_status.add_argument('--group', help='Get status of a group of bots',
choices=BOT_GROUP.keys())
parser_status.set_defaults(func=self.bot_status)
parser_status = subparsers.add_parser('enable', help='Enable a bot')
parser_status.add_argument('bot_id',
choices=self.runtime_configuration.keys())
parser_status.set_defaults(func=self.bot_enable)
parser_status = subparsers.add_parser('disable', help='Disable a bot')
parser_status.add_argument('bot_id',
choices=self.runtime_configuration.keys())
parser_status.set_defaults(func=self.bot_disable)
self.parser = parser
