def __print_issue(self,
issues: GeneralIssueDetail,
full: bool,
count: int = 0,
print_lines: bool = True,
include_params: bool = True) -> None:
bots = 'Default BOTS'
if count > 0:
bots = 'Running Configuration'
if print_lines:
print('----------------------------------------')
if issues.additional_keys:
print(' ' * count + '{} has more keys: {}'.format(
bots, colorize_text('{}'.format(issues.additional_keys),
'Red')))
if issues.missing_keys:
print(' ' * count + '{} is missing keys: {}'.format(
bots, colorize_text('{}'.format(issues.missing_keys),
'Magenta')))
if issues.different_values:
param = 'Parameter(s)'
if count > 0:
param = ' SubParameter(s)'
print(' ' * count + '{}{} with issues:'.format(bots, param))
for different_value in issues.different_values:
self.print_parameter_issue(different_value, full, count + 1,
include_params)
if len(issues.bots_issues) > 0:
for item in issues.bots_issues:
print(' ' * count +
colorize_text('{}'.format(item), 'BackgroundRed'))
if print_lines:
print('----------------------------------------')
print()
def print_bot_full(bot_detail: IntelMQBot, full: bool = False) -> None:
internal_detail = bot_detail
if isinstance(bot_detail, StrangeBot):
internal_detail = internal_detail.bot
if full and internal_detail.installed:
print('{}: {}'.format(
colorize_text('Default Running Config', 'Cyan'),
pretty_json(internal_detail.default_bots_parameters)))
if full:
print('{}: {}'.format(
colorize_text('Default Config', 'Cyan'),
pretty_json(internal_detail.custom_default_parameters)))
len_instances = len(internal_detail.instances)
print('Running Instances {}'.format(
colorize_text('{}'.format(len_instances), 'Magenta')))
if len_instances > 0 and full:
print('Intances: -----------------'.format(len_instances))
counter = 1
for instance in internal_detail.instances:
print('Instance {}: Name: {}'.format(
counter, instance.name))
print('Instance {}: Parameters: {}'.format(
counter, pretty_json(instance.parameters)))
counter = counter + 1
if counter > 1:
print()
def print_parameter_issue(self, issue: Union[ParameterIssueDetail,
ParameterIssue], full: bool,
count: int, include_params: bool) -> None:
base = ' ' * count + 'For Parameter: {}'.format(
colorize_text(issue.parameter_name, 'Yellow'))
if isinstance(issue, ParameterIssueDetail):
print('{} are:'.format(base))
self.__print_issue(issue, full, count + 1, False, include_params)
else:
print('{} has value {} but should be {}'.format(
base, colorize_text(issue.has_value, 'Red'),
colorize_text(issue.should_be, 'Magenta')))
def check_runtime(self, full: bool) -> None:
bot_details = self.get_installed_bots()
issues = self.get_issues(bot_details, 'runtime')
if issues:
for issue in issues:
print('BOT Class: {}'.format(
colorize_text(issue.bot.class_name, 'LightYellow')))
print('BOT ID : {}'.format(
colorize_text(issue.instance.bot_id, 'Yellow')))
for sub_issue in issue.issues:
self.__print_issue(sub_issue, full, 1, True, full)
else:
if len(bot_details) == 0:
print('Not Bots found')
else:
print('No issue found')
def print_bot_strange(self, bot_detail: IntelMQBot) -> None:
internal_detail = bot_detail
if isinstance(bot_detail, StrangeBot):
internal_detail = internal_detail.bot
if internal_detail.strange:
print(colorize_text('Errors:', 'Red'))
if not internal_detail.executable_exists and (
internal_detail.bots_config
or internal_detail.has_running_config):
print('- Executable "{}" does not exist in {}'.format(
internal_detail.module, self.config.bin_folder))
if not internal_detail.default_bots:
if bot_detail.custom:
path = bot_detail.file_path.replace(
basename(bot_detail.file_path), '')
print(
'- BOT has no config.json in {}, or nothing was specified in the class'
.format(path))
else:
path = self.config.base_bots_file
print('- BOT has no default configuration in {}'.format(path))
if internal_detail.executable_exists and not (
internal_detail.bots_config
or internal_detail.has_running_config):
print('- BOT has an executable but is not installed')
def print_bot_meta(bot_detail: IntelMQBot) -> None:
internal_detail = bot_detail
if isinstance(bot_detail, StrangeBot):
internal_detail = internal_detail.bot
type_text = internal_detail.bot_type
if internal_detail.custom:
type_text = '{} (Custom)'.format(type_text)
print('BOT Type: {}'.format(
colorize_text(type_text, 'Gray')))
print('BOT Class: {}'.format(
colorize_text(internal_detail.class_name, 'LightYellow')))
print('Description: {}'.format(
colorize_text(internal_detail.description, 'LightGray')))
print('Module: {}'.format(internal_detail.module))
print('File: {}'.format(internal_detail.file_path))
def handle_bots_issues(self, issues: List[BotIssue],
auto: bool) -> List[IntelMQBot]:
result = list()
for item in issues:
print('Fixing {}:'.format(
colorize_text(item.bot.class_name, 'LightYellow')))
self.__handle_issue(item, False, auto)
result.append(item.bot)
return result
def run(self) -> int:
self.__setup_argument_parser()
args, argv = self.parser.parse_known_args()
self.__set_config(args)
if self.config.is_dev:
print(colorize_text('Tool is in DEV mode', 'Red'))
key = args.command
if key:
try:
if argv:
# there are unknown commands
raise IncorrectArgumentException('Unknown Arguments')
else:
# check if one has root access
try:
file = '/etc/foo'
if os.path.exists(file):
os.remove(file)
else:
os.mknod(file)
except PermissionError:
if not self.config.is_dev:
raise SetupException('You need root permissions to run this tool!')
return self.run_tool(key, args)
except IncorrectArgumentException:
sub_parser = None
for item in getattr(getattr(self.parser, '_subparsers'), '_actions'):
if isinstance(item, getattr(argparse, '_SubParsersAction')):
sub_parser = item
if sub_parser:
sub_parser = sub_parser.choices.get(key, None)
if sub_parser:
if sub_parser.prog.endswith(key):
sub_parser.print_help()
return -1
return -2
raise IntelMQToolException('No Tools are defined')
elif args.details:
print('IntelMQ Version: {}'.format(self.config.version))
print('IntelMQ Installation Directory: {}'.format(self.config.intelmq_folder))
print('Configuration Directory: {}'.format(self.config.config_dir))
print('running BOTS File location: {}'.format(self.config.running_bots_file))
print('default BOTS File location: {}'.format(self.config.base_bots_file))
print('defaults.conf location: {}'.format(self.config.defaults_conf_file))
print('runtime.conf location: {}'.format(self.config.runtime_file))
print('Custom IntelMQ Bots location {}'.format(self.config.custom_bot_folder))
return 0
else:
self.parser.print_help()
return -10
def print_bot_detail(self,
bot_detail: IntelMQBot,
full: bool = False) -> None:
self.print_bot_meta(bot_detail)
strange = bot_detail.strange
if strange:
strange = colorize_text(strange, 'Red')
print('Installed/Strange: {}/{}'.format(bot_detail.installed,
strange))
self.print_bot_full(bot_detail, full)
def handle_runtime_issues(self, issues: List[BotIssue],
auto: bool) -> List[Union[IntelMQBotInstance]]:
result = list()
if issues:
for item in issues:
# Note: the items are instances of bots!
print('Fixing {}:'.format(
colorize_text(item.instance.bot_id, 'LightYellow')))
self.__handle_issue(item, True, auto)
result.append(item.instance)
return result
return result
def check_bots(self, full: bool) -> None:
bot_details = self.get_installed_bots()
issues = self.get_issues(bot_details, 'BOTS')
if issues:
for issue in issues:
print('BOT Class: {}'.format(
colorize_text(issue.bot.class_name, 'LightYellow')))
self.__print_issue(issue.issue, full, full)
else:
if len(bot_details) == 0:
print('Not Bots found')
else:
print('No issue found')
def __fix_parameter_issue(bot: [IntelMQBot,
IntelMQBotInstance], parameter_name: str,
issue: ParameterIssue, auto: bool) -> None:
text = 'Parameter {} with Value: {} was {}.'.format(
colorize_text(parameter_name, 'Red'),
colorize_text(issue.should_be, 'Magenta'),
colorize_text(issue.has_value, 'Gray'))
if auto:
print(text + ' Use Manual Processing for changing.')
else:
if query_yes_no('Do you want to replace ' + text, default='no'):
if isinstance(bot, IntelMQBot):
field = issue.parameter_name
if field in vars(bot.bots_config):
setattr(bot.bots_config, issue.parameter_name,
issue.should_be)
else:
bot.bots_config.parameters[
issue.parameter_name] = issue.should_be
else:
set_value(issue.parameter_name, bot.config,
issue.should_be)
def start(self, args: Namespace) -> int:
if args.auto:
print('{} Parameter values will not be changed!\n'.format(
colorize_text('Note:', 'Red')))
bot_details = self.get_all_bots()
if args.bots:
default_issues = self.get_issues(bot_details, 'BOTS')
if default_issues:
print('Found {} issues in BOTS file\n'.format(
colorize_text('{}'.format(len(default_issues)),
'Magenta')))
print(colorize_text('Fixing BOTS File', 'Underlined'))
fixed_bots = self.handle_bots_issues(default_issues, args.auto)
self.update_bots_file(fixed_bots, 'update')
return 0
else:
print(
'There are not issues! Don\'t fix stuff which is not broken!'
)
return -10
elif args.runtime:
runtime_issues = self.get_issues(bot_details, 'runtime')
print(colorize_text('Fixing runtime.conf File', 'Underlined'))
fixed_bots = self.handle_runtime_issues(runtime_issues, args.auto)
if fixed_bots:
# save runtime.conf
runtime_cfg = dict()
with open(self.config.runtime_file, 'r') as f:
runtime_cfg = json.loads(f.read())
for bot_instance in fixed_bots:
config = runtime_cfg.get(bot_instance.bot_id)
if config:
runtime_cfg[
bot_instance.bot_id] = bot_instance.to_dict(
bot_instance.bot.module)
with open(self.config.runtime_file, 'w') as f:
f.write(pretty_json(runtime_cfg))
return 0
else:
print(
'There are not issues! Don\'t fix stuff which is not broken!'
)
return -10
elif args.strange:
strange_bots = self.get_strange_bots(False)
for strange_bot in strange_bots:
text = '{}'.format(strange_bot.bot.class_name)
if strange_bot.bot.executable_exists and not strange_bot.bot.has_running_config:
print(
'Bot {} is not referenced in BOTS but has an executable.'
.format(text))
if len(strange_bot.bot.instances) > 0:
# not in bots but in instances then it is a strange one but still ok
print(
'However it is referenced in the running.conf hence this works.'
)
else:
if args.auto:
print(
'Removing executable for BOT {}'.format(text))
do_remove = True
else:
do_remove = query_yes_no(
'Do you want to remove executable for BOT {}'.
format(text),
default='no')
if do_remove:
self.manipulate_execution_file(
strange_bot.bot, False)
if not strange_bot.bot.executable_exists and strange_bot.bot.has_running_config:
# referenced in bots but has not an executable
print(
'Bot {} has no default configuration but an executable'
.format(text))
text = '{} to {}'.format(strange_bot.bot.module,
self.config.bin_folder)
if args.auto:
print('Adding executable {}'.format(text))
do_add = True
else:
do_add = query_yes_no(
'Do you want to add {}'.format(text), default='no')
if do_add:
self.manipulate_execution_file(strange_bot.bot, True)
else:
raise IncorrectArgumentException()
def __fix_decider(self, bot: Union[IntelMQBot,
IntelMQBotInstance], parameter_name,
issue: Union[ParameterIssue, ParameterIssueDetail,
GeneralIssueDetail], auto: bool) -> None:
if isinstance(issue, GeneralIssueDetail) or isinstance(
issue, ParameterIssueDetail):
if issue.additional_keys:
for additional_key in issue.additional_keys:
text = 'Key {} from to Parameters {}'.format(
colorize_text(additional_key, 'Yellow'),
colorize_text(parameter_name, 'Yellow'))
if auto:
print('Adding ' + text)
do_add = True
else:
do_add = query_yes_no('Do you want to add ' + text,
default='no')
if do_add:
if isinstance(bot, IntelMQBot):
if parameter_name is None:
bot.bots_config.parameters[
additional_key] = bot.default_bots.parameters.get(
additional_key)
# del bot.bots_config.parameters[additional_key]
else:
del get_value(
parameter_name,
bot.bots_config.parameters)[additional_key]
else:
del bot.config[parameter_name][additional_key]
if issue.missing_keys:
# and set their value
for missing_key in issue.missing_keys:
if parameter_name is None:
check_key = missing_key
else:
check_key = '{}.{}'.format(parameter_name, missing_key)
if isinstance(bot, IntelMQBot):
desired_value = get_value(
check_key, bot.custom_default_parameters)
else:
desired_value = get_value(
check_key, bot.bot.default_bots_parameters)
text = 'Key {} to Parameter {} with default value {}'.format(
colorize_text(missing_key, 'Yellow'),
colorize_text(parameter_name, 'Yellow'),
colorize_text(desired_value, 'Blue'))
if auto:
print('Adding ' + text)
do_add = True
else:
do_add = query_yes_no('Do you want to add ' + text,
default='no')
if do_add:
if isinstance(bot, IntelMQBot):
if parameter_name is None:
bot.default_bots_parameters[
missing_key] = desired_value
else:
get_value(parameter_name,
bot.default_bots_parameters
)[missing_key] = desired_value
else:
bot.config[parameter_name][
missing_key] = desired_value
if issue.different_values:
for different_value in issue.different_values:
self.__fix_decider(bot, different_value.parameter_name,
different_value, auto)
if isinstance(issue, ParameterIssue):
self.__fix_parameter_issue(bot, parameter_name, issue, auto)
def __handle_issue(self, bot_issue: BotIssue, is_runtime: bool,
auto: bool):
internal_instance = bot_issue.bot.bots_config
if is_runtime:
internal_instance = bot_issue.instance
for issue in bot_issue.issues:
if issue.additional_keys:
# basically add keys with their default value
for key in issue.additional_keys:
default_value = bot_issue.bot.default_bots.parameters.get(
key, 'MissingKey')
text = 'Parameter {} with value {} is missing from running configuration'.format(
colorize_text('parameters.{}'.format(key), 'Yellow'),
colorize_text(default_value, 'Blue'))
print(text)
if auto:
do_add = True
print('Automatically added.')
else:
do_add = query_yes_no(
'Do you want to add this parameter?',
default='yes')
if do_add:
internal_instance.parameters[key] = default_value
if issue.missing_keys:
for key in issue.missing_keys:
text = 'Parameter {} with value {} is not present in default configuration'.format(
colorize_text('parameters.{}'.format(key), 'Yellow'),
colorize_text(
internal_instance.parameters.get(
key, 'MissingKey'), 'Blue'))
print(text)
if auto:
do_remove = True
print('Automatically removed.')
else:
do_remove = query_yes_no(
'Do you want to remove this parameter?',
default='yes')
if do_remove:
del internal_instance.parameters[key]
if issue.different_values:
for different_value in issue.different_values:
if different_value.parameter_name in vars(
internal_instance).keys():
is_parameter = False
param_name = different_value.parameter_name
else:
is_parameter = True
param_name = 'parameter.{}'.format(
different_value.parameter_name)
text = 'Parameter {} has value {} but differs from default {}.'.format(
colorize_text(param_name, 'Red'),
colorize_text(different_value.has_value, 'Magenta'),
colorize_text(different_value.should_be, 'Green'))
print(text)
if auto:
do_change = different_value.parameter_name in [
'name', 'description'
]
print(
'Change will not be applied use Manual Processing instead.'
)
else:
do_change = query_yes_no(
'Do you want to apply default value?',
default='no')
if do_change:
if is_parameter:
internal_instance.parameters[
different_value.
parameter_name] = different_value.should_be
else:
setattr(internal_instance,
different_value.parameter_name,
different_value.should_be)
# check if exe needs fixing
self.__fix_executable(bot_issue.bot, auto)