def __create_runtime(self, data: dict) -> Runtime:
output = Runtime()
for bot_id, item_data in data.items():
runtime_item = RuntimeItem()
runtime_item.bot_id = bot_id
for key, value in item_data.items():
if key == 'parameters':
runtime_item.parameters = self.parse_parameters(value)
else:
setattr(runtime_item, key, value)
output.add_item(runtime_item)
return output
def remove_bots_item(self, type_: str, module: str, name: str, bots: BOTS,
runtime: Runtime, bin_folder: str) -> bool:
if bots:
self.logger.debug('Removing BOTS Item "{}" ({})'.format(
name, module))
do_remove = True
runtime_items = runtime.get_runtime_items_for_module(module)
if len(runtime_items) > 0:
# basically the bot is referenced an cannot be removed
do_remove = False
if do_remove:
bots.remove_element(type_, module, name)
# remove also executable
path = join(bin_folder, module)
if isfile(path):
self.logger.debug('Executable "{}" exists'.format(path))
remove(path)
return True
else:
self.logger.error(
'BOTS Item "{}" ({}) cannot be deleted as it is still referenced in a pipe.'
.format(name, module))
return False
else:
self.logger.error('No BOTS provided')
return False
def convert_to_new_runtime(self, pipeline: Pipeline,
runtime: Runtime) -> Runtime:
self.logger.info(
'Converting pipeline.conf and runtime.conf to new setup')
# modify runtime to fit the new pattern
runtime_items_to_remove = list()
for runtime_item in runtime.get_items():
pipeline_item = pipeline.get_item_for(runtime_item.bot_id)
if pipeline_item:
parameter = Parameters()
parameter.add_values({
'destination_queues': {
'_default': pipeline_item.destinations
}
})
runtime_item.parameters.merge_parameters(parameter)
else:
if runtime_item.group in ['Collector']:
message = 'No Pipeline Found fir bot "{}"'.format(
runtime_item.bot_id)
self.logger.error(message)
result = query_yes_no('Do you want to remove this item?')
if result:
runtime_items_to_remove.append(runtime_item)
else:
raise IntelMQParsingException(
'Manual Action required to solve the issue.')
elif runtime_item.group in ['Output']:
self.logger.debug(
'Output Bot "{}" found. Taking no action'.format(
runtime_item.bot_id))
else:
self.logger.critical(
'Non Collector bot found with no Pipeline. Removing "{}"'
.format(runtime_item.bot_id))
runtime_items_to_remove.append(runtime_item)
for item in runtime_items_to_remove:
runtime.remove_by_bot_id(item.bot_id)
return runtime
def remove_runtime_item_by_bot_id(self, bot_id: str, runtime: Runtime,
pipeline: Optional[Pipeline],
bin_folder: str) -> bool:
self.logger.debug('Removing Runtime Item "{}"'.format(bot_id))
do_remove = True
if pipeline:
if pipeline.is_bot_id_contained(bot_id):
do_remove = False
if do_remove:
runtime_item = runtime.get_item_by_id(bot_id)
runtime.remove_by_bot_id(bot_id)
# remove also executable
path = join(bin_folder, runtime_item.module)
if isfile(path):
self.logger.debug('Executable "{}" exists'.format(path))
remove(path)
return True
else:
self.logger.error(
'Bot with ID "{}" cannot be deleted as it is still referenced in a pipe.'
.format(bot_id))
return False
def remove_bot(
self,
bot: IntelMQBot,
bots_conf: BOTS,
runtime: Runtime
) -> int:
self.logger.info('Removing "{}" ({})'.format(bot.name, bot.module))
if not bot.installed:
raise IntelMQToolException('Bot "{}" ({}) is not installed'.format(bot.name, bot.module))
removed = False
runtime_items = runtime.get_runtime_items_for_module(bot.module)
for item in runtime_items:
result = self.intelmq_handler.remove_runtime_item_by_bot_id(item.bot_id)
if result:
self.logger.info('Removed BOT "{}" from runtime'.format(item.bot_id))
removed = True
else:
raise IntelMQToolException(
'Cannot removed BOT "{}" from runtime as it is still referenced'.format(item.bot_id)
)
if removed:
# can occur if the bot is not part of any pipeline
self.output_handler.save_runtime(runtime)
if bots_conf:
self.logger.debug('IntelMQ 2.x uninstallation')
removed = self.intelmq_handler.remove_bots_item(
bot.group, bot.module, bot.name, bots_conf, runtime, self.config.bin_folder
)
if removed:
self.output_handler.save_bots(bots_conf)
self.logger.info('Removed from BOTS')
else:
self.logger.debug('IntelMQ 3.x uninstallation')
# removal for IntelMQ > 3.0
destination = self.output_handler.get_paths(bot, self.config.bot_folder)[1]
rmtree(destination)
# Remove executable
file_name = self.output_handler.get_executable_filename(bot, self.config.bin_folder)
path = join(self.config.bin_folder, file_name)
if isfile(path):
self.logger.debug('Executable "{}" exists'.format(path))
remove(path)
print('BOT "{}" ({}) removed.'.format(bot.name, bot.module))
return 0
def merge_bots_and_runtime(self,
all_bots: List[IntelMQBot],
runtime: Runtime,
bot_location: str,
set_install_by_path: bool = True) -> None:
self.logger.info('Merging bots an runtime')
is_intelmq3 = not is_intelmq_2()
for bot in all_bots:
# refix module if intelmq 3
module = bot.module
if is_intelmq3 and bot.custom:
# TODO: verify if this works!
module = 'intelmq.bots.{}'.format(bot.module)
runtime_items = runtime.get_runtime_items_for_module(module)
bot.runtime_items = runtime_items
if set_install_by_path:
if bot_location in bot.file_path:
bot.installed = True
def get_install_issues(
self, bots: List[IntelMQBot], runtime: Runtime, bin_folder: str,
bot_folder: str,
bots_conf: Optional[BOTS]) -> Optional[IntelMQBotInstallIssue]:
self.logger.debug('Checking reference issues')
# check Runtime
output = IntelMQBotInstallIssue()
for item in runtime.get_items():
found = False
for bot in bots:
# if the module is existing so is the bot
if item.module == bot.module:
found = True
break
if not found:
# there is an issue
issue = ReferenceIssue()
issue.module = item.module
issue.name = item.name
issue.reference = item.bot_id
issue.location = InstallIssueLocations.RUNTIME
output.issues.append(issue)
if bots_conf:
for item in bots_conf.get_items():
found = False
for bot in bots:
# if the module is existing so is the bot
if item.module == bot.module:
found = True
break
if not found:
# there is an issue
issue = ReferenceIssue()
issue.module = item.module
issue.name = item.name
issue.reference = item.type_
issue.location = InstallIssueLocations.BOTS
output.issues.append(issue)
# check bin folder
for file in listdir(bin_folder):
if re.match(r'.+\..+\..+\..+$', file):
found = False
for bot in bots:
# if the module is existing so is the bot
executable_filename = get_executable_filename(
bot, bot_folder)
if file == executable_filename:
found = True
break
if not found:
issue = AvailableExecutableIssue()
issue.path = bin_folder
issue.file_name = file
output.issues.append(issue)
for bot in bots:
if not bot.installed:
issue = NotInstalledIssue()
issue.bot = bot
output.issues.append(issue)
if output.has_issues():
return output
else:
return None
def handle_runtime_issues(self, issues: List[IntelMQRuntimeIssue],
runtime: Runtime, auto: bool) -> None:
self.logger.info('Handling Runtime Issues')
for item in issues:
runtime_item = runtime.get_item_by_id(item.bot_id)
for issue in item.issues:
if isinstance(issue, MismatchIssue):
self.output_handler.print_issue(issue)
do_fix = query_yes_no(
'Do you want to set the value "{}" to key "{}"'.format(
issue.should_value, issue.key))
if do_fix:
print(colorize_text('Fixed', 'Green'))
setattr(runtime_item, issue.key, issue.should_value)
elif isinstance(issue, MissingIssue):
raise NotImplementedError()
elif isinstance(issue, AdditionalIssue):
raise NotImplementedError()
else:
raise IntelMQToolException('Issue is not known. Stopping')
for issue in item.parameter_issues:
if isinstance(issue, MismatchIssue):
self.output_handler.print_issue(issue)
do_fix = query_yes_no(
'Do you want to set the value "{}" to key "{}"'.format(
issue.should_value, issue.key))
if do_fix:
print(colorize_text('Fixed', 'Green'))
runtime_item.parameters.set_value(
issue.key, issue.should_value)
elif isinstance(issue, MissingIssue):
self.output_handler.print_issue(issue)
if auto:
do_fix = True
else:
do_fix = query_yes_no(
'Do you want to add the key "{}" with value "{}"'.
format(issue.key, issue.default_value))
if do_fix:
print(colorize_text('Fixed', 'Green'))
runtime_item.parameters.add_value(
issue.key, issue.default_value)
elif isinstance(issue, AdditionalIssue):
self.output_handler.print_issue(issue)
if auto:
do_fix = True
else:
do_fix = query_yes_no(
'Do you want to add the key "{}" with value "{}"'.
format(issue.key, issue.value))
if do_fix:
print(colorize_text('Fixed', 'Green'))
runtime_item.parameters.add_value(
issue.key, issue.value)
elif isinstance(issue, AbsentIssue):
self.output_handler.print_issue(issue)
if auto:
do_fix = True
else:
do_fix = query_yes_no(
'Do you want to remove the key "{}" with value "{}"'
.format(issue.key, issue.default_value))
if do_fix:
print(colorize_text('Fixed', 'Green'))
runtime_item.parameters.remove_key(issue.key)
else:
raise IntelMQToolException('Issue is not known. Stopping')
def save_runtime(self, runtime: Runtime) -> None:
output = pretty_json(runtime.to_json())
location = runtime.location
with open(location, 'w') as f:
f.write(output)
self.logger.info('Saved Runtime to {}'.format(location))