def tearDown(self):
self.aminer_config = AMinerConfig.load_config(self.__configFilePath)
persistence_file_name = AMinerConfig.build_persistence_file_name(
self.aminer_config)
if os.path.exists(persistence_file_name):
shutil.rmtree(persistence_file_name)
if not os.path.exists(persistence_file_name):
os.makedirs(persistence_file_name)
def setUp(self):
self.aminer_config = AMinerConfig.load_config(self.__configFilePath)
self.analysis_context = AnalysisContext(self.aminer_config)
self.output_stream = StringIO()
self.stream_printer_event_handler = StreamPrinterEventHandler(
self.analysis_context, self.output_stream)
persistence_file_name = AMinerConfig.build_persistence_file_name(
self.aminer_config)
if os.path.exists(persistence_file_name):
shutil.rmtree(persistence_file_name)
if not os.path.exists(persistence_file_name):
os.makedirs(persistence_file_name)
def main():
"""Run the aminer main program."""
# Extract program name, but only when sure to contain no problematic characters.
warnings.filterwarnings('ignore', category=ImportWarning)
program_name = sys.argv[0].split('/')[-1]
if (program_name == '.') or (program_name == '..') or (re.match(
'^[a-zA-Z0-9._-]+$', program_name) is None):
print('Invalid program name, check your execution args',
file=sys.stderr)
sys.exit(1)
# We will not read stdin from here on, so get rid of it immediately, thus aberrant child cannot manipulate caller's stdin using it.
stdin_fd = os.open('/dev/null', os.O_RDONLY)
os.dup2(stdin_fd, 0)
os.close(stdin_fd)
help_message = 'aminer - logdata-anomaly-miner\n'
if supports_color():
help_message += colflame
else:
help_message += flame
parser = argparse.ArgumentParser(
description=help_message,
formatter_class=argparse.RawTextHelpFormatter)
parser.add_argument('-v',
'--version',
action='version',
version=__version_string__)
parser.add_argument('-c',
'--config',
default='/etc/aminer/config.yml',
type=str,
help='path to the config-file')
parser.add_argument('-D',
'--daemon',
action='store_false',
help='run as a daemon process')
parser.add_argument(
'-s',
'--stat',
choices=[0, 1, 2],
type=int,
help='set the stat level. Possible stat-levels are 0 for no statistics'
', 1 for normal statistic level and 2 for verbose statistics.')
parser.add_argument(
'-d',
'--debug',
choices=[0, 1, 2],
type=int,
help='set the debug level. Possible debug-levels are 0 for no '
'debugging, 1 for normal output (INFO and above), 2 for printing'
' all debug information.')
parser.add_argument('--run-analysis',
action='store_true',
help='enable/disable analysis')
parser.add_argument('-C',
'--clear',
action='store_true',
help='removes all persistence directories')
parser.add_argument('-r',
'--remove',
action='append',
type=str,
help='removes a specific persistence directory')
parser.add_argument('-R',
'--restore',
type=str,
help='restore a persistence backup')
parser.add_argument(
'-f',
'--from-begin',
action='store_true',
help='removes RepositioningData before starting the aminer')
args = parser.parse_args()
config_file_name = args.config
run_in_foreground_flag = args.daemon
run_analysis_child_flag = args.run_analysis
clear_persistence_flag = args.clear
remove_persistence_dirs = args.remove
from_begin_flag = args.from_begin
if args.restore is not None and ('.' in args.restore
or '/' in args.restore):
parser.error('The restore path %s must not contain any . or /' %
args.restore)
if args.remove is not None:
for remove in args.remove:
if '.' in remove or '/' in remove:
parser.error('The remove path %s must not contain any . or /' %
remove)
restore_relative_persistence_path = args.restore
stat_level = 1
debug_level = 1
stat_level_console_flag = False
debug_level_console_flag = False
if args.stat is not None:
stat_level = args.stat
stat_level_console_flag = True
if args.debug is not None:
debug_level = args.stat
debug_level_console_flag = True
# Load the main configuration file.
if not os.path.exists(config_file_name):
print('%s: config "%s" not (yet) available!' %
(program_name, config_file_name),
file=sys.stderr)
sys.exit(1)
# Minimal import to avoid loading too much within the privileged process.
try:
aminer_config = AMinerConfig.load_config(config_file_name)
except ValueError as e:
print("Config-Error: %s" % e)
sys.exit(1)
persistence_dir = aminer_config.config_properties.get(
AMinerConfig.KEY_PERSISTENCE_DIR, AMinerConfig.DEFAULT_PERSISTENCE_DIR)
child_user_name = aminer_config.config_properties.get(
AMinerConfig.KEY_AMINER_USER)
child_group_name = aminer_config.config_properties.get(
AMinerConfig.KEY_AMINER_GROUP)
child_user_id = -1
child_group_id = -1
try:
if child_user_name is not None:
from pwd import getpwnam
child_user_id = getpwnam(child_user_name).pw_uid
if child_group_name is not None:
from grp import getgrnam
child_group_id = getgrnam(child_user_name).gr_gid
except: # skipcq: FLK-E722
print('Failed to resolve %s or %s' %
(AMinerConfig.KEY_AMINER_USER, AMinerConfig.KEY_AMINER_GROUP),
file=sys.stderr)
sys.exit(1)
initialize_loggers(aminer_config, child_user_name, child_group_name)
if restore_relative_persistence_path is not None and (
clear_persistence_flag or remove_persistence_dirs):
msg = 'The --restore parameter removes all persistence files. Do not use this parameter with --Clear or --Remove!'
print(msg, sys.stderr)
logging.getLogger(AMinerConfig.DEBUG_LOG_NAME).error(msg)
sys.exit(1)
if not stat_level_console_flag and AMinerConfig.KEY_LOG_STAT_LEVEL in aminer_config.config_properties:
stat_level = aminer_config.config_properties[
AMinerConfig.KEY_LOG_STAT_LEVEL]
if not debug_level_console_flag and AMinerConfig.KEY_LOG_DEBUG_LEVEL in aminer_config.config_properties:
debug_level = aminer_config.config_properties[
AMinerConfig.KEY_LOG_DEBUG_LEVEL]
AMinerConfig.STAT_LEVEL = stat_level
AMinerConfig.DEBUG_LEVEL = debug_level
if clear_persistence_flag:
if remove_persistence_dirs:
msg = 'The --clear and --remove arguments must not be used together!'
print(msg, file=sys.stderr)
logging.getLogger(AMinerConfig.DEBUG_LOG_NAME).error(msg)
sys.exit(1)
clear_persistence(persistence_dir)
if remove_persistence_dirs:
persistence_dir_name = aminer_config.config_properties.get(
AMinerConfig.KEY_PERSISTENCE_DIR,
AMinerConfig.DEFAULT_PERSISTENCE_DIR)
for filename in os.listdir(persistence_dir_name):
file_path = os.path.join(persistence_dir_name, filename)
try:
if not os.path.isdir(file_path):
msg = 'The aminer persistence directory should not contain any files.'
print(msg, file=sys.stderr)
logging.getLogger(AMinerConfig.DEBUG_LOG_NAME).warning(msg)
continue
shutil.rmtree(file_path)
except OSError as e:
msg = 'Failed to delete %s. Reason: %s' % (file_path, e)
print(msg, file=sys.stderr)
logging.getLogger(AMinerConfig.DEBUG_LOG_NAME).error(msg)
for filename in remove_persistence_dirs:
file_path = os.path.join(persistence_dir, filename)
try:
if not os.path.exists(file_path):
continue
if not os.path.isdir(file_path):
msg = 'The aminer persistence directory should not contain any files.'
print(msg, file=sys.stderr)
logging.getLogger(AMinerConfig.DEBUG_LOG_NAME).warning(msg)
continue
shutil.rmtree(file_path)
except OSError as e:
msg = 'Failed to delete %s. Reason: %s' % (file_path, e)
print(msg, file=sys.stderr)
logging.getLogger(AMinerConfig.DEBUG_LOG_NAME).error(msg)
if restore_relative_persistence_path is not None:
absolute_persistence_path = os.path.join(
persistence_dir, 'backup', restore_relative_persistence_path)
if not os.path.exists(absolute_persistence_path):
msg = '%s does not exist. Continuing without restoring persistence.' % absolute_persistence_path
print(msg, file=sys.stderr)
logging.getLogger(AMinerConfig.DEBUG_LOG_NAME).warning(msg)
else:
clear_persistence(persistence_dir)
copytree(absolute_persistence_path, persistence_dir)
aminer_user = aminer_config.config_properties.get(
AMinerConfig.KEY_AMINER_USER)
aminer_grp = aminer_config.config_properties.get(
AMinerConfig.KEY_AMINER_GROUP)
for dirpath, _dirnames, filenames in os.walk(persistence_dir):
shutil.chown(dirpath, aminer_user, aminer_grp)
for filename in filenames:
shutil.chown(os.path.join(dirpath, filename), aminer_user,
aminer_grp)
if from_begin_flag:
repositioning_data_path = os.path.join(
aminer_config.config_properties.get(
AMinerConfig.KEY_PERSISTENCE_DIR,
AMinerConfig.DEFAULT_PERSISTENCE_DIR), 'AnalysisChild',
'RepositioningData')
if os.path.exists(repositioning_data_path):
os.remove(repositioning_data_path)
if run_analysis_child_flag:
# Call analysis process, this function will never return.
run_analysis_child(aminer_config, program_name)
# Start importing of aminer specific components after reading of "config.py" to allow replacement of components via sys.path
# from within configuration.
from aminer.util import SecureOSFunctions
from aminer.util import decode_string_as_byte_string
log_sources_list = aminer_config.config_properties.get(
AMinerConfig.KEY_LOG_SOURCES_LIST)
if (log_sources_list is None) or not log_sources_list:
msg = '%s: %s not defined' % (program_name,
AMinerConfig.KEY_LOG_SOURCES_LIST)
print(msg, file=sys.stderr)
logging.getLogger(AMinerConfig.DEBUG_LOG_NAME).error(msg)
sys.exit(1)
# Now create the management entries for each logfile.
log_data_resource_dict = {}
for log_resource_name in log_sources_list:
# From here on log_resource_name is a byte array.
log_resource_name = decode_string_as_byte_string(log_resource_name)
log_resource = None
if log_resource_name.startswith(b'file://'):
from aminer.input.LogStream import FileLogDataResource
log_resource = FileLogDataResource(log_resource_name, -1)
elif log_resource_name.startswith(b'unix://'):
from aminer.input.LogStream import UnixSocketLogDataResource
log_resource = UnixSocketLogDataResource(log_resource_name, -1)
else:
msg = 'Unsupported schema in %s: %s' % (
AMinerConfig.KEY_LOG_SOURCES_LIST, repr(log_resource_name))
print(msg, file=sys.stderr)
logging.getLogger(AMinerConfig.DEBUG_LOG_NAME).error(msg)
sys.exit(1)
if not os.path.exists(log_resource_name[7:].decode()):
msg = "WARNING: file or socket '%s' does not exist (yet)!" % log_resource_name[
7:].decode()
print(msg, file=sys.stderr)
logging.getLogger(AMinerConfig.DEBUG_LOG_NAME).warning(msg)
try:
log_resource.open()
except OSError as open_os_error:
if open_os_error.errno == errno.EACCES:
msg = '%s: no permission to access %s' % (
program_name, repr(log_resource_name))
print(msg, file=sys.stderr)
logging.getLogger(AMinerConfig.DEBUG_LOG_NAME).error(msg)
sys.exit(1)
else:
msg = '%s: unexpected error opening %s: %d (%s)' % (
program_name, repr(log_resource_name), open_os_error.errno,
os.strerror(open_os_error.errno))
print(msg, file=sys.stderr)
logging.getLogger(AMinerConfig.DEBUG_LOG_NAME).error(msg)
sys.exit(1)
log_data_resource_dict[log_resource_name] = log_resource
# Create the remote control socket, if any. Do this in privileged mode to allow binding it at arbitrary locations and support restricted
# permissions of any type for current (privileged) uid.
remote_control_socket_name = aminer_config.config_properties.get(
AMinerConfig.KEY_REMOTE_CONTROL_SOCKET_PATH, None)
remote_control_socket = None
if remote_control_socket_name is not None:
if os.path.exists(remote_control_socket_name):
try:
os.unlink(remote_control_socket_name)
except OSError:
msg = 'Failed to clean up old remote control socket at %s' % remote_control_socket_name
print(msg, file=sys.stderr)
logging.getLogger(AMinerConfig.DEBUG_LOG_NAME).error(msg)
sys.exit(1)
# Create the local socket: there is no easy way to create it with correct permissions, hence a fork is needed, setting umask,
# bind the socket. It is also recommended to create the socket in a directory having the correct permissions already.
remote_control_socket = socket.socket(socket.AF_UNIX,
socket.SOCK_STREAM)
remote_control_socket.setblocking(False)
bind_child_pid = os.fork()
if bind_child_pid == 0:
os.umask(0o177)
remote_control_socket.bind(remote_control_socket_name)
# Do not perform any cleanup, flushing of streams. Use _exit(0) to avoid interference with fork.
os._exit(0) # skipcq: PYL-W0212
os.waitpid(bind_child_pid, 0)
remote_control_socket.listen(4)
# Now have checked all we can get from the configuration in the privileged process. Detach from the TTY when in daemon mode.
if not run_in_foreground_flag:
child_pid = 0
try:
# Fork a child to make sure, we are not the process group leader already.
child_pid = os.fork()
except Exception as fork_exception: # skipcq: PYL-W0703
msg = 'Failed to daemonize: %s' % fork_exception
print(msg, file=sys.stderr)
logging.getLogger(AMinerConfig.DEBUG_LOG_NAME).error(msg)
sys.exit(1)
if child_pid != 0:
# This is the parent.
os._exit(0) # skipcq: PYL-W0212
# This is the child. Create a new session and become process group leader. Here we get rid of the controlling tty.
os.setsid()
# Fork again to become an orphaned process not being session leader, hence not able to get a controlling tty again.
try:
child_pid = os.fork()
except Exception as fork_exception: # skipcq: PYL-W0703
msg = 'Failed to daemonize: %s' % fork_exception
print(msg, file=sys.stderr)
logging.getLogger(AMinerConfig.DEBUG_LOG_NAME).error(msg)
sys.exit(1)
if child_pid != 0:
# This is the parent.
os._exit(0) # skipcq: PYL-W0212
# Move to root directory to avoid lingering in some cwd someone else might want to unmount.
os.chdir('/')
# Change the umask here to clean all group/other mask bits so that accidentially created files are not accessible by other.
os.umask(0o77)
# Install a signal handler catching common stop signals and relaying it to all children for sure.
# skipcq: PYL-W0603
global child_termination_triggered_flag
child_termination_triggered_flag = False
def graceful_shutdown_handler(_signo, _stackFrame):
"""React on typical shutdown signals."""
msg = '%s: caught signal, shutting down' % program_name
print(msg, file=sys.stderr)
logging.getLogger(AMinerConfig.DEBUG_LOG_NAME).info(msg)
# Just set the flag. It is likely, that child received same signal also so avoid multiple signaling, which could interrupt the
# shutdown procedure again.
# skipcq: PYL-W0603
global child_termination_triggered_flag
child_termination_triggered_flag = True
import signal
signal.signal(signal.SIGHUP, graceful_shutdown_handler)
signal.signal(signal.SIGINT, graceful_shutdown_handler)
signal.signal(signal.SIGTERM, graceful_shutdown_handler)
# Now create the socket to connect the analysis child.
(parent_socket, child_socket) = socket.socketpair(socket.AF_UNIX,
socket.SOCK_DGRAM, 0)
# Have it nonblocking from here on.
parent_socket.setblocking(False)
child_socket.setblocking(False)
# Use normal fork, we should have been detached from TTY already. Flush stderr to avoid duplication of output if both child and
# parent want to write something.
sys.stderr.flush()
child_pid = os.fork()
if child_pid == 0:
# Relocate the child socket fd to 3 if needed
if child_socket.fileno() != 3:
os.dup2(child_socket.fileno(), 3)
child_socket.close()
# Clear the supplementary groups before dropping privileges. This makes only sense when changing the uid or gid.
if os.getuid() == 0:
if ((child_user_id != -1) and (child_user_id != os.getuid())) or (
(child_group_id != -1) and (child_group_id != os.getgid())):
os.setgroups([])
# Drop privileges before executing child. setuid/gid will raise an exception when call has failed.
if child_group_id != -1:
os.setgid(child_group_id)
if child_user_id != -1:
os.setuid(child_user_id)
else:
msg = 'INFO: No privilege separation when started as unprivileged user'
print(msg, file=sys.stderr)
initialize_loggers(aminer_config, 'aminer', 'aminer')
logging.getLogger(AMinerConfig.DEBUG_LOG_NAME).info(msg)
# Now resolve the specific analysis configuration file (if any).
analysis_config_file_name = aminer_config.config_properties.get(
AMinerConfig.KEY_ANALYSIS_CONFIG_FILE, None)
if analysis_config_file_name is None:
analysis_config_file_name = config_file_name
elif not os.path.isabs(analysis_config_file_name):
analysis_config_file_name = os.path.join(
os.path.dirname(config_file_name), analysis_config_file_name)
# This is the child. Close all parent file descriptors, we do not need. Perhaps this could be done more elegantly.
for close_fd in range(4, 1 << 16):
try:
os.close(close_fd)
except OSError as open_os_error:
if open_os_error.errno == errno.EBADF:
continue
msg = '%s: unexpected exception closing file descriptors: %s' % (
program_name, open_os_error)
print(msg, file=sys.stderr)
logging.getLogger(AMinerConfig.DEBUG_LOG_NAME).error(msg)
# Flush stderr before exit without any cleanup.
sys.stderr.flush()
os._exit(1) # skipcq: PYL-W0212
# Now execute the very same program again, but user might have moved or renamed it meanwhile. This would be problematic with
# SUID-binaries (which we do not yet support). Do NOT just fork but also exec to avoid child circumventing
# parent's ALSR due to cloned kernel VMA.
execArgs = [
'AMinerChild', '--run-analysis', '--config',
analysis_config_file_name, '--stat',
str(stat_level), '--debug',
str(debug_level)
]
os.execve(sys.argv[0], execArgs, {}) # skipcq: BAN-B606
msg = 'Failed to execute child process'
print(msg, file=sys.stderr)
logging.getLogger(AMinerConfig.DEBUG_LOG_NAME).error(msg)
sys.stderr.flush()
os._exit(1) # skipcq: PYL-W0212
child_socket.close()
# Send all log resource information currently available to child process.
for log_resource_name, log_resource in log_data_resource_dict.items():
if (log_resource
is not None) and (log_resource.get_file_descriptor() >= 0):
SecureOSFunctions.send_logstream_descriptor(
parent_socket, log_resource.get_file_descriptor(),
log_resource_name)
log_resource.close()
# Send the remote control server socket, if any and close it afterwards. It is not needed any more on parent side.
if remote_control_socket is not None:
SecureOSFunctions.send_annotated_file_descriptor(
parent_socket, remote_control_socket.fileno(), 'remotecontrol', '')
remote_control_socket.close()
exit_status = 0
child_termination_triggered_count = 0
while True:
if child_termination_triggered_flag:
if child_termination_triggered_count == 0:
time.sleep(1)
elif child_termination_triggered_count < 5:
os.kill(child_pid, signal.SIGTERM)
else:
os.kill(0, signal.SIGKILL)
child_termination_triggered_count += 1
(sig_child_pid, sig_status) = os.waitpid(-1, os.WNOHANG)
if sig_child_pid != 0:
if sig_child_pid == child_pid:
if child_termination_triggered_flag:
# This was expected, just terminate.
break
msg = '%s: Analysis child process %d terminated unexpectedly with signal 0x%x' % (
program_name, sig_child_pid, sig_status)
print(msg, file=sys.stderr)
logging.getLogger(AMinerConfig.DEBUG_LOG_NAME).error(msg)
exit_status = 1
break
# So the child has been cloned, the clone has terminated. This should not happen either.
msg = '%s: untracked child %d terminated with with signal 0x%x' % (
program_name, sig_child_pid, sig_status)
print(msg, file=sys.stderr)
logging.getLogger(AMinerConfig.DEBUG_LOG_NAME).error(msg)
exit_status = 1
# Child information handled, scan for rotated logfiles or other resources, where reopening might make sense.
for log_resouce_name, log_data_resource in log_data_resource_dict.items(
):
try:
if not log_data_resource.open(reopen_flag=True):
continue
except OSError as open_os_error:
if open_os_error.errno == errno.EACCES:
msg = '%s: no permission to access %s' % (program_name,
log_resouce_name)
print(msg, file=sys.stderr)
logging.getLogger(AMinerConfig.DEBUG_LOG_NAME).error(msg)
else:
msg = '%s: unexpected error reopening %s: %d (%s)' % (
program_name, log_resouce_name, open_os_error.errno,
os.strerror(open_os_error.errno))
print(msg, file=sys.stderr)
logging.getLogger(AMinerConfig.DEBUG_LOG_NAME).error(msg)
exit_status = 2
continue
SecureOSFunctions.send_logstream_descriptor(
parent_socket, log_data_resource.get_file_descriptor(),
log_resouce_name)
log_data_resource.close()
time.sleep(1)
parent_socket.close()
SecureOSFunctions.close_base_directory()
sys.exit(exit_status)
def test17_demo_yaml_config_equals_python_config(self):
"""This test checks if the yaml demo config is the same as the python version."""
spec = importlib.util.spec_from_file_location(
'aminer_config',
'/usr/lib/logdata-anomaly-miner/aminer/YamlConfig.py')
aminer_config = importlib.util.module_from_spec(spec)
spec.loader.exec_module(aminer_config)
aminer_config.load_yaml('demo/AMiner/demo-config.yml')
yml_context = AnalysisContext(aminer_config)
yml_context.build_analysis_pipeline()
aminer_config = AMinerConfig.load_config('demo/AMiner/demo-config.py')
py_context = AnalysisContext(aminer_config)
py_context.build_analysis_pipeline()
import copy
yml_config_properties = copy.deepcopy(
yml_context.aminer_config.config_properties)
del yml_config_properties['Parser']
del yml_config_properties['Input']
del yml_config_properties['Analysis']
del yml_config_properties['EventHandlers']
del yml_config_properties['LearnMode']
del yml_config_properties['SuppressNewMatchPathDetector']
# remove SimpleUnparsedAtomHandler, VerboseUnparsedAtomHandler and NewMatchPathDetector as they are added by the YamlConfig.
py_registered_components = copy.copy(py_context.registered_components)
del py_registered_components[0]
del py_registered_components[1]
del py_registered_components[2]
del py_registered_components[10]
yml_registered_components = copy.copy(
yml_context.registered_components)
del yml_registered_components[0]
del yml_registered_components[1]
tmp = {}
keys = list(py_registered_components.keys())
for i in range(1, len(py_registered_components) + 1):
tmp[i] = py_registered_components[keys[i - 1]]
py_registered_components = tmp
py_registered_components_by_name = copy.copy(
py_context.registered_components_by_name)
del py_registered_components_by_name['SimpleUnparsedHandler']
del py_registered_components_by_name['VerboseUnparsedHandler']
del py_registered_components_by_name['NewMatchPath']
del py_registered_components_by_name['SimpleMonotonicTimestampAdjust']
yml_registered_components_by_name = copy.copy(
yml_context.registered_components_by_name)
del yml_registered_components_by_name['DefaultNewMatchPathDetector']
del yml_registered_components_by_name['AtomFilter']
self.assertEqual(yml_config_properties,
py_context.aminer_config.config_properties)
# there actually is no easy way to compare AMiner components as they do not implement the __eq__ method.
self.assertEqual(len(yml_registered_components),
len(py_registered_components))
for i in range(2, len(yml_registered_components)):
self.assertEqual(type(yml_registered_components[i]),
type(py_registered_components[i]))
self.assertEqual(yml_registered_components_by_name.keys(),
py_registered_components_by_name.keys())
for name in yml_registered_components_by_name.keys():
self.assertEqual(type(yml_registered_components_by_name[name]),
type(py_registered_components_by_name[name]))
self.assertEqual(len(yml_context.real_time_triggered_components),
len(py_context.real_time_triggered_components))
# the atom_handler_list is not equal as the python version uses a SimpleMonotonicTimestampAdjust.
self.assertEqual(yml_context.atomizer_factory.default_timestamp_paths,
py_context.atomizer_factory.default_timestamp_paths)
self.assertEqual(type(yml_context.atomizer_factory.event_handler_list),
type(py_context.atomizer_factory.event_handler_list))
