def tearDown(self): self.aminer_config = AMinerConfig.load_config(self.__configFilePath) persistence_file_name = AMinerConfig.build_persistence_file_name( self.aminer_config) if os.path.exists(persistence_file_name): shutil.rmtree(persistence_file_name) if not os.path.exists(persistence_file_name): os.makedirs(persistence_file_name)
def setUp(self): self.aminer_config = AMinerConfig.load_config(self.__configFilePath) self.analysis_context = AnalysisContext(self.aminer_config) self.output_stream = StringIO() self.stream_printer_event_handler = StreamPrinterEventHandler( self.analysis_context, self.output_stream) persistence_file_name = AMinerConfig.build_persistence_file_name( self.aminer_config) if os.path.exists(persistence_file_name): shutil.rmtree(persistence_file_name) if not os.path.exists(persistence_file_name): os.makedirs(persistence_file_name)
def main(): """Run the aminer main program.""" # Extract program name, but only when sure to contain no problematic characters. warnings.filterwarnings('ignore', category=ImportWarning) program_name = sys.argv[0].split('/')[-1] if (program_name == '.') or (program_name == '..') or (re.match( '^[a-zA-Z0-9._-]+$', program_name) is None): print('Invalid program name, check your execution args', file=sys.stderr) sys.exit(1) # We will not read stdin from here on, so get rid of it immediately, thus aberrant child cannot manipulate caller's stdin using it. stdin_fd = os.open('/dev/null', os.O_RDONLY) os.dup2(stdin_fd, 0) os.close(stdin_fd) help_message = 'aminer - logdata-anomaly-miner\n' if supports_color(): help_message += colflame else: help_message += flame parser = argparse.ArgumentParser( description=help_message, formatter_class=argparse.RawTextHelpFormatter) parser.add_argument('-v', '--version', action='version', version=__version_string__) parser.add_argument('-c', '--config', default='/etc/aminer/config.yml', type=str, help='path to the config-file') parser.add_argument('-D', '--daemon', action='store_false', help='run as a daemon process') parser.add_argument( '-s', '--stat', choices=[0, 1, 2], type=int, help='set the stat level. Possible stat-levels are 0 for no statistics' ', 1 for normal statistic level and 2 for verbose statistics.') parser.add_argument( '-d', '--debug', choices=[0, 1, 2], type=int, help='set the debug level. Possible debug-levels are 0 for no ' 'debugging, 1 for normal output (INFO and above), 2 for printing' ' all debug information.') parser.add_argument('--run-analysis', action='store_true', help='enable/disable analysis') parser.add_argument('-C', '--clear', action='store_true', help='removes all persistence directories') parser.add_argument('-r', '--remove', action='append', type=str, help='removes a specific persistence directory') parser.add_argument('-R', '--restore', type=str, help='restore a persistence backup') parser.add_argument( '-f', '--from-begin', action='store_true', help='removes RepositioningData before starting the aminer') args = parser.parse_args() config_file_name = args.config run_in_foreground_flag = args.daemon run_analysis_child_flag = args.run_analysis clear_persistence_flag = args.clear remove_persistence_dirs = args.remove from_begin_flag = args.from_begin if args.restore is not None and ('.' in args.restore or '/' in args.restore): parser.error('The restore path %s must not contain any . or /' % args.restore) if args.remove is not None: for remove in args.remove: if '.' in remove or '/' in remove: parser.error('The remove path %s must not contain any . or /' % remove) restore_relative_persistence_path = args.restore stat_level = 1 debug_level = 1 stat_level_console_flag = False debug_level_console_flag = False if args.stat is not None: stat_level = args.stat stat_level_console_flag = True if args.debug is not None: debug_level = args.stat debug_level_console_flag = True # Load the main configuration file. if not os.path.exists(config_file_name): print('%s: config "%s" not (yet) available!' % (program_name, config_file_name), file=sys.stderr) sys.exit(1) # Minimal import to avoid loading too much within the privileged process. try: aminer_config = AMinerConfig.load_config(config_file_name) except ValueError as e: print("Config-Error: %s" % e) sys.exit(1) persistence_dir = aminer_config.config_properties.get( AMinerConfig.KEY_PERSISTENCE_DIR, AMinerConfig.DEFAULT_PERSISTENCE_DIR) child_user_name = aminer_config.config_properties.get( AMinerConfig.KEY_AMINER_USER) child_group_name = aminer_config.config_properties.get( AMinerConfig.KEY_AMINER_GROUP) child_user_id = -1 child_group_id = -1 try: if child_user_name is not None: from pwd import getpwnam child_user_id = getpwnam(child_user_name).pw_uid if child_group_name is not None: from grp import getgrnam child_group_id = getgrnam(child_user_name).gr_gid except: # skipcq: FLK-E722 print('Failed to resolve %s or %s' % (AMinerConfig.KEY_AMINER_USER, AMinerConfig.KEY_AMINER_GROUP), file=sys.stderr) sys.exit(1) initialize_loggers(aminer_config, child_user_name, child_group_name) if restore_relative_persistence_path is not None and ( clear_persistence_flag or remove_persistence_dirs): msg = 'The --restore parameter removes all persistence files. Do not use this parameter with --Clear or --Remove!' print(msg, sys.stderr) logging.getLogger(AMinerConfig.DEBUG_LOG_NAME).error(msg) sys.exit(1) if not stat_level_console_flag and AMinerConfig.KEY_LOG_STAT_LEVEL in aminer_config.config_properties: stat_level = aminer_config.config_properties[ AMinerConfig.KEY_LOG_STAT_LEVEL] if not debug_level_console_flag and AMinerConfig.KEY_LOG_DEBUG_LEVEL in aminer_config.config_properties: debug_level = aminer_config.config_properties[ AMinerConfig.KEY_LOG_DEBUG_LEVEL] AMinerConfig.STAT_LEVEL = stat_level AMinerConfig.DEBUG_LEVEL = debug_level if clear_persistence_flag: if remove_persistence_dirs: msg = 'The --clear and --remove arguments must not be used together!' print(msg, file=sys.stderr) logging.getLogger(AMinerConfig.DEBUG_LOG_NAME).error(msg) sys.exit(1) clear_persistence(persistence_dir) if remove_persistence_dirs: persistence_dir_name = aminer_config.config_properties.get( AMinerConfig.KEY_PERSISTENCE_DIR, AMinerConfig.DEFAULT_PERSISTENCE_DIR) for filename in os.listdir(persistence_dir_name): file_path = os.path.join(persistence_dir_name, filename) try: if not os.path.isdir(file_path): msg = 'The aminer persistence directory should not contain any files.' print(msg, file=sys.stderr) logging.getLogger(AMinerConfig.DEBUG_LOG_NAME).warning(msg) continue shutil.rmtree(file_path) except OSError as e: msg = 'Failed to delete %s. Reason: %s' % (file_path, e) print(msg, file=sys.stderr) logging.getLogger(AMinerConfig.DEBUG_LOG_NAME).error(msg) for filename in remove_persistence_dirs: file_path = os.path.join(persistence_dir, filename) try: if not os.path.exists(file_path): continue if not os.path.isdir(file_path): msg = 'The aminer persistence directory should not contain any files.' print(msg, file=sys.stderr) logging.getLogger(AMinerConfig.DEBUG_LOG_NAME).warning(msg) continue shutil.rmtree(file_path) except OSError as e: msg = 'Failed to delete %s. Reason: %s' % (file_path, e) print(msg, file=sys.stderr) logging.getLogger(AMinerConfig.DEBUG_LOG_NAME).error(msg) if restore_relative_persistence_path is not None: absolute_persistence_path = os.path.join( persistence_dir, 'backup', restore_relative_persistence_path) if not os.path.exists(absolute_persistence_path): msg = '%s does not exist. Continuing without restoring persistence.' % absolute_persistence_path print(msg, file=sys.stderr) logging.getLogger(AMinerConfig.DEBUG_LOG_NAME).warning(msg) else: clear_persistence(persistence_dir) copytree(absolute_persistence_path, persistence_dir) aminer_user = aminer_config.config_properties.get( AMinerConfig.KEY_AMINER_USER) aminer_grp = aminer_config.config_properties.get( AMinerConfig.KEY_AMINER_GROUP) for dirpath, _dirnames, filenames in os.walk(persistence_dir): shutil.chown(dirpath, aminer_user, aminer_grp) for filename in filenames: shutil.chown(os.path.join(dirpath, filename), aminer_user, aminer_grp) if from_begin_flag: repositioning_data_path = os.path.join( aminer_config.config_properties.get( AMinerConfig.KEY_PERSISTENCE_DIR, AMinerConfig.DEFAULT_PERSISTENCE_DIR), 'AnalysisChild', 'RepositioningData') if os.path.exists(repositioning_data_path): os.remove(repositioning_data_path) if run_analysis_child_flag: # Call analysis process, this function will never return. run_analysis_child(aminer_config, program_name) # Start importing of aminer specific components after reading of "config.py" to allow replacement of components via sys.path # from within configuration. from aminer.util import SecureOSFunctions from aminer.util import decode_string_as_byte_string log_sources_list = aminer_config.config_properties.get( AMinerConfig.KEY_LOG_SOURCES_LIST) if (log_sources_list is None) or not log_sources_list: msg = '%s: %s not defined' % (program_name, AMinerConfig.KEY_LOG_SOURCES_LIST) print(msg, file=sys.stderr) logging.getLogger(AMinerConfig.DEBUG_LOG_NAME).error(msg) sys.exit(1) # Now create the management entries for each logfile. log_data_resource_dict = {} for log_resource_name in log_sources_list: # From here on log_resource_name is a byte array. log_resource_name = decode_string_as_byte_string(log_resource_name) log_resource = None if log_resource_name.startswith(b'file://'): from aminer.input.LogStream import FileLogDataResource log_resource = FileLogDataResource(log_resource_name, -1) elif log_resource_name.startswith(b'unix://'): from aminer.input.LogStream import UnixSocketLogDataResource log_resource = UnixSocketLogDataResource(log_resource_name, -1) else: msg = 'Unsupported schema in %s: %s' % ( AMinerConfig.KEY_LOG_SOURCES_LIST, repr(log_resource_name)) print(msg, file=sys.stderr) logging.getLogger(AMinerConfig.DEBUG_LOG_NAME).error(msg) sys.exit(1) if not os.path.exists(log_resource_name[7:].decode()): msg = "WARNING: file or socket '%s' does not exist (yet)!" % log_resource_name[ 7:].decode() print(msg, file=sys.stderr) logging.getLogger(AMinerConfig.DEBUG_LOG_NAME).warning(msg) try: log_resource.open() except OSError as open_os_error: if open_os_error.errno == errno.EACCES: msg = '%s: no permission to access %s' % ( program_name, repr(log_resource_name)) print(msg, file=sys.stderr) logging.getLogger(AMinerConfig.DEBUG_LOG_NAME).error(msg) sys.exit(1) else: msg = '%s: unexpected error opening %s: %d (%s)' % ( program_name, repr(log_resource_name), open_os_error.errno, os.strerror(open_os_error.errno)) print(msg, file=sys.stderr) logging.getLogger(AMinerConfig.DEBUG_LOG_NAME).error(msg) sys.exit(1) log_data_resource_dict[log_resource_name] = log_resource # Create the remote control socket, if any. Do this in privileged mode to allow binding it at arbitrary locations and support restricted # permissions of any type for current (privileged) uid. remote_control_socket_name = aminer_config.config_properties.get( AMinerConfig.KEY_REMOTE_CONTROL_SOCKET_PATH, None) remote_control_socket = None if remote_control_socket_name is not None: if os.path.exists(remote_control_socket_name): try: os.unlink(remote_control_socket_name) except OSError: msg = 'Failed to clean up old remote control socket at %s' % remote_control_socket_name print(msg, file=sys.stderr) logging.getLogger(AMinerConfig.DEBUG_LOG_NAME).error(msg) sys.exit(1) # Create the local socket: there is no easy way to create it with correct permissions, hence a fork is needed, setting umask, # bind the socket. It is also recommended to create the socket in a directory having the correct permissions already. remote_control_socket = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) remote_control_socket.setblocking(False) bind_child_pid = os.fork() if bind_child_pid == 0: os.umask(0o177) remote_control_socket.bind(remote_control_socket_name) # Do not perform any cleanup, flushing of streams. Use _exit(0) to avoid interference with fork. os._exit(0) # skipcq: PYL-W0212 os.waitpid(bind_child_pid, 0) remote_control_socket.listen(4) # Now have checked all we can get from the configuration in the privileged process. Detach from the TTY when in daemon mode. if not run_in_foreground_flag: child_pid = 0 try: # Fork a child to make sure, we are not the process group leader already. child_pid = os.fork() except Exception as fork_exception: # skipcq: PYL-W0703 msg = 'Failed to daemonize: %s' % fork_exception print(msg, file=sys.stderr) logging.getLogger(AMinerConfig.DEBUG_LOG_NAME).error(msg) sys.exit(1) if child_pid != 0: # This is the parent. os._exit(0) # skipcq: PYL-W0212 # This is the child. Create a new session and become process group leader. Here we get rid of the controlling tty. os.setsid() # Fork again to become an orphaned process not being session leader, hence not able to get a controlling tty again. try: child_pid = os.fork() except Exception as fork_exception: # skipcq: PYL-W0703 msg = 'Failed to daemonize: %s' % fork_exception print(msg, file=sys.stderr) logging.getLogger(AMinerConfig.DEBUG_LOG_NAME).error(msg) sys.exit(1) if child_pid != 0: # This is the parent. os._exit(0) # skipcq: PYL-W0212 # Move to root directory to avoid lingering in some cwd someone else might want to unmount. os.chdir('/') # Change the umask here to clean all group/other mask bits so that accidentially created files are not accessible by other. os.umask(0o77) # Install a signal handler catching common stop signals and relaying it to all children for sure. # skipcq: PYL-W0603 global child_termination_triggered_flag child_termination_triggered_flag = False def graceful_shutdown_handler(_signo, _stackFrame): """React on typical shutdown signals.""" msg = '%s: caught signal, shutting down' % program_name print(msg, file=sys.stderr) logging.getLogger(AMinerConfig.DEBUG_LOG_NAME).info(msg) # Just set the flag. It is likely, that child received same signal also so avoid multiple signaling, which could interrupt the # shutdown procedure again. # skipcq: PYL-W0603 global child_termination_triggered_flag child_termination_triggered_flag = True import signal signal.signal(signal.SIGHUP, graceful_shutdown_handler) signal.signal(signal.SIGINT, graceful_shutdown_handler) signal.signal(signal.SIGTERM, graceful_shutdown_handler) # Now create the socket to connect the analysis child. (parent_socket, child_socket) = socket.socketpair(socket.AF_UNIX, socket.SOCK_DGRAM, 0) # Have it nonblocking from here on. parent_socket.setblocking(False) child_socket.setblocking(False) # Use normal fork, we should have been detached from TTY already. Flush stderr to avoid duplication of output if both child and # parent want to write something. sys.stderr.flush() child_pid = os.fork() if child_pid == 0: # Relocate the child socket fd to 3 if needed if child_socket.fileno() != 3: os.dup2(child_socket.fileno(), 3) child_socket.close() # Clear the supplementary groups before dropping privileges. This makes only sense when changing the uid or gid. if os.getuid() == 0: if ((child_user_id != -1) and (child_user_id != os.getuid())) or ( (child_group_id != -1) and (child_group_id != os.getgid())): os.setgroups([]) # Drop privileges before executing child. setuid/gid will raise an exception when call has failed. if child_group_id != -1: os.setgid(child_group_id) if child_user_id != -1: os.setuid(child_user_id) else: msg = 'INFO: No privilege separation when started as unprivileged user' print(msg, file=sys.stderr) initialize_loggers(aminer_config, 'aminer', 'aminer') logging.getLogger(AMinerConfig.DEBUG_LOG_NAME).info(msg) # Now resolve the specific analysis configuration file (if any). analysis_config_file_name = aminer_config.config_properties.get( AMinerConfig.KEY_ANALYSIS_CONFIG_FILE, None) if analysis_config_file_name is None: analysis_config_file_name = config_file_name elif not os.path.isabs(analysis_config_file_name): analysis_config_file_name = os.path.join( os.path.dirname(config_file_name), analysis_config_file_name) # This is the child. Close all parent file descriptors, we do not need. Perhaps this could be done more elegantly. for close_fd in range(4, 1 << 16): try: os.close(close_fd) except OSError as open_os_error: if open_os_error.errno == errno.EBADF: continue msg = '%s: unexpected exception closing file descriptors: %s' % ( program_name, open_os_error) print(msg, file=sys.stderr) logging.getLogger(AMinerConfig.DEBUG_LOG_NAME).error(msg) # Flush stderr before exit without any cleanup. sys.stderr.flush() os._exit(1) # skipcq: PYL-W0212 # Now execute the very same program again, but user might have moved or renamed it meanwhile. This would be problematic with # SUID-binaries (which we do not yet support). Do NOT just fork but also exec to avoid child circumventing # parent's ALSR due to cloned kernel VMA. execArgs = [ 'AMinerChild', '--run-analysis', '--config', analysis_config_file_name, '--stat', str(stat_level), '--debug', str(debug_level) ] os.execve(sys.argv[0], execArgs, {}) # skipcq: BAN-B606 msg = 'Failed to execute child process' print(msg, file=sys.stderr) logging.getLogger(AMinerConfig.DEBUG_LOG_NAME).error(msg) sys.stderr.flush() os._exit(1) # skipcq: PYL-W0212 child_socket.close() # Send all log resource information currently available to child process. for log_resource_name, log_resource in log_data_resource_dict.items(): if (log_resource is not None) and (log_resource.get_file_descriptor() >= 0): SecureOSFunctions.send_logstream_descriptor( parent_socket, log_resource.get_file_descriptor(), log_resource_name) log_resource.close() # Send the remote control server socket, if any and close it afterwards. It is not needed any more on parent side. if remote_control_socket is not None: SecureOSFunctions.send_annotated_file_descriptor( parent_socket, remote_control_socket.fileno(), 'remotecontrol', '') remote_control_socket.close() exit_status = 0 child_termination_triggered_count = 0 while True: if child_termination_triggered_flag: if child_termination_triggered_count == 0: time.sleep(1) elif child_termination_triggered_count < 5: os.kill(child_pid, signal.SIGTERM) else: os.kill(0, signal.SIGKILL) child_termination_triggered_count += 1 (sig_child_pid, sig_status) = os.waitpid(-1, os.WNOHANG) if sig_child_pid != 0: if sig_child_pid == child_pid: if child_termination_triggered_flag: # This was expected, just terminate. break msg = '%s: Analysis child process %d terminated unexpectedly with signal 0x%x' % ( program_name, sig_child_pid, sig_status) print(msg, file=sys.stderr) logging.getLogger(AMinerConfig.DEBUG_LOG_NAME).error(msg) exit_status = 1 break # So the child has been cloned, the clone has terminated. This should not happen either. msg = '%s: untracked child %d terminated with with signal 0x%x' % ( program_name, sig_child_pid, sig_status) print(msg, file=sys.stderr) logging.getLogger(AMinerConfig.DEBUG_LOG_NAME).error(msg) exit_status = 1 # Child information handled, scan for rotated logfiles or other resources, where reopening might make sense. for log_resouce_name, log_data_resource in log_data_resource_dict.items( ): try: if not log_data_resource.open(reopen_flag=True): continue except OSError as open_os_error: if open_os_error.errno == errno.EACCES: msg = '%s: no permission to access %s' % (program_name, log_resouce_name) print(msg, file=sys.stderr) logging.getLogger(AMinerConfig.DEBUG_LOG_NAME).error(msg) else: msg = '%s: unexpected error reopening %s: %d (%s)' % ( program_name, log_resouce_name, open_os_error.errno, os.strerror(open_os_error.errno)) print(msg, file=sys.stderr) logging.getLogger(AMinerConfig.DEBUG_LOG_NAME).error(msg) exit_status = 2 continue SecureOSFunctions.send_logstream_descriptor( parent_socket, log_data_resource.get_file_descriptor(), log_resouce_name) log_data_resource.close() time.sleep(1) parent_socket.close() SecureOSFunctions.close_base_directory() sys.exit(exit_status)
def test17_demo_yaml_config_equals_python_config(self): """This test checks if the yaml demo config is the same as the python version.""" spec = importlib.util.spec_from_file_location( 'aminer_config', '/usr/lib/logdata-anomaly-miner/aminer/YamlConfig.py') aminer_config = importlib.util.module_from_spec(spec) spec.loader.exec_module(aminer_config) aminer_config.load_yaml('demo/AMiner/demo-config.yml') yml_context = AnalysisContext(aminer_config) yml_context.build_analysis_pipeline() aminer_config = AMinerConfig.load_config('demo/AMiner/demo-config.py') py_context = AnalysisContext(aminer_config) py_context.build_analysis_pipeline() import copy yml_config_properties = copy.deepcopy( yml_context.aminer_config.config_properties) del yml_config_properties['Parser'] del yml_config_properties['Input'] del yml_config_properties['Analysis'] del yml_config_properties['EventHandlers'] del yml_config_properties['LearnMode'] del yml_config_properties['SuppressNewMatchPathDetector'] # remove SimpleUnparsedAtomHandler, VerboseUnparsedAtomHandler and NewMatchPathDetector as they are added by the YamlConfig. py_registered_components = copy.copy(py_context.registered_components) del py_registered_components[0] del py_registered_components[1] del py_registered_components[2] del py_registered_components[10] yml_registered_components = copy.copy( yml_context.registered_components) del yml_registered_components[0] del yml_registered_components[1] tmp = {} keys = list(py_registered_components.keys()) for i in range(1, len(py_registered_components) + 1): tmp[i] = py_registered_components[keys[i - 1]] py_registered_components = tmp py_registered_components_by_name = copy.copy( py_context.registered_components_by_name) del py_registered_components_by_name['SimpleUnparsedHandler'] del py_registered_components_by_name['VerboseUnparsedHandler'] del py_registered_components_by_name['NewMatchPath'] del py_registered_components_by_name['SimpleMonotonicTimestampAdjust'] yml_registered_components_by_name = copy.copy( yml_context.registered_components_by_name) del yml_registered_components_by_name['DefaultNewMatchPathDetector'] del yml_registered_components_by_name['AtomFilter'] self.assertEqual(yml_config_properties, py_context.aminer_config.config_properties) # there actually is no easy way to compare AMiner components as they do not implement the __eq__ method. self.assertEqual(len(yml_registered_components), len(py_registered_components)) for i in range(2, len(yml_registered_components)): self.assertEqual(type(yml_registered_components[i]), type(py_registered_components[i])) self.assertEqual(yml_registered_components_by_name.keys(), py_registered_components_by_name.keys()) for name in yml_registered_components_by_name.keys(): self.assertEqual(type(yml_registered_components_by_name[name]), type(py_registered_components_by_name[name])) self.assertEqual(len(yml_context.real_time_triggered_components), len(py_context.real_time_triggered_components)) # the atom_handler_list is not equal as the python version uses a SimpleMonotonicTimestampAdjust. self.assertEqual(yml_context.atomizer_factory.default_timestamp_paths, py_context.atomizer_factory.default_timestamp_paths) self.assertEqual(type(yml_context.atomizer_factory.event_handler_list), type(py_context.atomizer_factory.event_handler_list))