def test17_demo_yaml_config_equals_python_config(self):
"""This test checks if the yaml demo config is the same as the python version."""
spec = importlib.util.spec_from_file_location('aminer_config', '/usr/lib/logdata-anomaly-miner/aminer/YamlConfig.py')
aminer_config = importlib.util.module_from_spec(spec)
spec.loader.exec_module(aminer_config)
aminer_config.load_yaml('demo/aminer/demo-config.yml')
yml_context = AnalysisContext(aminer_config)
yml_context.build_analysis_pipeline()
aminer_config = AminerConfig.load_config('demo/aminer/demo-config.py')
py_context = AnalysisContext(aminer_config)
py_context.build_analysis_pipeline()
import copy
yml_config_properties = copy.deepcopy(yml_context.aminer_config.config_properties)
del yml_config_properties['Parser']
del yml_config_properties['Input']
del yml_config_properties['Analysis']
del yml_config_properties['EventHandlers']
del yml_config_properties['LearnMode']
del yml_config_properties['SuppressNewMatchPathDetector']
# remove SimpleUnparsedAtomHandler, VerboseUnparsedAtomHandler and NewMatchPathDetector as they are added by the YamlConfig.
py_registered_components = copy.copy(py_context.registered_components)
del py_registered_components[0]
del py_registered_components[1]
del py_registered_components[2]
del py_registered_components[10]
yml_registered_components = copy.copy(yml_context.registered_components)
del yml_registered_components[0]
del yml_registered_components[1]
tmp = {}
keys = list(py_registered_components.keys())
for i in range(1, len(py_registered_components)+1):
tmp[i] = py_registered_components[keys[i-1]]
py_registered_components = tmp
py_registered_components_by_name = copy.copy(py_context.registered_components_by_name)
del py_registered_components_by_name['SimpleUnparsedHandler']
del py_registered_components_by_name['VerboseUnparsedHandler']
del py_registered_components_by_name['NewMatchPath']
del py_registered_components_by_name['SimpleMonotonicTimestampAdjust']
yml_registered_components_by_name = copy.copy(yml_context.registered_components_by_name)
del yml_registered_components_by_name['DefaultNewMatchPathDetector']
del yml_registered_components_by_name['AtomFilter']
self.assertEqual(yml_config_properties, py_context.aminer_config.config_properties)
# there actually is no easy way to compare aminer components as they do not implement the __eq__ method.
self.assertEqual(len(yml_registered_components), len(py_registered_components))
for i in range(2, len(yml_registered_components)): # skipcq: PTC-W0060
self.assertEqual(type(yml_registered_components[i]), type(py_registered_components[i]))
self.assertEqual(yml_registered_components_by_name.keys(), py_registered_components_by_name.keys())
for name in yml_registered_components_by_name.keys():
self.assertEqual(type(yml_registered_components_by_name[name]), type(py_registered_components_by_name[name]))
self.assertEqual(len(yml_context.real_time_triggered_components), len(py_context.real_time_triggered_components))
# the atom_handler_list is not equal as the python version uses a SimpleMonotonicTimestampAdjust.
self.assertEqual(yml_context.atomizer_factory.default_timestamp_paths, py_context.atomizer_factory.default_timestamp_paths)
self.assertEqual(type(yml_context.atomizer_factory.event_handler_list), type(py_context.atomizer_factory.event_handler_list))
def tearDown(self):
"""Delete all persisted data after the tests."""
self.aminer_config = AminerConfig.load_config(self.__configFilePath)
persistence_file_name = AminerConfig.build_persistence_file_name(
self.aminer_config)
if os.path.exists(persistence_file_name):
shutil.rmtree(persistence_file_name)
if not os.path.exists(persistence_file_name):
os.makedirs(persistence_file_name)
SecureOSFunctions.close_base_directory()
def setUp(self):
"""Set up all needed variables and remove persisted data."""
PersistenceUtil.persistable_components = []
self.aminer_config = AminerConfig.load_config(self.__configFilePath)
self.analysis_context = AnalysisContext(self.aminer_config)
self.output_stream = StringIO()
self.stream_printer_event_handler = StreamPrinterEventHandler(
self.analysis_context, self.output_stream)
persistence_dir_name = AminerConfig.build_persistence_file_name(
self.aminer_config)
if os.path.exists(persistence_dir_name):
shutil.rmtree(persistence_dir_name)
if not os.path.exists(persistence_dir_name):
os.makedirs(persistence_dir_name)
initialize_loggers(self.aminer_config,
getpwnam('aminer').pw_uid,
getgrnam('aminer').gr_gid)
if isinstance(persistence_dir_name, str):
persistence_dir_name = persistence_dir_name.encode()
SecureOSFunctions.secure_open_base_directory(
persistence_dir_name, os.O_RDONLY | os.O_DIRECTORY | os.O_PATH)
PersistenceUtil.SKIP_PERSISTENCE_ID_WARNING = True
def main():
"""Run the aminer main program."""
# Extract program name, but only when sure to contain no problematic characters.
warnings.filterwarnings('ignore', category=ImportWarning)
program_name = sys.argv[0].split('/')[-1]
if (program_name == '.') or (program_name == '..') or (re.match(
'^[a-zA-Z0-9._-]+$', program_name) is None):
print('Invalid program name, check your execution args',
file=sys.stderr)
sys.exit(1)
# We will not read stdin from here on, so get rid of it immediately, thus aberrant child cannot manipulate caller's stdin using it.
stdin_fd = os.open('/dev/null', os.O_RDONLY)
os.dup2(stdin_fd, 0)
os.close(stdin_fd)
help_message = 'aminer - logdata-anomaly-miner\n'
if supports_color():
help_message += colflame
else:
help_message += flame
parser = argparse.ArgumentParser(
description=help_message,
formatter_class=argparse.RawTextHelpFormatter)
parser.add_argument('-v',
'--version',
action='version',
version=__version_string__)
parser.add_argument('-u',
'--check-updates',
action='store_true',
help='check if updates for the aminer are available.')
parser.add_argument('-c',
'--config',
default='/etc/aminer/config.yml',
type=str,
help='path to the config-file')
parser.add_argument('-D',
'--daemon',
action='store_false',
help='run as a daemon process')
parser.add_argument(
'-s',
'--stat',
choices=["0", "1", "2"],
type=str,
help=
'set the stat level. Possible stat-levels are 0 for no statistics, 1 for normal statistic level and 2 for '
'verbose statistics.')
parser.add_argument(
'-d',
'--debug',
choices=["0", "1", "2"],
type=str,
help=
'set the debug level. Possible debug-levels are 0 for no debugging, 1 for normal output (INFO and above), 2 '
'for printing all debug information.')
parser.add_argument('--run-analysis',
action='store_true',
help='enable/disable analysis')
parser.add_argument('-C',
'--clear',
action='store_true',
help='removes all persistence directories')
parser.add_argument('-r',
'--remove',
action='append',
type=str,
help='removes a specific persistence directory')
parser.add_argument('-R',
'--restore',
type=str,
help='restore a persistence backup')
parser.add_argument(
'-f',
'--from-begin',
action='store_true',
help='removes RepositioningData before starting the aminer')
parser.add_argument(
'-o',
'--offline-mode',
action='store_true',
help='stop the aminer after all logs have been processed.')
parser.add_argument(
"--config-properties",
metavar="KEY=VALUE",
nargs='+',
help=
"Set a number of config_properties by using key-value pairs (do not put spaces before or after the = sign). "
"If a value contains spaces, you should define it with double quotes: 'foo=\"this is a sentence\". Note that "
"values are always treated as strings. If values are already defined in the config_properties, the input "
"types are converted to the ones already existing.")
args = parser.parse_args()
if args.check_updates:
import urllib3
url = 'https://raw.githubusercontent.com/ait-aecid/logdata-anomaly-miner/main/source/root/usr/lib/logdata-anomaly-miner/metadata.py'
http = urllib3.PoolManager()
r = http.request('GET', url, preload_content=True)
metadata = r.data.decode()
http.clear()
lines = metadata.split('\n')
curr_version = None
for line in lines:
if '__version__ = ' in line:
curr_version = line.split('__version__ = ')[1].strip('"')
break
if __version__ == curr_version:
print("The current aminer version %s is installed." % curr_version)
else:
print(
"A new aminer version exists (%s). Currently version %s is installed."
% (curr_version, __version__))
print("Use git pull to update the aminer version.")
sys.exit(0)
config_file_name = args.config
run_in_foreground_flag = args.daemon
run_analysis_child_flag = args.run_analysis
clear_persistence_flag = args.clear
remove_persistence_dirs = args.remove
from_begin_flag = args.from_begin
global offline_mode # skipcq: PYL-W0603
offline_mode = args.offline_mode
if args.restore is not None and ('.' in args.restore
or '/' in args.restore):
parser.error('The restore path %s must not contain any . or /' %
args.restore)
if args.remove is not None:
for remove in args.remove:
if '.' in remove or '/' in remove:
parser.error('The remove path %s must not contain any . or /' %
remove)
restore_relative_persistence_path = args.restore
stat_level = 1
debug_level = 1
stat_level_console_flag = False
debug_level_console_flag = False
if args.stat is not None:
stat_level = int(args.stat)
stat_level_console_flag = True
if args.debug is not None:
debug_level = int(args.debug)
debug_level_console_flag = True
# Load the main configuration file.
if not os.path.exists(config_file_name):
print('%s: config "%s" not (yet) available!' %
(program_name, config_file_name),
file=sys.stderr)
sys.exit(1)
# using the solution here to override config_properties:
# https://stackoverflow.com/questions/27146262/create-variable-key-value-pairs-with-argparse-python
use_temp_config = False
config_properties = parse_vars(args.config_properties)
if args.config_properties and "LearnMode" in config_properties:
ymlext = [".YAML", ".YML", ".yaml", ".yml"]
extension = os.path.splitext(config_file_name)[1]
if extension in ymlext:
use_temp_config = True
fd, temp_config = tempfile.mkstemp(suffix=".yml")
with open(config_file_name) as f:
for line in f:
if "LearnMode" in line:
line = "LearnMode: %s" % config_properties["LearnMode"]
os.write(fd, line.encode())
config_file_name = temp_config
os.close(fd)
else:
msg = "The LearnMode parameter does not exist in .py configs!"
print(msg, sys.stderr)
logging.getLogger(AminerConfig.DEBUG_LOG_NAME).error(msg)
sys.exit(1)
# Minimal import to avoid loading too much within the privileged process.
try:
aminer_config = AminerConfig.load_config(config_file_name)
if use_temp_config:
os.remove(config_file_name)
config_file_name = args.config
except ValueError:
sys.exit(1)
for config_property in config_properties:
if config_property == "LearnMode":
continue
old_value = aminer_config.config_properties.get(config_property)
value = config_properties[config_property]
if old_value is not None:
try:
if isinstance(old_value, bool):
if value == "True":
value = True
elif value == "False":
value = False
else:
msg = "The %s parameter must be of type %s!" % (
config_property, type(old_value))
print(msg, sys.stderr)
logging.getLogger(
AminerConfig.DEBUG_LOG_NAME).error(msg)
sys.exit(1)
elif isinstance(old_value, int):
value = int(value)
elif isinstance(old_value, float):
value = float(value)
elif isinstance(old_value, list):
value = ast.literal_eval(value)
except ValueError:
msg = "The %s parameter must be of type %s!" % (
config_property, type(old_value))
print(msg, sys.stderr)
logging.getLogger(AminerConfig.DEBUG_LOG_NAME).error(msg)
sys.exit(1)
else:
msg = "The %s parameter is not set in the config. It will be treated as a string!" % config_property
print("WARNING: " + msg, sys.stderr)
logging.getLogger(AminerConfig.DEBUG_LOG_NAME).warning(msg)
aminer_config.config_properties[config_property] = value
persistence_dir = aminer_config.config_properties.get(
AminerConfig.KEY_PERSISTENCE_DIR, AminerConfig.DEFAULT_PERSISTENCE_DIR)
child_user_name = aminer_config.config_properties.get(
AminerConfig.KEY_AMINER_USER)
child_group_name = aminer_config.config_properties.get(
AminerConfig.KEY_AMINER_GROUP)
child_user_id = -1
child_group_id = -1
try:
if child_user_name is not None:
child_user_id = getpwnam(child_user_name).pw_uid
if child_group_name is not None:
child_group_id = getgrnam(child_group_name).gr_gid
except: # skipcq: FLK-E722
print('Failed to resolve %s or %s' %
(AminerConfig.KEY_AMINER_USER, AminerConfig.KEY_AMINER_GROUP),
file=sys.stderr)
sys.exit(1)
if not stat_level_console_flag and AminerConfig.KEY_LOG_STAT_LEVEL in aminer_config.config_properties:
stat_level = aminer_config.config_properties[
AminerConfig.KEY_LOG_STAT_LEVEL]
if not debug_level_console_flag and AminerConfig.KEY_LOG_DEBUG_LEVEL in aminer_config.config_properties:
debug_level = aminer_config.config_properties[
AminerConfig.KEY_LOG_DEBUG_LEVEL]
if AminerConfig.CONFIG_KEY_ENCODING in aminer_config.config_properties:
AminerConfig.ENCODING = aminer_config.config_properties[
AminerConfig.CONFIG_KEY_ENCODING]
AminerConfig.STAT_LEVEL = stat_level
AminerConfig.DEBUG_LEVEL = debug_level
initialize_loggers(aminer_config, child_user_id, child_group_id)
if restore_relative_persistence_path is not None and (
clear_persistence_flag or remove_persistence_dirs):
msg = 'The --restore parameter removes all persistence files. Do not use this parameter with --Clear or --Remove!'
print(msg, sys.stderr)
logging.getLogger(AminerConfig.DEBUG_LOG_NAME).error(msg)
sys.exit(1)
if clear_persistence_flag:
if remove_persistence_dirs:
msg = 'The --clear and --remove arguments must not be used together!'
print(msg, file=sys.stderr)
logging.getLogger(AminerConfig.DEBUG_LOG_NAME).error(msg)
sys.exit(1)
clear_persistence(persistence_dir)
if remove_persistence_dirs:
persistence_dir_name = aminer_config.config_properties.get(
AminerConfig.KEY_PERSISTENCE_DIR,
AminerConfig.DEFAULT_PERSISTENCE_DIR)
for filename in os.listdir(persistence_dir_name):
file_path = os.path.join(persistence_dir_name, filename)
try:
if not os.path.isdir(file_path):
msg = 'The aminer persistence directory should not contain any files.'
print(msg, file=sys.stderr)
logging.getLogger(AminerConfig.DEBUG_LOG_NAME).warning(msg)
continue
shutil.rmtree(file_path)
except OSError as e:
msg = 'Failed to delete %s. Reason: %s' % (file_path, e)
print(msg, file=sys.stderr)
logging.getLogger(AminerConfig.DEBUG_LOG_NAME).error(msg)
for filename in remove_persistence_dirs:
file_path = os.path.join(persistence_dir, filename)
try:
if not os.path.exists(file_path):
continue
if not os.path.isdir(file_path):
msg = 'The aminer persistence directory should not contain any files.'
print(msg, file=sys.stderr)
logging.getLogger(AminerConfig.DEBUG_LOG_NAME).warning(msg)
continue
shutil.rmtree(file_path)
except OSError as e:
msg = 'Failed to delete %s. Reason: %s' % (file_path, e)
print(msg, file=sys.stderr)
logging.getLogger(AminerConfig.DEBUG_LOG_NAME).error(msg)
if restore_relative_persistence_path is not None:
absolute_persistence_path = os.path.join(
persistence_dir, 'backup', restore_relative_persistence_path)
if not os.path.exists(absolute_persistence_path):
msg = '%s does not exist. Continuing without restoring persistence.' % absolute_persistence_path
print(msg, file=sys.stderr)
logging.getLogger(AminerConfig.DEBUG_LOG_NAME).warning(msg)
else:
clear_persistence(persistence_dir)
copytree(absolute_persistence_path, persistence_dir)
persistence_dir_fd = SecureOSFunctions.secure_open_base_directory(
persistence_dir, os.O_RDONLY | os.O_DIRECTORY | os.O_PATH)
for dirpath, _dirnames, filenames in os.walk(persistence_dir):
os.chown(dirpath,
child_user_id,
child_group_id,
dir_fd=persistence_dir_fd,
follow_symlinks=False)
for filename in filenames:
os.chown(os.path.join(dirpath, filename),
child_user_id,
child_user_id,
dir_fd=persistence_dir_fd,
follow_symlinks=False)
if from_begin_flag:
repositioning_data_path = os.path.join(
aminer_config.config_properties.get(
AminerConfig.KEY_PERSISTENCE_DIR,
AminerConfig.DEFAULT_PERSISTENCE_DIR), 'AnalysisChild',
'RepositioningData')
if os.path.exists(repositioning_data_path):
os.remove(repositioning_data_path)
if run_analysis_child_flag:
# Call analysis process, this function will never return.
run_analysis_child(aminer_config, program_name)
# Start importing of aminer specific components after reading of "config.py" to allow replacement of components via sys.path
# from within configuration.
log_sources_list = aminer_config.config_properties.get(
AminerConfig.KEY_LOG_SOURCES_LIST)
if (log_sources_list is None) or not log_sources_list:
msg = '%s: %s not defined' % (program_name,
AminerConfig.KEY_LOG_SOURCES_LIST)
print(msg, file=sys.stderr)
logging.getLogger(AminerConfig.DEBUG_LOG_NAME).error(msg)
sys.exit(1)
# Now create the management entries for each logfile.
log_data_resource_dict = {}
for log_resource_name in log_sources_list:
# From here on log_resource_name is a byte array.
log_resource_name = decode_string_as_byte_string(log_resource_name)
log_resource = None
if log_resource_name.startswith(b'file://'):
log_resource = FileLogDataResource(log_resource_name, -1)
elif log_resource_name.startswith(b'unix://'):
log_resource = UnixSocketLogDataResource(log_resource_name, -1)
else:
msg = 'Unsupported schema in %s: %s' % (
AminerConfig.KEY_LOG_SOURCES_LIST, repr(log_resource_name))
print(msg, file=sys.stderr)
logging.getLogger(AminerConfig.DEBUG_LOG_NAME).error(msg)
sys.exit(1)
if not os.path.exists(log_resource_name[7:].decode()):
msg = "WARNING: file or socket '%s' does not exist (yet)!" % log_resource_name[
7:].decode()
print(msg, file=sys.stderr)
logging.getLogger(AminerConfig.DEBUG_LOG_NAME).warning(msg)
try:
log_resource.open()
except OSError as open_os_error:
if open_os_error.errno == errno.EACCES:
msg = '%s: no permission to access %s' % (
program_name, repr(log_resource_name))
print(msg, file=sys.stderr)
logging.getLogger(AminerConfig.DEBUG_LOG_NAME).error(msg)
sys.exit(1)
else:
msg = '%s: unexpected error opening %s: %d (%s)' % (
program_name, repr(log_resource_name), open_os_error.errno,
os.strerror(open_os_error.errno))
print(msg, file=sys.stderr)
logging.getLogger(AminerConfig.DEBUG_LOG_NAME).error(msg)
sys.exit(1)
log_data_resource_dict[log_resource_name] = log_resource
# Create the remote control socket, if any. Do this in privileged mode to allow binding it at arbitrary locations and support restricted
# permissions of any type for current (privileged) uid.
remote_control_socket_name = aminer_config.config_properties.get(
AminerConfig.KEY_REMOTE_CONTROL_SOCKET_PATH, None)
remote_control_socket = None
if remote_control_socket_name is not None:
if os.path.exists(remote_control_socket_name):
try:
os.unlink(remote_control_socket_name)
except OSError:
msg = 'Failed to clean up old remote control socket at %s' % remote_control_socket_name
print(msg, file=sys.stderr)
logging.getLogger(AminerConfig.DEBUG_LOG_NAME).error(msg)
sys.exit(1)
# Create the local socket: there is no easy way to create it with correct permissions, hence a fork is needed, setting umask,
# bind the socket. It is also recommended to create the socket in a directory having the correct permissions already.
remote_control_socket = socket.socket(socket.AF_UNIX,
socket.SOCK_STREAM)
remote_control_socket.setblocking(False)
bind_child_pid = os.fork()
if bind_child_pid == 0:
os.umask(0o177)
remote_control_socket.bind(remote_control_socket_name)
# Do not perform any cleanup, flushing of streams. Use _exit(0) to avoid interference with fork.
os._exit(0) # skipcq: PYL-W0212
os.waitpid(bind_child_pid, 0)
remote_control_socket.listen(4)
# Now have checked all we can get from the configuration in the privileged process. Detach from the TTY when in daemon mode.
if not run_in_foreground_flag:
child_pid = 0
try:
# Fork a child to make sure, we are not the process group leader already.
child_pid = os.fork()
except Exception as fork_exception: # skipcq: PYL-W0703
msg = 'Failed to daemonize: %s' % fork_exception
print(msg, file=sys.stderr)
logging.getLogger(AminerConfig.DEBUG_LOG_NAME).error(msg)
sys.exit(1)
if child_pid != 0:
# This is the parent.
os._exit(0) # skipcq: PYL-W0212
# This is the child. Create a new session and become process group leader. Here we get rid of the controlling tty.
os.setsid()
# Fork again to become an orphaned process not being session leader, hence not able to get a controlling tty again.
try:
child_pid = os.fork()
except Exception as fork_exception: # skipcq: PYL-W0703
msg = 'Failed to daemonize: %s' % fork_exception
print(msg, file=sys.stderr)
logging.getLogger(AminerConfig.DEBUG_LOG_NAME).error(msg)
sys.exit(1)
if child_pid != 0:
# This is the parent.
os._exit(0) # skipcq: PYL-W0212
# Move to root directory to avoid lingering in some cwd someone else might want to unmount.
os.chdir('/')
# Change the umask here to clean all group/other mask bits so that accidentially created files are not accessible by other.
os.umask(0o77)
# Install a signal handler catching common stop signals and relaying it to all children for sure.
# skipcq: PYL-W0603
global child_termination_triggered_flag
child_termination_triggered_flag = False
def graceful_shutdown_handler(_signo, _stackFrame):
"""React on typical shutdown signals."""
msg = '%s: caught signal, shutting down' % program_name
print(msg, file=sys.stderr)
logging.getLogger(AminerConfig.DEBUG_LOG_NAME).info(msg)
# Just set the flag. It is likely, that child received same signal also so avoid multiple signaling, which could interrupt the
# shutdown procedure again.
# skipcq: PYL-W0603
global child_termination_triggered_flag
child_termination_triggered_flag = True
import signal
signal.signal(signal.SIGHUP, graceful_shutdown_handler)
signal.signal(signal.SIGINT, graceful_shutdown_handler)
signal.signal(signal.SIGTERM, graceful_shutdown_handler)
# Now create the socket to connect the analysis child.
(parent_socket, child_socket) = socket.socketpair(socket.AF_UNIX,
socket.SOCK_DGRAM, 0)
# Have it nonblocking from here on.
parent_socket.setblocking(False)
child_socket.setblocking(False)
# Use normal fork, we should have been detached from TTY already. Flush stderr to avoid duplication of output if both child and
# parent want to write something.
sys.stderr.flush()
child_pid = os.fork()
if child_pid == 0:
# Relocate the child socket fd to 3 if needed
if child_socket.fileno() != 3:
os.dup2(child_socket.fileno(), 3)
child_socket.close()
# Clear the supplementary groups before dropping privileges. This makes only sense when changing the uid or gid.
if os.getuid() == 0:
if ((child_user_id != -1) and (child_user_id != os.getuid())) or (
(child_group_id != -1) and (child_group_id != os.getgid())):
os.setgroups([])
# Drop privileges before executing child. setuid/gid will raise an exception when call has failed.
if child_group_id != -1:
os.setgid(child_group_id)
if child_user_id != -1:
os.setuid(child_user_id)
else:
msg = 'INFO: No privilege separation when started as unprivileged user'
print(msg, file=sys.stderr)
tmp_username = aminer_config.config_properties.get(
AminerConfig.KEY_AMINER_USER)
tmp_group = aminer_config.config_properties.get(
AminerConfig.KEY_AMINER_GROUP)
aminer_user_id = -1
aminer_group_id = -1
try:
if tmp_username is not None:
aminer_user_id = getpwnam(tmp_username).pw_uid
if tmp_group is not None:
aminer_group_id = getgrnam(tmp_group).gr_gid
except: # skipcq: FLK-E722
print('Failed to resolve %s or %s' %
(AminerConfig.KEY_AMINER_USER,
AminerConfig.KEY_AMINER_GROUP),
file=sys.stderr)
sys.exit(1)
initialize_loggers(aminer_config, aminer_user_id, aminer_group_id)
logging.getLogger(AminerConfig.DEBUG_LOG_NAME).info(msg)
# Now resolve the specific analysis configuration file (if any).
analysis_config_file_name = aminer_config.config_properties.get(
AminerConfig.KEY_ANALYSIS_CONFIG_FILE, None)
if analysis_config_file_name is None:
analysis_config_file_name = config_file_name
elif not os.path.isabs(analysis_config_file_name):
analysis_config_file_name = os.path.join(
os.path.dirname(config_file_name), analysis_config_file_name)
# This is the child. Close all parent file descriptors, we do not need. Perhaps this could be done more elegantly.
for close_fd in range(4, 1 << 16):
try:
os.close(close_fd)
except OSError as open_os_error:
if open_os_error.errno == errno.EBADF:
continue
msg = '%s: unexpected exception closing file descriptors: %s' % (
program_name, open_os_error)
print(msg, file=sys.stderr)
logging.getLogger(AminerConfig.DEBUG_LOG_NAME).error(msg)
# Flush stderr before exit without any cleanup.
sys.stderr.flush()
os._exit(1) # skipcq: PYL-W0212
# Now execute the very same program again, but user might have moved or renamed it meanwhile. This would be problematic with
# SUID-binaries (which we do not yet support). Do NOT just fork but also exec to avoid child circumventing
# parent's ALSR due to cloned kernel VMA.
exec_args = [
'aminerChild', '--run-analysis', '--config',
analysis_config_file_name, '--stat',
str(stat_level), '--debug',
str(debug_level)
]
if offline_mode:
exec_args.append("--offline-mode")
if args.config_properties:
exec_args.append("--config-properties")
for config_property in args.config_properties:
exec_args.append(config_property)
os.execv(sys.argv[0], exec_args) # skipcq: BAN-B606
msg = 'Failed to execute child process'
print(msg, file=sys.stderr)
logging.getLogger(AminerConfig.DEBUG_LOG_NAME).error(msg)
sys.stderr.flush()
os._exit(1) # skipcq: PYL-W0212
child_socket.close()
# Send all log resource information currently available to child process.
for log_resource_name, log_resource in log_data_resource_dict.items():
if (log_resource
is not None) and (log_resource.get_file_descriptor() >= 0):
SecureOSFunctions.send_logstream_descriptor(
parent_socket, log_resource.get_file_descriptor(),
log_resource_name)
log_resource.close()
# Send the remote control server socket, if any and close it afterwards. It is not needed any more on parent side.
if remote_control_socket is not None:
SecureOSFunctions.send_annotated_file_descriptor(
parent_socket, remote_control_socket.fileno(), 'remotecontrol', '')
remote_control_socket.close()
exit_status = 0
child_termination_triggered_count = 0
while True:
if child_termination_triggered_flag:
if child_termination_triggered_count == 0:
time.sleep(1)
elif child_termination_triggered_count < 5:
os.kill(child_pid, signal.SIGTERM)
else:
os.kill(0, signal.SIGKILL)
child_termination_triggered_count += 1
(sig_child_pid, sig_status) = os.waitpid(-1, os.WNOHANG)
if sig_child_pid != 0:
if sig_child_pid == child_pid:
if child_termination_triggered_flag or offline_mode:
# This was expected, just terminate.
break
msg = '%s: Analysis child process %d terminated unexpectedly with signal 0x%x' % (
program_name, sig_child_pid, sig_status)
print(msg, file=sys.stderr)
logging.getLogger(AminerConfig.DEBUG_LOG_NAME).error(msg)
exit_status = 1
break
# So the child has been cloned, the clone has terminated. This should not happen either.
msg = '%s: untracked child %d terminated with with signal 0x%x' % (
program_name, sig_child_pid, sig_status)
print(msg, file=sys.stderr)
logging.getLogger(AminerConfig.DEBUG_LOG_NAME).error(msg)
exit_status = 1
# Child information handled, scan for rotated logfiles or other resources, where reopening might make sense.
for log_resouce_name, log_data_resource in log_data_resource_dict.items(
):
try:
if not log_data_resource.open(reopen_flag=True):
continue
except OSError as open_os_error:
if open_os_error.errno == errno.EACCES:
msg = '%s: no permission to access %s' % (program_name,
log_resouce_name)
print(msg, file=sys.stderr)
logging.getLogger(AminerConfig.DEBUG_LOG_NAME).error(msg)
else:
msg = '%s: unexpected error reopening %s: %d (%s)' % (
program_name, log_resouce_name, open_os_error.errno,
os.strerror(open_os_error.errno))
print(msg, file=sys.stderr)
logging.getLogger(AminerConfig.DEBUG_LOG_NAME).error(msg)
exit_status = 2
continue
SecureOSFunctions.send_logstream_descriptor(
parent_socket, log_data_resource.get_file_descriptor(),
log_resouce_name)
log_data_resource.close()
time.sleep(1)
parent_socket.close()
SecureOSFunctions.close_base_directory()
SecureOSFunctions.close_log_directory()
sys.exit(exit_status)