def test6send_logstream_descriptor(self):
"""A valid logstream descriptor is to be sent."""
sock_name = '/tmp/test6unixSocket.sock'
data = b'logstream' + b'\x00' + b'/var/log/syslog'
# skipcq: BAN-B607, BAN-B603
subprocess.Popen(['python3', 'unit/util/clientTest6.py'])
if os.path.exists(sock_name):
os.remove(sock_name)
# print(self.opening_socket)
server = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
server.bind(sock_name)
server.listen(1)
connection = server.accept()[0]
unix_socket_log_data_resource = UnixSocketLogDataResource(
b'unix:///tmp/test6unixSocket.sock', connection.fileno())
# print(self.listening)
unix_socket_log_data_resource.fill_buffer()
self.assertEqual(unix_socket_log_data_resource.buffer, data)
# print('Data received: %s' % unix_socket_log_data_resource.buffer)
unix_socket_log_data_resource.update_position(
len(unix_socket_log_data_resource.buffer))
self.assertEqual(unix_socket_log_data_resource.total_consumed_length,
25)
self.assertEqual(unix_socket_log_data_resource.buffer, b'')
# print("Shutting down...")
unix_socket_log_data_resource.close()
def test4sendAnnotatedFileDescriptor(self):
"""A valid annotated file descriptor is to be sent by a socket."""
sock_name = '/tmp/test4unixSocket.sock'
data = b'readmeStream' + b'\x00' + b'You should read these README instructions for better understanding.'
# skipcq: BAN-B607, BAN-B603
proc = subprocess.Popen(['python3', 'unit/util/clientTest4.py'])
if os.path.exists(sock_name):
os.remove(sock_name)
# print(self.opening_socket)
server = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
server.bind(sock_name)
server.listen(1)
connection = server.accept()[0]
unix_socket_log_data_resource = UnixSocketLogDataResource(
b'unix:///tmp/test4unixSocket.sock', connection.fileno())
# print(self.listening)
unix_socket_log_data_resource.fill_buffer()
self.assertEqual(unix_socket_log_data_resource.buffer, data)
# print('Data received: %s' % unix_socket_log_data_resource.buffer)
unix_socket_log_data_resource.update_position(
len(unix_socket_log_data_resource.buffer))
self.assertEqual(unix_socket_log_data_resource.total_consumed_length,
80)
self.assertEqual(unix_socket_log_data_resource.buffer, b'')
# print("Shutting down...")
unix_socket_log_data_resource.close()
proc.terminate()
proc.wait()
def handle_master_control_socket_receive(self):
"""
Receive information from the parent process via the master control socket.
This method may only be invoked when receiving is guaranteed to be nonblocking and to return data.
"""
# We cannot fail with None here as the socket was in the readList.
(received_fd, received_type_info,
annotation_data) = SecureOSFunctions.receive_annoted_file_descriptor(
self.master_control_socket)
if received_type_info == b'logstream':
repositioning_data = self.repositioning_data_dict.get(
annotation_data, None)
if repositioning_data is not None:
del self.repositioning_data_dict[annotation_data]
res = None
if annotation_data.startswith(b'file://'):
from aminer.input.LogStream import FileLogDataResource
res = FileLogDataResource(
annotation_data,
received_fd,
repositioning_data=repositioning_data)
elif annotation_data.startswith(b'unix://'):
from aminer.input.LogStream import UnixSocketLogDataResource
res = UnixSocketLogDataResource(annotation_data, received_fd)
else:
msg = 'Filedescriptor of unknown type received'
logging.getLogger(DEBUG_LOG_NAME).error(msg)
raise Exception(msg)
# Make fd nonblocking.
fd_flags = fcntl.fcntl(res.get_file_descriptor(), fcntl.F_GETFL)
fcntl.fcntl(res.get_file_descriptor(), fcntl.F_SETFL,
fd_flags | os.O_NONBLOCK)
log_stream = self.log_streams_by_name.get(res.get_resource_name())
if log_stream is None:
stream_atomizer = self.analysis_context.atomizer_factory.get_atomizer_for_resource(
res.get_resource_name())
log_stream = LogStream(res, stream_atomizer)
self.tracked_fds_dict[res.get_file_descriptor()] = log_stream
self.log_streams_by_name[res.get_resource_name()] = log_stream
else:
log_stream.add_next_resource(res)
elif received_type_info == b'remotecontrol':
if self.remote_control_socket is not None:
msg = 'Received another remote control socket: multiple remote control not supported (yet?).'
logging.getLogger(DEBUG_LOG_NAME).error(msg)
raise Exception(msg)
self.remote_control_socket = socket.fromfd(received_fd,
socket.AF_UNIX,
socket.SOCK_STREAM, 0)
os.close(received_fd)
self.tracked_fds_dict[self.remote_control_socket.fileno(
)] = self.remote_control_socket
else:
msg = 'Unhandled type info on received fd: %s' % repr(
received_type_info)
logging.getLogger(DEBUG_LOG_NAME).error(msg)
raise Exception(msg)
def handleMasterControlSocketReceive(self):
"""Receive information from the parent process via the master
control socket. This method may only be invoked when receiving
is guaranteed to be nonblocking and to return data."""
# We cannot fail with None here as the socket was in the readList.
(receivedFd, receivedTypeInfo, annotationData) = \
SecureOSFunctions.receiveAnnotedFileDescriptor(self.masterControlSocket)
if receivedTypeInfo == b'logstream':
repositioningData = self.repositioningDataDict.get(
annotationData, None)
if repositioningData != None:
del self.repositioningDataDict[annotationData]
resource = None
if annotationData.startswith(b'file://'):
from aminer.input.LogStream import FileLogDataResource
resource = FileLogDataResource(annotationData, receivedFd, \
repositioningData=repositioningData)
elif annotationData.startswith(b'unix://'):
from aminer.input.LogStream import UnixSocketLogDataResource
resource = UnixSocketLogDataResource(annotationData,
receivedFd)
else:
raise Exception('Filedescriptor of unknown type received')
# Make fd nonblocking.
fdFlags = fcntl.fcntl(resource.getFileDescriptor(), fcntl.F_GETFL)
fcntl.fcntl(resource.getFileDescriptor(), fcntl.F_SETFL,
fdFlags | os.O_NONBLOCK)
logStream = self.logStreamsByName.get(resource.getResourceName())
if logStream is None:
streamAtomizer = self.analysisContext.atomizerFactory.getAtomizerForResource(
resource.getResourceName())
logStream = LogStream(resource, streamAtomizer)
self.trackedFdsDict[resource.getFileDescriptor()] = logStream
self.logStreamsByName[resource.getResourceName()] = logStream
else:
logStream.addNextResource(resource)
elif receivedTypeInfo == b'remotecontrol':
if self.remoteControlSocket != None:
raise Exception('Received another remote control ' \
'socket: multiple remote control not (yet?) supported.')
self.remoteControlSocket = socket.fromfd(receivedFd,
socket.AF_UNIX,
socket.SOCK_STREAM, 0)
os.close(receivedFd)
self.trackedFdsDict[self.remoteControlSocket.fileno()] = \
self.remoteControlSocket
else:
raise Exception('Unhandled type info on received fd: %s' %
(repr(receivedTypeInfo)))
def test5unix_socket_log_data_resource(self):
"""
In this case the log_stream_fd is -1. The next step is to open the stream successfully.
Therefor a server socket is set up listen to data to the server. Afterwards the buffer object is filled with data and the position
is updated.
"""
sockName = b'/tmp/test5unixSocket.sock'
# skipcq: BAN-B607, BAN-B603
proc = subprocess.Popen(['python3', 'unit/input/client.py'])
if os.path.exists(sockName):
os.remove(sockName)
print("Opening socket...")
server = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
server.bind(sockName)
server.listen(1)
connection = server.accept()[0]
unix_socket_log_data_resource = UnixSocketLogDataResource(
b'unix://' + sockName, connection.fileno())
print("Listening...")
unix_socket_log_data_resource.fill_buffer()
self.assertEqual(repr(unix_socket_log_data_resource.buffer),
repr(b'data'))
print('Data received: %s' %
unix_socket_log_data_resource.buffer.decode())
unix_socket_log_data_resource.update_position(
len(unix_socket_log_data_resource.buffer))
self.assertEqual(unix_socket_log_data_resource.total_consumed_length,
4)
self.assertEqual(unix_socket_log_data_resource.buffer, b'')
print("Shutting down...")
unix_socket_log_data_resource.close()
server.close()
proc.terminate()
proc.wait()
print("Done")
def main():
"""Run the aminer main program."""
# Extract program name, but only when sure to contain no problematic characters.
warnings.filterwarnings('ignore', category=ImportWarning)
program_name = sys.argv[0].split('/')[-1]
if (program_name == '.') or (program_name == '..') or (re.match(
'^[a-zA-Z0-9._-]+$', program_name) is None):
print('Invalid program name, check your execution args',
file=sys.stderr)
sys.exit(1)
# We will not read stdin from here on, so get rid of it immediately, thus aberrant child cannot manipulate caller's stdin using it.
stdin_fd = os.open('/dev/null', os.O_RDONLY)
os.dup2(stdin_fd, 0)
os.close(stdin_fd)
help_message = 'aminer - logdata-anomaly-miner\n'
if supports_color():
help_message += colflame
else:
help_message += flame
parser = argparse.ArgumentParser(
description=help_message,
formatter_class=argparse.RawTextHelpFormatter)
parser.add_argument('-v',
'--version',
action='version',
version=__version_string__)
parser.add_argument('-u',
'--check-updates',
action='store_true',
help='check if updates for the aminer are available.')
parser.add_argument('-c',
'--config',
default='/etc/aminer/config.yml',
type=str,
help='path to the config-file')
parser.add_argument('-D',
'--daemon',
action='store_false',
help='run as a daemon process')
parser.add_argument(
'-s',
'--stat',
choices=["0", "1", "2"],
type=str,
help=
'set the stat level. Possible stat-levels are 0 for no statistics, 1 for normal statistic level and 2 for '
'verbose statistics.')
parser.add_argument(
'-d',
'--debug',
choices=["0", "1", "2"],
type=str,
help=
'set the debug level. Possible debug-levels are 0 for no debugging, 1 for normal output (INFO and above), 2 '
'for printing all debug information.')
parser.add_argument('--run-analysis',
action='store_true',
help='enable/disable analysis')
parser.add_argument('-C',
'--clear',
action='store_true',
help='removes all persistence directories')
parser.add_argument('-r',
'--remove',
action='append',
type=str,
help='removes a specific persistence directory')
parser.add_argument('-R',
'--restore',
type=str,
help='restore a persistence backup')
parser.add_argument(
'-f',
'--from-begin',
action='store_true',
help='removes RepositioningData before starting the aminer')
parser.add_argument(
'-o',
'--offline-mode',
action='store_true',
help='stop the aminer after all logs have been processed.')
parser.add_argument(
"--config-properties",
metavar="KEY=VALUE",
nargs='+',
help=
"Set a number of config_properties by using key-value pairs (do not put spaces before or after the = sign). "
"If a value contains spaces, you should define it with double quotes: 'foo=\"this is a sentence\". Note that "
"values are always treated as strings. If values are already defined in the config_properties, the input "
"types are converted to the ones already existing.")
args = parser.parse_args()
if args.check_updates:
import urllib3
url = 'https://raw.githubusercontent.com/ait-aecid/logdata-anomaly-miner/main/source/root/usr/lib/logdata-anomaly-miner/metadata.py'
http = urllib3.PoolManager()
r = http.request('GET', url, preload_content=True)
metadata = r.data.decode()
http.clear()
lines = metadata.split('\n')
curr_version = None
for line in lines:
if '__version__ = ' in line:
curr_version = line.split('__version__ = ')[1].strip('"')
break
if __version__ == curr_version:
print("The current aminer version %s is installed." % curr_version)
else:
print(
"A new aminer version exists (%s). Currently version %s is installed."
% (curr_version, __version__))
print("Use git pull to update the aminer version.")
sys.exit(0)
config_file_name = args.config
run_in_foreground_flag = args.daemon
run_analysis_child_flag = args.run_analysis
clear_persistence_flag = args.clear
remove_persistence_dirs = args.remove
from_begin_flag = args.from_begin
global offline_mode # skipcq: PYL-W0603
offline_mode = args.offline_mode
if args.restore is not None and ('.' in args.restore
or '/' in args.restore):
parser.error('The restore path %s must not contain any . or /' %
args.restore)
if args.remove is not None:
for remove in args.remove:
if '.' in remove or '/' in remove:
parser.error('The remove path %s must not contain any . or /' %
remove)
restore_relative_persistence_path = args.restore
stat_level = 1
debug_level = 1
stat_level_console_flag = False
debug_level_console_flag = False
if args.stat is not None:
stat_level = int(args.stat)
stat_level_console_flag = True
if args.debug is not None:
debug_level = int(args.debug)
debug_level_console_flag = True
# Load the main configuration file.
if not os.path.exists(config_file_name):
print('%s: config "%s" not (yet) available!' %
(program_name, config_file_name),
file=sys.stderr)
sys.exit(1)
# using the solution here to override config_properties:
# https://stackoverflow.com/questions/27146262/create-variable-key-value-pairs-with-argparse-python
use_temp_config = False
config_properties = parse_vars(args.config_properties)
if args.config_properties and "LearnMode" in config_properties:
ymlext = [".YAML", ".YML", ".yaml", ".yml"]
extension = os.path.splitext(config_file_name)[1]
if extension in ymlext:
use_temp_config = True
fd, temp_config = tempfile.mkstemp(suffix=".yml")
with open(config_file_name) as f:
for line in f:
if "LearnMode" in line:
line = "LearnMode: %s" % config_properties["LearnMode"]
os.write(fd, line.encode())
config_file_name = temp_config
os.close(fd)
else:
msg = "The LearnMode parameter does not exist in .py configs!"
print(msg, sys.stderr)
logging.getLogger(AminerConfig.DEBUG_LOG_NAME).error(msg)
sys.exit(1)
# Minimal import to avoid loading too much within the privileged process.
try:
aminer_config = AminerConfig.load_config(config_file_name)
if use_temp_config:
os.remove(config_file_name)
config_file_name = args.config
except ValueError:
sys.exit(1)
for config_property in config_properties:
if config_property == "LearnMode":
continue
old_value = aminer_config.config_properties.get(config_property)
value = config_properties[config_property]
if old_value is not None:
try:
if isinstance(old_value, bool):
if value == "True":
value = True
elif value == "False":
value = False
else:
msg = "The %s parameter must be of type %s!" % (
config_property, type(old_value))
print(msg, sys.stderr)
logging.getLogger(
AminerConfig.DEBUG_LOG_NAME).error(msg)
sys.exit(1)
elif isinstance(old_value, int):
value = int(value)
elif isinstance(old_value, float):
value = float(value)
elif isinstance(old_value, list):
value = ast.literal_eval(value)
except ValueError:
msg = "The %s parameter must be of type %s!" % (
config_property, type(old_value))
print(msg, sys.stderr)
logging.getLogger(AminerConfig.DEBUG_LOG_NAME).error(msg)
sys.exit(1)
else:
msg = "The %s parameter is not set in the config. It will be treated as a string!" % config_property
print("WARNING: " + msg, sys.stderr)
logging.getLogger(AminerConfig.DEBUG_LOG_NAME).warning(msg)
aminer_config.config_properties[config_property] = value
persistence_dir = aminer_config.config_properties.get(
AminerConfig.KEY_PERSISTENCE_DIR, AminerConfig.DEFAULT_PERSISTENCE_DIR)
child_user_name = aminer_config.config_properties.get(
AminerConfig.KEY_AMINER_USER)
child_group_name = aminer_config.config_properties.get(
AminerConfig.KEY_AMINER_GROUP)
child_user_id = -1
child_group_id = -1
try:
if child_user_name is not None:
child_user_id = getpwnam(child_user_name).pw_uid
if child_group_name is not None:
child_group_id = getgrnam(child_group_name).gr_gid
except: # skipcq: FLK-E722
print('Failed to resolve %s or %s' %
(AminerConfig.KEY_AMINER_USER, AminerConfig.KEY_AMINER_GROUP),
file=sys.stderr)
sys.exit(1)
if not stat_level_console_flag and AminerConfig.KEY_LOG_STAT_LEVEL in aminer_config.config_properties:
stat_level = aminer_config.config_properties[
AminerConfig.KEY_LOG_STAT_LEVEL]
if not debug_level_console_flag and AminerConfig.KEY_LOG_DEBUG_LEVEL in aminer_config.config_properties:
debug_level = aminer_config.config_properties[
AminerConfig.KEY_LOG_DEBUG_LEVEL]
if AminerConfig.CONFIG_KEY_ENCODING in aminer_config.config_properties:
AminerConfig.ENCODING = aminer_config.config_properties[
AminerConfig.CONFIG_KEY_ENCODING]
AminerConfig.STAT_LEVEL = stat_level
AminerConfig.DEBUG_LEVEL = debug_level
initialize_loggers(aminer_config, child_user_id, child_group_id)
if restore_relative_persistence_path is not None and (
clear_persistence_flag or remove_persistence_dirs):
msg = 'The --restore parameter removes all persistence files. Do not use this parameter with --Clear or --Remove!'
print(msg, sys.stderr)
logging.getLogger(AminerConfig.DEBUG_LOG_NAME).error(msg)
sys.exit(1)
if clear_persistence_flag:
if remove_persistence_dirs:
msg = 'The --clear and --remove arguments must not be used together!'
print(msg, file=sys.stderr)
logging.getLogger(AminerConfig.DEBUG_LOG_NAME).error(msg)
sys.exit(1)
clear_persistence(persistence_dir)
if remove_persistence_dirs:
persistence_dir_name = aminer_config.config_properties.get(
AminerConfig.KEY_PERSISTENCE_DIR,
AminerConfig.DEFAULT_PERSISTENCE_DIR)
for filename in os.listdir(persistence_dir_name):
file_path = os.path.join(persistence_dir_name, filename)
try:
if not os.path.isdir(file_path):
msg = 'The aminer persistence directory should not contain any files.'
print(msg, file=sys.stderr)
logging.getLogger(AminerConfig.DEBUG_LOG_NAME).warning(msg)
continue
shutil.rmtree(file_path)
except OSError as e:
msg = 'Failed to delete %s. Reason: %s' % (file_path, e)
print(msg, file=sys.stderr)
logging.getLogger(AminerConfig.DEBUG_LOG_NAME).error(msg)
for filename in remove_persistence_dirs:
file_path = os.path.join(persistence_dir, filename)
try:
if not os.path.exists(file_path):
continue
if not os.path.isdir(file_path):
msg = 'The aminer persistence directory should not contain any files.'
print(msg, file=sys.stderr)
logging.getLogger(AminerConfig.DEBUG_LOG_NAME).warning(msg)
continue
shutil.rmtree(file_path)
except OSError as e:
msg = 'Failed to delete %s. Reason: %s' % (file_path, e)
print(msg, file=sys.stderr)
logging.getLogger(AminerConfig.DEBUG_LOG_NAME).error(msg)
if restore_relative_persistence_path is not None:
absolute_persistence_path = os.path.join(
persistence_dir, 'backup', restore_relative_persistence_path)
if not os.path.exists(absolute_persistence_path):
msg = '%s does not exist. Continuing without restoring persistence.' % absolute_persistence_path
print(msg, file=sys.stderr)
logging.getLogger(AminerConfig.DEBUG_LOG_NAME).warning(msg)
else:
clear_persistence(persistence_dir)
copytree(absolute_persistence_path, persistence_dir)
persistence_dir_fd = SecureOSFunctions.secure_open_base_directory(
persistence_dir, os.O_RDONLY | os.O_DIRECTORY | os.O_PATH)
for dirpath, _dirnames, filenames in os.walk(persistence_dir):
os.chown(dirpath,
child_user_id,
child_group_id,
dir_fd=persistence_dir_fd,
follow_symlinks=False)
for filename in filenames:
os.chown(os.path.join(dirpath, filename),
child_user_id,
child_user_id,
dir_fd=persistence_dir_fd,
follow_symlinks=False)
if from_begin_flag:
repositioning_data_path = os.path.join(
aminer_config.config_properties.get(
AminerConfig.KEY_PERSISTENCE_DIR,
AminerConfig.DEFAULT_PERSISTENCE_DIR), 'AnalysisChild',
'RepositioningData')
if os.path.exists(repositioning_data_path):
os.remove(repositioning_data_path)
if run_analysis_child_flag:
# Call analysis process, this function will never return.
run_analysis_child(aminer_config, program_name)
# Start importing of aminer specific components after reading of "config.py" to allow replacement of components via sys.path
# from within configuration.
log_sources_list = aminer_config.config_properties.get(
AminerConfig.KEY_LOG_SOURCES_LIST)
if (log_sources_list is None) or not log_sources_list:
msg = '%s: %s not defined' % (program_name,
AminerConfig.KEY_LOG_SOURCES_LIST)
print(msg, file=sys.stderr)
logging.getLogger(AminerConfig.DEBUG_LOG_NAME).error(msg)
sys.exit(1)
# Now create the management entries for each logfile.
log_data_resource_dict = {}
for log_resource_name in log_sources_list:
# From here on log_resource_name is a byte array.
log_resource_name = decode_string_as_byte_string(log_resource_name)
log_resource = None
if log_resource_name.startswith(b'file://'):
log_resource = FileLogDataResource(log_resource_name, -1)
elif log_resource_name.startswith(b'unix://'):
log_resource = UnixSocketLogDataResource(log_resource_name, -1)
else:
msg = 'Unsupported schema in %s: %s' % (
AminerConfig.KEY_LOG_SOURCES_LIST, repr(log_resource_name))
print(msg, file=sys.stderr)
logging.getLogger(AminerConfig.DEBUG_LOG_NAME).error(msg)
sys.exit(1)
if not os.path.exists(log_resource_name[7:].decode()):
msg = "WARNING: file or socket '%s' does not exist (yet)!" % log_resource_name[
7:].decode()
print(msg, file=sys.stderr)
logging.getLogger(AminerConfig.DEBUG_LOG_NAME).warning(msg)
try:
log_resource.open()
except OSError as open_os_error:
if open_os_error.errno == errno.EACCES:
msg = '%s: no permission to access %s' % (
program_name, repr(log_resource_name))
print(msg, file=sys.stderr)
logging.getLogger(AminerConfig.DEBUG_LOG_NAME).error(msg)
sys.exit(1)
else:
msg = '%s: unexpected error opening %s: %d (%s)' % (
program_name, repr(log_resource_name), open_os_error.errno,
os.strerror(open_os_error.errno))
print(msg, file=sys.stderr)
logging.getLogger(AminerConfig.DEBUG_LOG_NAME).error(msg)
sys.exit(1)
log_data_resource_dict[log_resource_name] = log_resource
# Create the remote control socket, if any. Do this in privileged mode to allow binding it at arbitrary locations and support restricted
# permissions of any type for current (privileged) uid.
remote_control_socket_name = aminer_config.config_properties.get(
AminerConfig.KEY_REMOTE_CONTROL_SOCKET_PATH, None)
remote_control_socket = None
if remote_control_socket_name is not None:
if os.path.exists(remote_control_socket_name):
try:
os.unlink(remote_control_socket_name)
except OSError:
msg = 'Failed to clean up old remote control socket at %s' % remote_control_socket_name
print(msg, file=sys.stderr)
logging.getLogger(AminerConfig.DEBUG_LOG_NAME).error(msg)
sys.exit(1)
# Create the local socket: there is no easy way to create it with correct permissions, hence a fork is needed, setting umask,
# bind the socket. It is also recommended to create the socket in a directory having the correct permissions already.
remote_control_socket = socket.socket(socket.AF_UNIX,
socket.SOCK_STREAM)
remote_control_socket.setblocking(False)
bind_child_pid = os.fork()
if bind_child_pid == 0:
os.umask(0o177)
remote_control_socket.bind(remote_control_socket_name)
# Do not perform any cleanup, flushing of streams. Use _exit(0) to avoid interference with fork.
os._exit(0) # skipcq: PYL-W0212
os.waitpid(bind_child_pid, 0)
remote_control_socket.listen(4)
# Now have checked all we can get from the configuration in the privileged process. Detach from the TTY when in daemon mode.
if not run_in_foreground_flag:
child_pid = 0
try:
# Fork a child to make sure, we are not the process group leader already.
child_pid = os.fork()
except Exception as fork_exception: # skipcq: PYL-W0703
msg = 'Failed to daemonize: %s' % fork_exception
print(msg, file=sys.stderr)
logging.getLogger(AminerConfig.DEBUG_LOG_NAME).error(msg)
sys.exit(1)
if child_pid != 0:
# This is the parent.
os._exit(0) # skipcq: PYL-W0212
# This is the child. Create a new session and become process group leader. Here we get rid of the controlling tty.
os.setsid()
# Fork again to become an orphaned process not being session leader, hence not able to get a controlling tty again.
try:
child_pid = os.fork()
except Exception as fork_exception: # skipcq: PYL-W0703
msg = 'Failed to daemonize: %s' % fork_exception
print(msg, file=sys.stderr)
logging.getLogger(AminerConfig.DEBUG_LOG_NAME).error(msg)
sys.exit(1)
if child_pid != 0:
# This is the parent.
os._exit(0) # skipcq: PYL-W0212
# Move to root directory to avoid lingering in some cwd someone else might want to unmount.
os.chdir('/')
# Change the umask here to clean all group/other mask bits so that accidentially created files are not accessible by other.
os.umask(0o77)
# Install a signal handler catching common stop signals and relaying it to all children for sure.
# skipcq: PYL-W0603
global child_termination_triggered_flag
child_termination_triggered_flag = False
def graceful_shutdown_handler(_signo, _stackFrame):
"""React on typical shutdown signals."""
msg = '%s: caught signal, shutting down' % program_name
print(msg, file=sys.stderr)
logging.getLogger(AminerConfig.DEBUG_LOG_NAME).info(msg)
# Just set the flag. It is likely, that child received same signal also so avoid multiple signaling, which could interrupt the
# shutdown procedure again.
# skipcq: PYL-W0603
global child_termination_triggered_flag
child_termination_triggered_flag = True
import signal
signal.signal(signal.SIGHUP, graceful_shutdown_handler)
signal.signal(signal.SIGINT, graceful_shutdown_handler)
signal.signal(signal.SIGTERM, graceful_shutdown_handler)
# Now create the socket to connect the analysis child.
(parent_socket, child_socket) = socket.socketpair(socket.AF_UNIX,
socket.SOCK_DGRAM, 0)
# Have it nonblocking from here on.
parent_socket.setblocking(False)
child_socket.setblocking(False)
# Use normal fork, we should have been detached from TTY already. Flush stderr to avoid duplication of output if both child and
# parent want to write something.
sys.stderr.flush()
child_pid = os.fork()
if child_pid == 0:
# Relocate the child socket fd to 3 if needed
if child_socket.fileno() != 3:
os.dup2(child_socket.fileno(), 3)
child_socket.close()
# Clear the supplementary groups before dropping privileges. This makes only sense when changing the uid or gid.
if os.getuid() == 0:
if ((child_user_id != -1) and (child_user_id != os.getuid())) or (
(child_group_id != -1) and (child_group_id != os.getgid())):
os.setgroups([])
# Drop privileges before executing child. setuid/gid will raise an exception when call has failed.
if child_group_id != -1:
os.setgid(child_group_id)
if child_user_id != -1:
os.setuid(child_user_id)
else:
msg = 'INFO: No privilege separation when started as unprivileged user'
print(msg, file=sys.stderr)
tmp_username = aminer_config.config_properties.get(
AminerConfig.KEY_AMINER_USER)
tmp_group = aminer_config.config_properties.get(
AminerConfig.KEY_AMINER_GROUP)
aminer_user_id = -1
aminer_group_id = -1
try:
if tmp_username is not None:
aminer_user_id = getpwnam(tmp_username).pw_uid
if tmp_group is not None:
aminer_group_id = getgrnam(tmp_group).gr_gid
except: # skipcq: FLK-E722
print('Failed to resolve %s or %s' %
(AminerConfig.KEY_AMINER_USER,
AminerConfig.KEY_AMINER_GROUP),
file=sys.stderr)
sys.exit(1)
initialize_loggers(aminer_config, aminer_user_id, aminer_group_id)
logging.getLogger(AminerConfig.DEBUG_LOG_NAME).info(msg)
# Now resolve the specific analysis configuration file (if any).
analysis_config_file_name = aminer_config.config_properties.get(
AminerConfig.KEY_ANALYSIS_CONFIG_FILE, None)
if analysis_config_file_name is None:
analysis_config_file_name = config_file_name
elif not os.path.isabs(analysis_config_file_name):
analysis_config_file_name = os.path.join(
os.path.dirname(config_file_name), analysis_config_file_name)
# This is the child. Close all parent file descriptors, we do not need. Perhaps this could be done more elegantly.
for close_fd in range(4, 1 << 16):
try:
os.close(close_fd)
except OSError as open_os_error:
if open_os_error.errno == errno.EBADF:
continue
msg = '%s: unexpected exception closing file descriptors: %s' % (
program_name, open_os_error)
print(msg, file=sys.stderr)
logging.getLogger(AminerConfig.DEBUG_LOG_NAME).error(msg)
# Flush stderr before exit without any cleanup.
sys.stderr.flush()
os._exit(1) # skipcq: PYL-W0212
# Now execute the very same program again, but user might have moved or renamed it meanwhile. This would be problematic with
# SUID-binaries (which we do not yet support). Do NOT just fork but also exec to avoid child circumventing
# parent's ALSR due to cloned kernel VMA.
exec_args = [
'aminerChild', '--run-analysis', '--config',
analysis_config_file_name, '--stat',
str(stat_level), '--debug',
str(debug_level)
]
if offline_mode:
exec_args.append("--offline-mode")
if args.config_properties:
exec_args.append("--config-properties")
for config_property in args.config_properties:
exec_args.append(config_property)
os.execv(sys.argv[0], exec_args) # skipcq: BAN-B606
msg = 'Failed to execute child process'
print(msg, file=sys.stderr)
logging.getLogger(AminerConfig.DEBUG_LOG_NAME).error(msg)
sys.stderr.flush()
os._exit(1) # skipcq: PYL-W0212
child_socket.close()
# Send all log resource information currently available to child process.
for log_resource_name, log_resource in log_data_resource_dict.items():
if (log_resource
is not None) and (log_resource.get_file_descriptor() >= 0):
SecureOSFunctions.send_logstream_descriptor(
parent_socket, log_resource.get_file_descriptor(),
log_resource_name)
log_resource.close()
# Send the remote control server socket, if any and close it afterwards. It is not needed any more on parent side.
if remote_control_socket is not None:
SecureOSFunctions.send_annotated_file_descriptor(
parent_socket, remote_control_socket.fileno(), 'remotecontrol', '')
remote_control_socket.close()
exit_status = 0
child_termination_triggered_count = 0
while True:
if child_termination_triggered_flag:
if child_termination_triggered_count == 0:
time.sleep(1)
elif child_termination_triggered_count < 5:
os.kill(child_pid, signal.SIGTERM)
else:
os.kill(0, signal.SIGKILL)
child_termination_triggered_count += 1
(sig_child_pid, sig_status) = os.waitpid(-1, os.WNOHANG)
if sig_child_pid != 0:
if sig_child_pid == child_pid:
if child_termination_triggered_flag or offline_mode:
# This was expected, just terminate.
break
msg = '%s: Analysis child process %d terminated unexpectedly with signal 0x%x' % (
program_name, sig_child_pid, sig_status)
print(msg, file=sys.stderr)
logging.getLogger(AminerConfig.DEBUG_LOG_NAME).error(msg)
exit_status = 1
break
# So the child has been cloned, the clone has terminated. This should not happen either.
msg = '%s: untracked child %d terminated with with signal 0x%x' % (
program_name, sig_child_pid, sig_status)
print(msg, file=sys.stderr)
logging.getLogger(AminerConfig.DEBUG_LOG_NAME).error(msg)
exit_status = 1
# Child information handled, scan for rotated logfiles or other resources, where reopening might make sense.
for log_resouce_name, log_data_resource in log_data_resource_dict.items(
):
try:
if not log_data_resource.open(reopen_flag=True):
continue
except OSError as open_os_error:
if open_os_error.errno == errno.EACCES:
msg = '%s: no permission to access %s' % (program_name,
log_resouce_name)
print(msg, file=sys.stderr)
logging.getLogger(AminerConfig.DEBUG_LOG_NAME).error(msg)
else:
msg = '%s: unexpected error reopening %s: %d (%s)' % (
program_name, log_resouce_name, open_os_error.errno,
os.strerror(open_os_error.errno))
print(msg, file=sys.stderr)
logging.getLogger(AminerConfig.DEBUG_LOG_NAME).error(msg)
exit_status = 2
continue
SecureOSFunctions.send_logstream_descriptor(
parent_socket, log_data_resource.get_file_descriptor(),
log_resouce_name)
log_data_resource.close()
time.sleep(1)
parent_socket.close()
SecureOSFunctions.close_base_directory()
SecureOSFunctions.close_log_directory()
sys.exit(exit_status)