def create_config(cls, config_filepath):
"""
Attemp to read the file at config_filepath and generate a config
Dictionary object based on a defined JSON schema
@param config_filepath File from which to generate a config object
"""
shutdown_reason = None
try:
with open(config_filepath) as config_file:
# PyYAML gives better error messages for streams than for files
config_file_data = config_file.read()
config = yaml.full_load(config_file_data)
# Check config against a schema to ensure all the needed fields
# and values are defined
config = cls._validate_and_normalize_config(config)
if config.get('dls_settings').get('api').get(
'timeout') < cls.API_TIMEOUT_DEFAULT:
config['dls_settings']['api'][
'timeout'] = cls.API_TIMEOUT_DEFAULT
Program.log(
'DuoLogSync: Setting default api timeout to 120 seconds.'
)
# Will occur when given a bad filepath or a bad file
except OSError as os_error:
shutdown_reason = f"{os_error}"
Program.log('DuoLogSync: Failed to open the config file. Check '
'that the filename is correct')
# Will occur if the config file does not contain valid YAML
except YAMLError as yaml_error:
shutdown_reason = f"{yaml_error}"
Program.log('DuoLogSync: Failed to parse the config file. Check '
'that the config file has valid YAML.')
# Validation of the config against a schema failed
except ValueError:
shutdown_reason = f"{cls.SCHEMA_VALIDATOR.errors}"
Program.log('DuoLogSync: Validation of the config file failed. '
'Check that required fields have proper values.')
# No exception raised during the try block, return config
else:
# Calculate offset as a timestamp and rewrite its value in config
offset = config.get('dls_settings').get('api').get('offset')
offset = datetime.utcnow() - timedelta(days=offset)
config['dls_settings']['api']['offset'] = int(offset.timestamp())
return config
# At this point, it is guaranteed that an exception was raised, which
# means that it is shutdown time
Program.initiate_shutdown(shutdown_reason)
return None
async def create_writer(self, host, port, cert_filepath):
"""
Wrapper for functions to create TCP or UDP connections.
@param host Hostname of the network connection to establish
@param port Port of the network connection to establish
@param cert_filepath Path to file containing SSL certificate
@return a 'writer' object for writing data over the connection made
"""
Program.log(f"DuoLogSync: Opening connection to {host}:{port}",
logging.INFO)
# Message to be logged if an error occurs in this function
help_message = (f"DuoLogSync: check that host-{host} and port-{port} "
"are correct in the config file")
writer = None
try:
if self.protocol == 'UDP':
writer = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
elif self.protocol == 'TCPSSL':
ssl_context = ssl.create_default_context(
ssl.Purpose.SERVER_AUTH, cafile=cert_filepath)
writer = await Writer.create_tcp_writer(
host, port, ssl_context)
elif self.protocol == 'TCP':
writer = await Writer.create_tcp_writer(host, port)
# Failed to open the certificate file
except FileNotFoundError:
shutdown_reason = f"{cert_filepath} could not be opened."
help_message = (
'DuoLogSync: Make sure the filepath for SSL cert file is '
'correct.')
# Couldn't establish a connection within 60 seconds
except asyncio.TimeoutError:
shutdown_reason = 'connection to server timed-out after 60 seconds'
# If an invalid hostname or port number is given or simply failed to
# connect using the host and port given
except (gaierror, OSError) as error:
shutdown_reason = f"{error}"
# An error did not occur and the writer was successfully created
else:
return writer
Program.initiate_shutdown(shutdown_reason)
Program.log(help_message, logging.ERROR)
return None
def sigint_handler(signal_number, stack_frame):
"""
Handler for SIGINT (Ctrl-C) to gracefully shutdown DuoLogSync
"""
shutdown_reason = f"received signal {signal_number} (Ctrl-C)"
Program.initiate_shutdown(shutdown_reason)
if stack_frame:
Program.log(f"DuoLogSync: stack frame from Ctrl-C is {stack_frame}",
logging.INFO)
async def produce(self):
"""
The main function of this class and subclasses. Runs a loop, sleeping
for the polling duration then making an API call, consuming the logs
from that API call and saving the offset of the latest log read.
"""
# Exit when DuoLogSync is shutting down (due to error or Ctrl-C)
while Program.is_running():
shutdown_reason = None
Program.log(
f"{self.log_type} producer: fetching next logs after "
f"{Config.get_api_timeout()} seconds", logging.INFO)
try:
# Sleep for api_timeout amount of time, but check for program
# shutdown every second
await restless_sleep(Config.get_api_timeout())
Program.log(f"{self.log_type} producer: fetching logs",
logging.INFO)
api_result = await self.call_log_api()
if api_result:
await self.add_logs_to_queue(self.get_logs(api_result))
else:
Program.log(
f"{self.log_type} producer: no new logs available",
logging.INFO)
# Incorrect api_hostname or proxy_server was provided
# duo_client throws the same error if either the api_hostname or proxy_server is incorrect
except (gaierror, OSError) as error:
shutdown_reason = f"{self.log_type} producer: [{error}]"
Program.log(
'DuoLogSync: check that the duoclient host and/or proxy_server '
'provided in the config file is correct')
# duo_client throws a RuntimeError if the ikey or skey is invalid
except RuntimeError as runtime_error:
shutdown_reason = f"{self.log_type} producer: [{runtime_error}]"
Program.log('DuoLogSync: check that the duoclient ikey and '
'skey in the config file are correct')
# Shutdown hath been noticed and thus shutdown shall begin
except ProgramShutdownError:
break
if shutdown_reason:
Program.initiate_shutdown(shutdown_reason)
# Unblock consumer but putting anything in the shared queue
await self.log_queue.put([])
Program.log(f"{self.log_type} producer: shutting down", logging.INFO)
def connection_lost(self, exc):
shutdown_reason = None
if exc:
shutdown_reason = (
f"UDP connection with host-{self.host} and port-{self.port}"
f"was closed for the following reason [{exc}]")
else:
shutdown_reason = (
f"UDP connection with host-{self.host} and port-{self.port} "
"was closed")
Program.initiate_shutdown(shutdown_reason)
async def consume(self):
"""
Consumer that will consume data from a queue shared with a producer
object. Data from the queue is then sent over a configured transport
protocol to respective SIEMs or servers.
"""
while Program.is_running():
Program.log(f"{self.log_type} consumer: waiting for logs",
logging.INFO)
# Call unblocks only when there is an element in the queue to get
logs = await self.log_queue.get()
# Time to shutdown
if not Program.is_running():
continue
Program.log(
f"{self.log_type} consumer: received {len(logs)} logs "
"from producer", logging.INFO)
# Keep track of the latest log written in the case that a problem
# occurs in the middle of writing logs
last_log_written = None
successful_write = False
# If we are sending empty [] to unblock consumers, nothing should be written to file
if logs:
try:
Program.log(f"{self.log_type} consumer: writing logs",
logging.INFO)
for log in logs:
if self.child_account_id:
log['child_account_id'] = self.child_account_id
await self.writer.write(self.format_log(log))
last_log_written = log
# All the logs were written successfully
successful_write = True
# Specifically watch out for errno 32 - Broken pipe. This means
# that the connect established by writer was reset or shutdown.
except BrokenPipeError as broken_pipe_error:
shutdown_reason = f"{broken_pipe_error}"
Program.initiate_shutdown(shutdown_reason)
Program.log("DuoLogSync: connection to server was reset",
logging.WARNING)
finally:
if successful_write:
Program.log(
f"{self.log_type} consumer: successfully wrote "
"all logs", logging.INFO)
else:
Program.log(
f"{self.log_type} consumer: failed to write "
"some logs", logging.WARNING)
self.log_offset = Producer.get_log_offset(last_log_written)
self.update_log_checkpoint(self.log_type, self.log_offset,
self.child_account_id)
else:
Program.log(f"{self.log_type} consumer: No logs to write",
logging.INFO)
Program.log(f"{self.log_type} consumer: shutting down", logging.INFO)
def test_initiate_shutdown(self):
self.assertEqual(Program._running, True)
Program.initiate_shutdown('test')
self.assertEqual(Program._running, False)
