async def produce(self):
"""
The main function of this class and subclasses. Runs a loop, sleeping
for the polling duration then making an API call, consuming the logs
from that API call and saving the offset of the latest log read.
"""
# Exit when DuoLogSync is shutting down (due to error or Ctrl-C)
while Program.is_running():
shutdown_reason = None
Program.log(
f"{self.log_type} producer: fetching next logs after "
f"{Config.get_api_timeout()} seconds", logging.INFO)
try:
# Sleep for api_timeout amount of time, but check for program
# shutdown every second
await restless_sleep(Config.get_api_timeout())
Program.log(f"{self.log_type} producer: fetching logs",
logging.INFO)
api_result = await self.call_log_api()
if api_result:
await self.add_logs_to_queue(self.get_logs(api_result))
else:
Program.log(
f"{self.log_type} producer: no new logs available",
logging.INFO)
# Incorrect api_hostname or proxy_server was provided
# duo_client throws the same error if either the api_hostname or proxy_server is incorrect
except (gaierror, OSError) as error:
shutdown_reason = f"{self.log_type} producer: [{error}]"
Program.log(
'DuoLogSync: check that the duoclient host and/or proxy_server '
'provided in the config file is correct')
# duo_client throws a RuntimeError if the ikey or skey is invalid
except RuntimeError as runtime_error:
shutdown_reason = f"{self.log_type} producer: [{runtime_error}]"
Program.log('DuoLogSync: check that the duoclient ikey and '
'skey in the config file are correct')
# Shutdown hath been noticed and thus shutdown shall begin
except ProgramShutdownError:
break
if shutdown_reason:
Program.initiate_shutdown(shutdown_reason)
# Unblock consumer but putting anything in the shared queue
await self.log_queue.put([])
Program.log(f"{self.log_type} producer: shutting down", logging.INFO)
async def restless_sleep(duration):
"""
Wrapper for the asyncio.sleep function to sleep for duration seconds
but check every second that DuoLogSync is still running. This is
necessary in the case that the program should be shutting down but
a producer is in the middle of a 2 minute poll and will not be aware
of program shutdown until much later.
@param duration The number of seconds to sleep for
"""
while duration > 0:
await asyncio.sleep(1)
# Poll for program running state
if Program.is_running():
duration = duration - 1
continue
# Otherwise, program is done running, raise an exception to be caught
raise ProgramShutdownError
async def consume(self):
"""
Consumer that will consume data from a queue shared with a producer
object. Data from the queue is then sent over a configured transport
protocol to respective SIEMs or servers.
"""
while Program.is_running():
Program.log(f"{self.log_type} consumer: waiting for logs",
logging.INFO)
# Call unblocks only when there is an element in the queue to get
logs = await self.log_queue.get()
# Time to shutdown
if not Program.is_running():
continue
Program.log(
f"{self.log_type} consumer: received {len(logs)} logs "
"from producer", logging.INFO)
# Keep track of the latest log written in the case that a problem
# occurs in the middle of writing logs
last_log_written = None
successful_write = False
# If we are sending empty [] to unblock consumers, nothing should be written to file
if logs:
try:
Program.log(f"{self.log_type} consumer: writing logs",
logging.INFO)
for log in logs:
if self.child_account_id:
log['child_account_id'] = self.child_account_id
await self.writer.write(self.format_log(log))
last_log_written = log
# All the logs were written successfully
successful_write = True
# Specifically watch out for errno 32 - Broken pipe. This means
# that the connect established by writer was reset or shutdown.
except BrokenPipeError as broken_pipe_error:
shutdown_reason = f"{broken_pipe_error}"
Program.initiate_shutdown(shutdown_reason)
Program.log("DuoLogSync: connection to server was reset",
logging.WARNING)
finally:
if successful_write:
Program.log(
f"{self.log_type} consumer: successfully wrote "
"all logs", logging.INFO)
else:
Program.log(
f"{self.log_type} consumer: failed to write "
"some logs", logging.WARNING)
self.log_offset = Producer.get_log_offset(last_log_written)
self.update_log_checkpoint(self.log_type, self.log_offset,
self.child_account_id)
else:
Program.log(f"{self.log_type} consumer: No logs to write",
logging.INFO)
Program.log(f"{self.log_type} consumer: shutting down", logging.INFO)
def test_is_running(self):
self.assertEqual(Program.is_running(), True)
Program._running = False
self.assertEqual(Program.is_running(), False)
