def PopItem(self):
"""Pops an item off the queue.
If no ZeroMQ socket has been created, one will be created the first
time this method is called.
Raises:
QueueEmpty: If the queue is empty, and no item could be popped within the
queue timeout.
zmq.error.ZMQError: If an error is encountered by ZeroMQ.
"""
logging.debug(u'Pop on {0:s} queue, port {1:d}'.format(
self.name, self.port))
if not self._zmq_socket:
self._CreateZMQSocket()
try:
return self._queue.get(timeout=self._buffer_timeout_seconds)
except Queue.Empty:
return plaso_queue.QueueAbort()
except zmq.error.ZMQError as exception:
if exception.errno == errno.EINTR:
logging.error(u'ZMQ syscall interrupted in {0:s}.'.format(
self.name))
return plaso_queue.QueueAbort()
else:
raise
except KeyboardInterrupt:
self.Close(abort=True)
raise
def _StopAnalysisProcesses(self, abort=False):
"""Stops the analysis processes.
Args:
abort (bool): True to indicated the stop is issued on abort.
"""
logger.debug('Stopping analysis processes.')
self._StopMonitoringProcesses()
if abort:
# Signal all the processes to abort.
self._AbortTerminate()
# Wake the processes to make sure that they are not blocking
# waiting for the queue new items.
for event_queue in self._event_queues.values():
event_queue.PushItem(plaso_queue.QueueAbort(), block=False)
# Try waiting for the processes to exit normally.
self._AbortJoin(timeout=self._PROCESS_JOIN_TIMEOUT)
for event_queue in self._event_queues.values():
event_queue.Close(abort=abort)
if abort:
# Kill any remaining processes.
self._AbortKill()
else:
# Check if the processes are still alive and terminate them if necessary.
self._AbortTerminate()
self._AbortJoin(timeout=self._PROCESS_JOIN_TIMEOUT)
for event_queue in self._event_queues.values():
event_queue.Close(abort=True)
def testMain(self):
"""Tests the _Main function."""
output_task_queue = zeromq_queue.ZeroMQBufferedReplyBindQueue(
delay_open=True,
linger_seconds=0,
maximum_items=1,
name='test output task queue',
timeout_seconds=self._QUEUE_TIMEOUT)
output_task_queue.Open()
input_task_queue = zeromq_queue.ZeroMQRequestConnectQueue(
delay_open=True,
linger_seconds=0,
name='test input task queue',
port=output_task_queue.port,
timeout_seconds=self._QUEUE_TIMEOUT)
configuration = configurations.ProcessingConfiguration()
test_process = worker_process.WorkerProcess(input_task_queue,
None,
None,
None,
None,
configuration,
name='TestWorker')
test_process.start()
output_task_queue.PushItem(plaso_queue.QueueAbort(), block=False)
output_task_queue.Close(abort=True)
def testMain(self):
"""Tests the _Main function."""
output_event_queue = zeromq_queue.ZeroMQPushBindQueue(
name='test output event queue', timeout_seconds=self._QUEUE_TIMEOUT)
output_event_queue.Open()
input_event_queue = zeromq_queue.ZeroMQPullConnectQueue(
name='test input event queue', delay_open=True,
port=output_event_queue.port,
timeout_seconds=self._QUEUE_TIMEOUT)
session = sessions.Session()
storage_writer = self._CreateStorageWriter(session)
analysis_plugin = TestAnalysisPlugin()
configuration = configurations.ProcessingConfiguration()
test_process = analysis_process.AnalysisProcess(
input_event_queue, storage_writer, None, analysis_plugin, configuration,
name='TestAnalysis')
test_process._FOREMAN_STATUS_WAIT = 1
test_process.start()
output_event_queue.PushItem(plaso_queue.QueueAbort(), block=False)
output_event_queue.Close(abort=True)
def _ZeroMQResponder(self, source_queue, socket, terminate_event):
"""Listens for requests and replies to clients.
Args:
source_queue: The queue to uses to pull items from.
socket: The socket to listen to, and send responses to.
terminate_event: The event that signals that the queue should terminate.
Raises:
QueueEmpty: If a timeout occurs when trying to reply to a request.
zmq.error.ZMQError: If an error occurs in ZeroMQ.
"""
logging.debug(u'ZeroMQ responder thread started')
while not terminate_event.isSet():
try:
# We need to receive a request before we send.
_ = socket.recv()
except zmq.error.Again:
logging.debug(u'{0:s} did not receive a request within the '
u'timeout of {1:d} seconds.'.format(
self.name, self.timeout_seconds))
continue
except zmq.error.ZMQError as exception:
if exception.errno == errno.EINTR:
logging.error(u'ZMQ syscall interrupted in {0:s}.'.format(
self.name))
break
else:
raise
try:
if self._closed:
item = source_queue.get_nowait()
else:
item = source_queue.get(True, self._buffer_timeout_seconds)
except Queue.Empty:
item = plaso_queue.QueueAbort()
try:
self._zmq_socket.send_pyobj(item)
except zmq.error.Again:
logging.debug(
u'{0:s} could not reply to a request within {1:d} seconds.'
.format(self.name, self.timeout_seconds))
raise errors.QueueEmpty
except zmq.error.ZMQError as exception:
if exception.errno == errno.EINTR:
logging.error(u'ZMQ syscall interrupted in {0:s}.'.format(
self.name))
break
else:
raise
socket.close(self._linger_seconds)
def _ZeroMQResponder(self, source_queue, socket, terminate_event):
"""Listens for requests and replies to clients.
Args:
source_queue: The queue to uses to pull items from.
socket: The socket to listen to, and send responses to.
terminate_event: The event that signals that the queue should terminate.
Raises:
zmq.error.ZMQError: If an error is encountered by ZeroMQ.
"""
logging.debug(u'ZeroMQ responder thread started')
while not terminate_event.isSet():
try:
item = socket.recv_pyobj()
except zmq.error.Again:
# No item received within timeout.
item = plaso_queue.QueueAbort()
except zmq.error.ZMQError as exception:
if exception.errno == errno.EINTR:
logging.error(u'ZMQ syscall interrupted in {0:s}.'.format(
self.name))
break
else:
raise
retries = 0
while retries < self._DOWNSTREAM_QUEUE_MAX_TRIES:
try:
self._queue.put(item, timeout=self._buffer_timeout_seconds)
break
except Queue.Full:
logging.debug(u'Queue {0:s} buffer limit hit.'.format(
self.name))
retries += 1
if retries >= self._DOWNSTREAM_QUEUE_MAX_TRIES:
logging.error(
u'Queue {0:s} unserved for too long, aborting'.
format(self.name))
break
else:
time.sleep(self._DOWNSTREAM_QUEUE_SLEEP_TIME)
continue
logging.info(u'Queue {0:s} responder exiting.'.format(self.name))
def _StopExtractionProcesses(self, abort=False):
"""Stops the extraction processes.
Args:
abort (bool): True to indicated the stop is issued on abort.
"""
logger.debug('Stopping extraction processes.')
self._StopMonitoringProcesses()
# Note that multiprocessing.Queue is very sensitive regarding
# blocking on either a get or a put. So we try to prevent using
# any blocking behavior.
if abort:
# Signal all the processes to abort.
self._AbortTerminate()
logger.debug('Emptying task queue.')
self._task_queue.Empty()
# Wake the processes to make sure that they are not blocking
# waiting for the queue new items.
for _ in self._processes_per_pid:
try:
self._task_queue.PushItem(plaso_queue.QueueAbort(),
block=False)
except errors.QueueFull:
logger.warning(
'Task queue full, unable to push abort message.')
# Try waiting for the processes to exit normally.
self._AbortJoin(timeout=self._PROCESS_JOIN_TIMEOUT)
self._task_queue.Close(abort=abort)
if not abort:
# Check if the processes are still alive and terminate them if necessary.
self._AbortTerminate()
self._AbortJoin(timeout=self._PROCESS_JOIN_TIMEOUT)
self._task_queue.Close(abort=True)
# Kill any lingering processes.
self._AbortKill()
def _StopAnalysisProcesses(self, abort=False):
"""Stops the analysis processes.
Args:
abort (bool): True to indicated the stop is issued on abort.
"""
logger.debug('Stopping analysis processes.')
self._StopMonitoringProcesses()
# Note that multiprocessing.Queue is very sensitive regarding
# blocking on either a get or a put. So we try to prevent using
# any blocking behavior.
if abort:
# Signal all the processes to abort.
self._AbortTerminate()
if not self._use_zeromq:
logger.debug('Emptying queues.')
for event_queue in self._event_queues.values():
event_queue.Empty()
# Wake the processes to make sure that they are not blocking
# waiting for the queue new items.
for event_queue in self._event_queues.values():
event_queue.PushItem(plaso_queue.QueueAbort(), block=False)
# Try waiting for the processes to exit normally.
self._AbortJoin(timeout=self._PROCESS_JOIN_TIMEOUT)
for event_queue in self._event_queues.values():
event_queue.Close(abort=abort)
if abort:
# Kill any remaining processes.
self._AbortKill()
else:
# Check if the processes are still alive and terminate them if necessary.
self._AbortTerminate()
self._AbortJoin(timeout=self._PROCESS_JOIN_TIMEOUT)
for event_queue in self._event_queues.values():
event_queue.Close(abort=True)
def testMain(self):
"""Tests the _Main function."""
output_event_queue = zeromq_queue.ZeroMQPushBindQueue(
name='test output event queue',
timeout_seconds=self._QUEUE_TIMEOUT)
output_event_queue.Open()
input_event_queue = zeromq_queue.ZeroMQPullConnectQueue(
name='test input event queue',
delay_open=True,
port=output_event_queue.port,
timeout_seconds=self._QUEUE_TIMEOUT)
session = sessions.Session()
analysis_plugin = TestAnalysisPlugin()
with shared_test_lib.TempDirectory() as temp_directory:
# Set up the processed for the task storage file generated by the
# analysis plugin.
os.mkdir(os.path.join(temp_directory, 'processed'))
configuration = configurations.ProcessingConfiguration()
configuration.task_storage_path = temp_directory
test_process = analysis_process.AnalysisProcess(
input_event_queue,
None,
session,
analysis_plugin,
configuration,
name='TestAnalysis')
test_process._FOREMAN_STATUS_WAIT = 1
test_process.start()
output_event_queue.PushItem(plaso_queue.QueueAbort(), block=False)
output_event_queue.Close(abort=True)
# Sleep for 1 second to allow the analysis process to terminate.
# Before the temporary directory is removed.
time.sleep(1)
def PushItem(self, item, block=True):
"""Push an item on to the queue.
If no ZeroMQ socket has been created, one will be created the first time
this method is called.
Args:
item: The item to push on to the queue.
block: Optional argument to indicate whether the push should be performed
in blocking or non-block mode.
Raises:
KeyboardInterrupt: If the process is sent a KeyboardInterrupt while
pushing an item.
zmq.error.Again: If it was not possible to push the item to the queue
within the timeout.
zmq.error.ZMQError: If a ZeroMQ specific error occurs.
"""
logging.debug(u'Push on {0:s} queue, port {1:d}'.format(
self.name, self.port))
if not self._zmq_socket:
self._CreateZMQSocket()
try:
if block:
self._zmq_socket.send_pyobj(item)
else:
self._zmq_socket.send_pyobj(item, zmq.DONTWAIT)
except zmq.error.Again:
logging.error(u'{0:s} unable to push item, raising.'.format(
self.name))
raise
except zmq.error.ZMQError as exception:
if exception.errno == errno.EINTR:
logging.error(u'ZMQ syscall interrupted in {0:s}.'.format(
self.name))
return plaso_queue.QueueAbort()
else:
raise
except KeyboardInterrupt:
self.Close(abort=True)
raise
def PopItem(self):
"""Pops an item off the queue.
If no ZeroMQ socket has been created, one will be created the first
time this method is called.
Returns:
The item retrieved from the queue, as a deserialized Python object.
Raises:
KeyboardInterrupt: If the process is sent a KeyboardInterrupt while
popping an item.
QueueEmpty: If the queue is empty, and no item could be popped within the
queue timeout.
zmq.error.ZMQError: If an error occurs in ZeroMQ.
"""
logging.debug(u'Pop on {0:s} queue, port {1:d}'.format(
self.name, self.port))
if not self._zmq_socket:
self._CreateZMQSocket()
try:
self._zmq_socket.send_pyobj(None)
return self._zmq_socket.recv_pyobj()
except zmq.error.Again:
raise errors.QueueEmpty
except zmq.error.ZMQError as exception:
if exception.errno == errno.EINTR:
logging.error(
u'ZMQ syscall interrupted in {0:s}. Queue aborting.'.
format(self.name))
return plaso_queue.QueueAbort()
else:
raise
except KeyboardInterrupt:
self.Close(abort=True)
raise
def _AnalyzeEvents(self, storage_writer, analysis_plugins, event_filter=None):
"""Analyzes events in a plaso storage.
Args:
storage_writer (StorageWriter): storage writer.
analysis_plugins (dict[str, AnalysisPlugin]): analysis plugins that
should be run and their names.
event_filter (Optional[EventObjectFilter]): event filter.
Returns:
collections.Counter: counter containing information about the events
processed and filtered.
Raises:
RuntimeError: if a non-recoverable situation is encountered.
"""
self._status = definitions.STATUS_INDICATOR_RUNNING
self._number_of_consumed_events = 0
self._number_of_consumed_reports = 0
self._number_of_consumed_sources = 0
self._number_of_consumed_warnings = 0
self._number_of_produced_events = 0
self._number_of_produced_reports = 0
self._number_of_produced_sources = 0
self._number_of_produced_warnings = 0
number_of_filtered_events = 0
logger.debug('Processing events.')
filter_limit = getattr(event_filter, 'limit', None)
for event in storage_writer.GetSortedEvents():
event_data_identifier = event.GetEventDataIdentifier()
event_data = storage_writer.GetEventDataByIdentifier(
event_data_identifier)
event_data_stream_identifier = event_data.GetEventDataStreamIdentifier()
if event_data_stream_identifier:
event_data_stream = storage_writer.GetEventDataStreamByIdentifier(
event_data_stream_identifier)
else:
event_data_stream = None
event_identifier = event.GetIdentifier()
event_tag = self._event_tag_index.GetEventTagByIdentifier(
storage_writer, event_identifier)
if event_filter:
filter_match = event_filter.Match(
event, event_data, event_data_stream, event_tag)
else:
filter_match = None
# pylint: disable=singleton-comparison
if filter_match == False:
number_of_filtered_events += 1
continue
for event_queue in self._event_queues.values():
# TODO: Check for premature exit of analysis plugins.
event_queue.PushItem((event, event_data, event_data_stream))
self._number_of_consumed_events += 1
if (event_filter and filter_limit and
filter_limit == self._number_of_consumed_events):
break
logger.debug('Finished pushing events to analysis plugins.')
# Signal that we have finished adding events.
for event_queue in self._event_queues.values():
event_queue.PushItem(plaso_queue.QueueAbort(), block=False)
logger.debug('Processing analysis plugin results.')
# TODO: use a task based approach.
plugin_names = list(analysis_plugins.keys())
while plugin_names:
for plugin_name in list(plugin_names):
if self._abort:
break
# TODO: temporary solution.
task = tasks.Task()
task.storage_format = definitions.STORAGE_FORMAT_SQLITE
task.identifier = plugin_name
merge_ready = storage_writer.CheckTaskReadyForMerge(task)
if merge_ready:
storage_writer.PrepareMergeTaskStorage(task)
self._status = definitions.STATUS_INDICATOR_MERGING
event_queue = self._event_queues[plugin_name]
del self._event_queues[plugin_name]
event_queue.Close()
storage_merge_reader = storage_writer.StartMergeTaskStorage(task)
storage_merge_reader.MergeAttributeContainers(
callback=self._MergeEventTag)
# TODO: temporary solution.
plugin_names.remove(plugin_name)
self._status = definitions.STATUS_INDICATOR_RUNNING
self._number_of_produced_event_tags = (
storage_writer.number_of_event_tags)
self._number_of_produced_reports = (
storage_writer.number_of_analysis_reports)
try:
storage_writer.StopTaskStorage(abort=self._abort)
except (IOError, OSError) as exception:
logger.error('Unable to stop task storage with error: {0!s}'.format(
exception))
if self._abort:
logger.debug('Processing aborted.')
else:
logger.debug('Processing completed.')
events_counter = collections.Counter()
events_counter['Events filtered'] = number_of_filtered_events
events_counter['Events processed'] = self._number_of_consumed_events
return events_counter
def _AnalyzeEvents(self, storage_writer, analysis_plugins, event_filter=None):
"""Analyzes events in a plaso storage.
Args:
storage_writer (StorageWriter): storage writer.
analysis_plugins (list[AnalysisPlugin]): analysis plugins that should
be run.
event_filter (Optional[FilterObject]): event filter.
Raises:
RuntimeError: if a non-recoverable situation is encountered.
"""
self._status = definitions.PROCESSING_STATUS_RUNNING
self._number_of_consumed_errors = 0
self._number_of_consumed_events = 0
self._number_of_consumed_reports = 0
self._number_of_consumed_sources = 0
self._number_of_produced_errors = 0
self._number_of_produced_events = 0
self._number_of_produced_reports = 0
self._number_of_produced_sources = 0
number_of_filtered_events = 0
logging.debug(u'Processing events.')
filter_limit = getattr(event_filter, u'limit', None)
for event in storage_writer.GetEvents():
if event_filter:
filter_match = event_filter.Match(event)
else:
filter_match = None
# pylint: disable=singleton-comparison
if filter_match == False:
number_of_filtered_events += 1
continue
for event_queue in self._event_queues.values():
# TODO: Check for premature exit of analysis plugins.
event_queue.PushItem(event)
self._number_of_consumed_events += 1
if (event_filter and filter_limit and
filter_limit == self._number_of_consumed_events):
break
logging.debug(u'Finished pushing events to analysis plugins.')
# Signal that we have finished adding events.
for event_queue in self._event_queues.values():
event_queue.PushItem(plaso_queue.QueueAbort(), block=False)
logging.debug(u'Processing analysis plugin results.')
# TODO: use a task based approach.
plugin_names = [plugin.plugin_name for plugin in analysis_plugins]
while plugin_names:
for plugin_name in list(plugin_names):
if self._abort:
break
# TODO: temporary solution.
task = tasks.Task()
task.identifier = plugin_name
merge_ready = storage_writer.CheckTaskReadyForMerge(task)
if merge_ready:
self._status = definitions.PROCESSING_STATUS_MERGING
event_queue = self._event_queues[plugin_name]
del self._event_queues[plugin_name]
event_queue.Close()
storage_merge_reader = storage_writer.StartMergeTaskStorage(task)
storage_merge_reader.MergeAttributeContainers()
# TODO: temporary solution.
plugin_names.remove(plugin_name)
self._status = definitions.PROCESSING_STATUS_RUNNING
self._number_of_produced_event_tags = (
storage_writer.number_of_event_tags)
self._number_of_produced_reports = (
storage_writer.number_of_analysis_reports)
try:
storage_writer.StopTaskStorage(abort=self._abort)
except (IOError, OSError) as exception:
logging.error(u'Unable to stop task storage with error: {0:s}'.format(
exception))
if self._abort:
logging.debug(u'Processing aborted.')
else:
logging.debug(u'Processing completed.')
events_counter = collections.Counter()
events_counter[u'Events filtered'] = number_of_filtered_events
events_counter[u'Events processed'] = self._number_of_consumed_events
return events_counter