def __init__(self):
"""Initializes the front-end object."""
super(ImageExportFrontend, self).__init__()
self._filter_collection = FileEntryFilterCollection()
self._knowledge_base = None
self._resolver_context = context.Context()
def setUp(self):
"""Sets up the needed objects used throughout the test."""
self._resolver_context = context.Context()
self._fake_path_spec = fake_path_spec.FakePathSpec(location=u'/')
class Resolver(object):
"""Path specification resolver."""
_resolver_context = context.Context()
_resolver_helpers_manager = None
key_chain = keychain.KeyChain()
@classmethod
def _GetResolverHelper(cls, type_indicator):
"""Retrieves the path specification resolver helper for the specified type.
Args:
type_indicator (str): type indicator.
Returns:
ResolverHelper: a resolver helper.
"""
if not cls._resolver_helpers_manager:
# Delay the import of the resolver helpers manager to prevent circular
# imports.
from dfvfs.resolver_helpers import manager # pylint: disable=import-outside-toplevel
cls._resolver_helpers_manager = manager.ResolverHelperManager
return cls._resolver_helpers_manager.GetHelper(type_indicator)
@classmethod
def OpenFileEntry(cls, path_spec_object, resolver_context=None):
"""Opens a file entry object defined by path specification.
Args:
path_spec_object (PathSpec): path specification.
resolver_context (Optional[Context]): resolver context, where None
represents the built in context which is not multi process safe.
Returns:
FileEntry: file entry or None if the path specification could not be
resolved.
"""
file_system = cls.OpenFileSystem(path_spec_object,
resolver_context=resolver_context)
if resolver_context is None:
resolver_context = cls._resolver_context
file_entry = file_system.GetFileEntryByPathSpec(path_spec_object)
# Release the file system so it will be removed from the cache
# when the file entry is destroyed.
resolver_context.ReleaseFileSystem(file_system)
return file_entry
@classmethod
def OpenFileObject(cls, path_spec_object, resolver_context=None):
"""Opens a file-like object defined by path specification.
Args:
path_spec_object (PathSpec): path specification.
resolver_context (Optional[Context]): resolver context, where None
represents the built in context which is not multi process safe.
Returns:
FileIO: file-like object or None if the path specification could not
be resolved.
Raises:
MountPointError: if the mount point specified in the path specification
does not exist.
PathSpecError: if the path specification is incorrect.
TypeError: if the path specification type is unsupported.
"""
if not isinstance(path_spec_object, path_spec.PathSpec):
raise TypeError('Unsupported path specification type.')
if resolver_context is None:
resolver_context = cls._resolver_context
if path_spec_object.type_indicator == definitions.TYPE_INDICATOR_MOUNT:
if path_spec_object.HasParent():
raise errors.PathSpecError(
'Unsupported mount path specification with parent.')
mount_point = getattr(path_spec_object, 'identifier', None)
if not mount_point:
raise errors.PathSpecError(
'Unsupported path specification without mount point identifier.'
)
path_spec_object = mount_manager.MountPointManager.GetMountPoint(
mount_point)
if not path_spec_object:
raise errors.MountPointError(
'No such mount point: {0:s}'.format(mount_point))
file_object = resolver_context.GetFileObject(path_spec_object)
if not file_object:
resolver_helper = cls._GetResolverHelper(
path_spec_object.type_indicator)
file_object = resolver_helper.NewFileObject(resolver_context)
file_object.open(path_spec=path_spec_object)
return file_object
@classmethod
def OpenFileSystem(cls, path_spec_object, resolver_context=None):
"""Opens a file system object defined by path specification.
Args:
path_spec_object (PathSpec): path specification.
resolver_context (Optional[Context]): resolver context, where None
represents the built in context which is not multi process safe.
Returns:
FileSystem: file system or None if the path specification could not
be resolved or has no file system object.
Raises:
AccessError: if the access to open the file system was denied.
BackEndError: if the file system cannot be opened.
MountPointError: if the mount point specified in the path specification
does not exist.
PathSpecError: if the path specification is incorrect.
TypeError: if the path specification type is unsupported.
"""
if not isinstance(path_spec_object, path_spec.PathSpec):
raise TypeError('Unsupported path specification type.')
if resolver_context is None:
resolver_context = cls._resolver_context
if path_spec_object.type_indicator == definitions.TYPE_INDICATOR_MOUNT:
if path_spec_object.HasParent():
raise errors.PathSpecError(
'Unsupported mount path specification with parent.')
mount_point = getattr(path_spec_object, 'identifier', None)
if not mount_point:
raise errors.PathSpecError(
'Unsupported path specification without mount point identifier.'
)
path_spec_object = mount_manager.MountPointManager.GetMountPoint(
mount_point)
if not path_spec_object:
raise errors.MountPointError(
'No such mount point: {0:s}'.format(mount_point))
file_system = resolver_context.GetFileSystem(path_spec_object)
if not file_system:
resolver_helper = cls._GetResolverHelper(
path_spec_object.type_indicator)
file_system = resolver_helper.NewFileSystem(resolver_context)
try:
file_system.Open(path_spec_object)
except (IOError, ValueError) as exception:
raise errors.BackEndError(
'Unable to open file system with error: {0!s}'.format(
exception))
return file_system
def setUp(self):
"""Sets up the needed objects used throughout the test."""
self._resolver_context = context.Context()
test_file = os.path.join(u'test_data', u'image.qcow2')
path_spec = os_path_spec.OSPathSpec(location=test_file)
self._qcow_path_spec = qcow_path_spec.QCOWPathSpec(parent=path_spec)
def __init__(
self, maximum_number_of_tasks=None, number_of_worker_processes=0,
worker_memory_limit=None, worker_timeout=None):
"""Initializes an engine.
Args:
maximum_number_of_tasks (Optional[int]): maximum number of concurrent
tasks, where 0 represents no limit.
number_of_worker_processes (Optional[int]): number of worker processes.
worker_memory_limit (Optional[int]): maximum amount of memory a worker is
allowed to consume, where None represents the default memory limit
and 0 represents no limit.
worker_timeout (Optional[float]): number of minutes before a worker
process that is not providing status updates is considered inactive,
where None or 0.0 represents the default timeout.
"""
if maximum_number_of_tasks is None:
maximum_number_of_tasks = self._MAXIMUM_NUMBER_OF_TASKS
if number_of_worker_processes < 1:
# One worker for each "available" CPU (minus other processes).
# The number here is derived from the fact that the engine starts up:
# * A main process.
#
# If we want to utilize all CPUs on the system we therefore need to start
# up workers that amounts to the total number of CPUs - the other
# processes.
try:
cpu_count = multiprocessing.cpu_count() - 1
if cpu_count <= self._WORKER_PROCESSES_MINIMUM:
cpu_count = self._WORKER_PROCESSES_MINIMUM
elif cpu_count >= self._WORKER_PROCESSES_MAXIMUM:
cpu_count = self._WORKER_PROCESSES_MAXIMUM
except NotImplementedError:
logger.error((
'Unable to determine number of CPUs defaulting to {0:d} worker '
'processes.').format(self._WORKER_PROCESSES_MINIMUM))
cpu_count = self._WORKER_PROCESSES_MINIMUM
number_of_worker_processes = cpu_count
if worker_memory_limit is None:
worker_memory_limit = definitions.DEFAULT_WORKER_MEMORY_LIMIT
if not worker_timeout:
worker_timeout = definitions.DEFAULT_WORKER_TIMEOUT
super(TaskMultiProcessEngine, self).__init__()
self._enable_sigsegv_handler = False
self._last_worker_number = 0
self._maximum_number_of_tasks = maximum_number_of_tasks
self._merge_task = None
self._merge_task_on_hold = None
self._number_of_consumed_event_tags = 0
self._number_of_consumed_events = 0
self._number_of_consumed_reports = 0
self._number_of_consumed_sources = 0
self._number_of_consumed_warnings = 0
self._number_of_produced_event_tags = 0
self._number_of_produced_events = 0
self._number_of_produced_reports = 0
self._number_of_produced_sources = 0
self._number_of_produced_warnings = 0
self._number_of_worker_processes = number_of_worker_processes
self._path_spec_extractor = extractors.PathSpecExtractor()
self._processing_configuration = None
self._redis_client = None
self._resolver_context = context.Context()
self._session_identifier = None
self._status = definitions.STATUS_INDICATOR_IDLE
self._storage_merge_reader = None
self._storage_merge_reader_on_hold = None
self._task_queue = None
self._task_queue_port = None
self._task_manager = task_manager.TaskManager()
self._worker_memory_limit = worker_memory_limit
self._worker_timeout = worker_timeout
def setUp(self):
"""Sets up the needed objects used throughout the test."""
self._resolver_context = context.Context()
test_file = self._GetTestFilePath(['lvmtest.qcow2'])
path_spec = os_path_spec.OSPathSpec(location=test_file)
self._qcow_path_spec = qcow_path_spec.QCOWPathSpec(parent=path_spec)
def GetEntries(self, parser_mediator, match=None, **unused_kwargs):
"""Extracts relevant user timestamp entries.
Args:
parser_mediator: A parser mediator object (instance of ParserMediator).
match: Optional dictionary containing keys extracted from PLIST_KEYS.
"""
if u'name' not in match or u'uid' not in match:
return
account = match[u'name'][0]
uid = match[u'uid'][0]
cocoa_zero = (timelib.Timestamp.COCOA_TIME_TO_POSIX_BASE *
timelib.Timestamp.MICRO_SECONDS_PER_SECOND)
# INFO: binplist return a string with the Plist XML.
for policy in match.get(u'passwordpolicyoptions', []):
try:
xml_policy = ElementTree.fromstring(policy)
except (ElementTree.ParseError, LookupError) as exception:
logging.error((
u'Unable to parse XML structure for an user policy, account: '
u'{0:s} and uid: {1!s}, with error: {2:s}').format(
account, uid, exception))
continue
for dict_elements in xml_policy.iterfind(u'dict'):
key_values = [
value.text for value in dict_elements.getchildren()
]
# Taking a list and converting it to a dict, using every other item
# as the key and the other one as the value.
policy_dict = dict(zip(key_values[0::2], key_values[1::2]))
time_string = policy_dict.get(u'passwordLastSetTime', None)
if time_string:
try:
timestamp = timelib.Timestamp.FromTimeString(time_string)
except errors.TimestampError:
parser_mediator.ProduceParseError(
u'Unable to parse time string: {0:s}'.format(
time_string))
timestamp = 0
shadow_hash_data = match.get(u'ShadowHashData', None)
if timestamp > cocoa_zero and isinstance(
shadow_hash_data, (list, tuple)):
# Extract the hash password information.
# It is store in the attribute ShadowHasData which is
# a binary plist data; However binplist only extract one
# level of binary plist, then it returns this information
# as a string.
# TODO: change this into a DataRange instead. For this we
# need the file offset and size of the ShadowHashData value data.
shadow_hash_data = shadow_hash_data[0]
resolver_context = context.Context()
fake_file = fake_file_io.FakeFile(resolver_context,
shadow_hash_data)
fake_file.open(path_spec=fake_path_spec.FakePathSpec(
location=u'ShadowHashData'))
try:
plist_file = binplist.BinaryPlist(file_obj=fake_file)
top_level = plist_file.Parse()
except binplist.FormatError:
top_level = dict()
salted_hash = top_level.get(u'SALTED-SHA512-PBKDF2', None)
if salted_hash:
password_hash = u'$ml${0:d}${1:s}${2:s}'.format(
salted_hash[u'iterations'],
binascii.hexlify(salted_hash[u'salt']),
binascii.hexlify(salted_hash[u'entropy']))
else:
password_hash = u'N/A'
description = (
u'Last time {0:s} ({1!s}) changed the password: {2!s}'
).format(account, uid, password_hash)
event_object = plist_event.PlistTimeEvent(
self._ROOT, u'passwordLastSetTime', timestamp,
description)
parser_mediator.ProduceEvent(event_object)
time_string = policy_dict.get(u'lastLoginTimestamp', None)
if time_string:
try:
timestamp = timelib.Timestamp.FromTimeString(time_string)
except errors.TimestampError:
parser_mediator.ProduceParseError(
u'Unable to parse time string: {0:s}'.format(
time_string))
timestamp = 0
description = u'Last login from {0:s} ({1!s})'.format(
account, uid)
if timestamp > cocoa_zero:
event_object = plist_event.PlistTimeEvent(
self._ROOT, u'lastLoginTimestamp', timestamp,
description)
parser_mediator.ProduceEvent(event_object)
time_string = policy_dict.get(u'failedLoginTimestamp', None)
if time_string:
try:
timestamp = timelib.Timestamp.FromTimeString(time_string)
except errors.TimestampError:
parser_mediator.ProduceParseError(
u'Unable to parse time string: {0:s}'.format(
time_string))
timestamp = 0
description = (
u'Last failed login from {0:s} ({1!s}) ({2!s} times)'
).format(account, uid, policy_dict.get(u'failedLoginCount', 0))
if timestamp > cocoa_zero:
event_object = plist_event.PlistTimeEvent(
self._ROOT, u'failedLoginTimestamp', timestamp,
description)
parser_mediator.ProduceEvent(event_object)
class Resolver(object):
"""Class that implements the path specification resolver."""
_resolver_context = context.Context()
_resolver_helpers = {}
key_chain = keychain.KeyChain()
@classmethod
def DeregisterHelper(cls, resolver_helper):
"""Deregisters a path specification resolver helper.
Args:
resolver_helper: the resolver helper object (instance of
ResolverHelper).
Raises:
KeyError: if resolver helper object is not set for the corresponding
type indicator.
"""
if resolver_helper.type_indicator not in cls._resolver_helpers:
raise KeyError(
u'Resolver helper object not set for type indicator: {0:s}.'.
format(resolver_helper.type_indicator))
del cls._resolver_helpers[resolver_helper.type_indicator]
@classmethod
def OpenFileEntry(cls, path_spec_object, resolver_context=None):
"""Opens a file entry object defined by path specification.
Args:
path_spec_object: the path specification (instance of PathSpec).
resolver_context: the optional resolver context (instance of
resolver.Context). The default is None which will use
the built in context which is not multi process safe.
Returns:
The file entry object (instance of vfs.FileEntry) or None if the path
specification could not be resolved.
"""
file_system = cls.OpenFileSystem(path_spec_object,
resolver_context=resolver_context)
if resolver_context is None:
resolver_context = cls._resolver_context
file_entry = file_system.GetFileEntryByPathSpec(path_spec_object)
# Release the file system so it will be removed from the cache
# when the file entry is destroyed.
resolver_context.ReleaseFileSystem(file_system)
return file_entry
@classmethod
def OpenFileObject(cls, path_spec_object, resolver_context=None):
"""Opens a file-like object defined by path specification.
Args:
path_spec_object: the path specification (instance of PathSpec).
resolver_context: the optional resolver context (instance of
resolver.Context). The default is None which will use
the built in context which is not multi process safe.
Returns:
The file-like object (instance of file.FileIO) or None if the path
specification could not be resolved.
Raises:
KeyError: if resolver helper object is not set for the corresponding
type indicator.
PathSpecError: if the path specification is incorrect.
TypeError: if the path specification type is unsupported.
"""
if not isinstance(path_spec_object, path_spec.PathSpec):
raise TypeError(u'Unsupported path specification type.')
if resolver_context is None:
resolver_context = cls._resolver_context
if path_spec_object.type_indicator == definitions.TYPE_INDICATOR_MOUNT:
if path_spec_object.HasParent():
raise errors.PathSpecError(
u'Unsupported mount path specification with parent.')
mount_point = getattr(path_spec_object, u'identifier', None)
if not mount_point:
raise errors.PathSpecError(
u'Unsupported path specification without mount point identifier.'
)
path_spec_object = mount_manager.MountPointManager.GetMountPoint(
mount_point)
if not path_spec_object:
raise errors.MountPointError(
u'No such mount point: {0:s}'.format(mount_point))
file_object = resolver_context.GetFileObject(path_spec_object)
if not file_object:
if path_spec_object.type_indicator not in cls._resolver_helpers:
raise KeyError(
(u'Resolver helper object not set for type indicator: '
u'{0:s}.').format(path_spec_object.type_indicator))
resolver_helper = cls._resolver_helpers[
path_spec_object.type_indicator]
file_object = resolver_helper.NewFileObject(resolver_context)
file_object.open(path_spec=path_spec_object)
return file_object
@classmethod
def OpenFileSystem(cls, path_spec_object, resolver_context=None):
"""Opens a file system object defined by path specification.
Args:
path_spec_object: the path specification (instance of PathSpec).
resolver_context: the optional resolver context (instance of
resolver.Context). The default is None which will use
the built in context which is not multi process safe.
Returns:
The file system object (instance of vfs.FileSystem) or None if the path
specification could not be resolved or has no file system object.
Raises:
AccessError: if the access to open the file system was denied.
BackEndError: if the file system cannot be opened.
KeyError: if resolver helper object is not set for the corresponding
type indicator.
PathSpecError: if the path specification is incorrect.
TypeError: if the path specification type is unsupported.
"""
if not isinstance(path_spec_object, path_spec.PathSpec):
raise TypeError(u'Unsupported path specification type.')
if resolver_context is None:
resolver_context = cls._resolver_context
if path_spec_object.type_indicator == definitions.TYPE_INDICATOR_MOUNT:
if path_spec_object.HasParent():
raise errors.PathSpecError(
u'Unsupported mount path specification with parent.')
mount_point = getattr(path_spec_object, u'identifier', None)
if not mount_point:
raise errors.PathSpecError(
u'Unsupported path specification without mount point identifier.'
)
path_spec_object = mount_manager.MountPointManager.GetMountPoint(
mount_point)
if not path_spec_object:
raise errors.MountPointError(
u'No such mount point: {0:s}'.format(mount_point))
file_system = resolver_context.GetFileSystem(path_spec_object)
if not file_system:
if path_spec_object.type_indicator not in cls._resolver_helpers:
raise KeyError(
(u'Resolver helper object not set for type indicator: '
u'{0:s}.').format(path_spec_object.type_indicator))
resolver_helper = cls._resolver_helpers[
path_spec_object.type_indicator]
file_system = resolver_helper.NewFileSystem(resolver_context)
try:
file_system.Open(path_spec_object)
except (errors.AccessError, errors.PathSpecError):
raise
except (IOError, ValueError) as exception:
raise errors.BackEndError(
u'Unable to open file system with error: {0:s}'.format(
exception))
return file_system
@classmethod
def RegisterHelper(cls, resolver_helper):
"""Registers a path specification resolver helper.
Args:
resolver_helper: the resolver helper object (instance of
ResolverHelper).
Raises:
KeyError: if resolver helper object is already set for the corresponding
type indicator.
"""
if resolver_helper.type_indicator in cls._resolver_helpers:
raise KeyError(
(u'Resolver helper object already set for type indicator: '
u'{0!s}.').format(resolver_helper.type_indicator))
cls._resolver_helpers[resolver_helper.type_indicator] = resolver_helper
def _TestImageWithPartitionsCollections(self, collect_directory_metadata):
"""Test collection on a storage media image with multiple partitions.
The image contains 2 partitions (p1 and p2) with NFTS file systems.
Args:
collect_directory_metadata: boolean value to indicate to collect
directory metadata.
"""
# Note that the source file is a RAW (VMDK flat) image.
test_file = self._GetTestFilePath([u'multi_partition_image.vmdk'])
image_path_spec = path_spec_factory.Factory.NewPathSpec(
dfvfs_definitions.TYPE_INDICATOR_OS, location=test_file)
p1_path_spec = path_spec_factory.Factory.NewPathSpec(
dfvfs_definitions.TYPE_INDICATOR_TSK_PARTITION,
location=u'/p1',
part_index=2,
start_offset=0x00010000,
parent=image_path_spec)
p1_file_system_path_spec = path_spec_factory.Factory.NewPathSpec(
dfvfs_definitions.TYPE_INDICATOR_TSK,
location=u'/',
parent=p1_path_spec)
p2_path_spec = path_spec_factory.Factory.NewPathSpec(
dfvfs_definitions.TYPE_INDICATOR_TSK_PARTITION,
location=u'/p2',
part_index=3,
start_offset=0x00510000,
parent=image_path_spec)
p2_file_system_path_spec = path_spec_factory.Factory.NewPathSpec(
dfvfs_definitions.TYPE_INDICATOR_TSK,
location=u'/',
parent=p2_path_spec)
test_path_spec_queue = single_process.SingleProcessQueue()
resolver_context = context.Context()
test_collector = collector.Collector(test_path_spec_queue,
resolver_context=resolver_context)
test_collector.SetCollectDirectoryMetadata(collect_directory_metadata)
test_collector.Collect(
[p1_file_system_path_spec, p2_file_system_path_spec])
test_collector_queue_consumer = TestCollectorQueueConsumer(
test_path_spec_queue)
test_collector_queue_consumer.ConsumeItems()
paths = test_collector_queue_consumer.GetFilePaths()
expected_paths_p1 = [
u'/$AttrDef', u'/$BadClus', u'/$BadClus:$Bad', u'/$Bitmap',
u'/$Boot', u'/$Extend/$ObjId', u'/$Extend/$Quota',
u'/$Extend/$Reparse', u'/$Extend/$RmMetadata/$Repair',
u'/$Extend/$RmMetadata/$Repair:$Config', u'/$LogFile', u'/$MFT',
u'/$MFTMirr', u'/$Secure', u'/$Secure:$SDS', u'/$UpCase',
u'/$Volume', u'/file1.txt', u'/file2.txt'
]
if collect_directory_metadata:
expected_directory_metadata_paths_p1 = [
u'/$Extend',
u'/$Extend/$RmMetadata',
u'/$Extend/$RmMetadata/$TxfLog',
]
expected_paths_p2 = [
u'/$AttrDef', u'/$BadClus', u'/$BadClus:$Bad', u'/$Bitmap',
u'/$Boot', u'/$Extend/$ObjId', u'/$Extend/$Quota',
u'/$Extend/$Reparse', u'/$Extend/$RmMetadata/$Repair',
u'/$Extend/$RmMetadata/$Repair:$Config', u'/$LogFile', u'/$MFT',
u'/$MFTMirr', u'/$Secure', u'/$Secure:$SDS', u'/$UpCase',
u'/$Volume', u'/file1_on_part_2.txt', u'/file2_on_part_2.txt'
]
if collect_directory_metadata:
expected_directory_metadata_paths_p2 = [
u'/$Extend',
u'/$Extend/$RmMetadata',
u'/$Extend/$RmMetadata/$TxfLog',
]
expected_paths = []
expected_paths.extend(expected_paths_p1)
expected_paths.extend(expected_paths_p2)
if collect_directory_metadata:
expected_paths.extend(expected_directory_metadata_paths_p1)
expected_paths.extend(expected_directory_metadata_paths_p2)
self.assertEqual(test_collector_queue_consumer.number_of_path_specs,
len(expected_paths))
self.assertEqual(sorted(paths), sorted(expected_paths))
def setUp(self):
"""Sets up the needed objects used throughout the test."""
self._resolver_context = context.Context()
test_file = os.path.join(u'test_data', u'syslog.gz')
path_spec = os_path_spec.OSPathSpec(location=test_file)
self._gzip_path_spec = gzip_path_spec.GzipPathSpec(parent=path_spec)
def testExtractionWorker(self):
"""Tests the extraction worker functionality."""
collection_queue = single_process.SingleProcessQueue()
storage_queue = single_process.SingleProcessQueue()
parse_error_queue = single_process.SingleProcessQueue()
event_queue_producer = single_process.SingleProcessItemQueueProducer(
storage_queue)
parse_error_queue_producer = single_process.SingleProcessItemQueueProducer(
parse_error_queue)
knowledge_base_object = knowledge_base.KnowledgeBase()
parser_mediator = parsers_mediator.ParserMediator(
event_queue_producer, parse_error_queue_producer,
knowledge_base_object)
resolver_context = context.Context()
extraction_worker = worker.BaseEventExtractionWorker(
0,
collection_queue,
event_queue_producer,
parse_error_queue_producer,
parser_mediator,
resolver_context=resolver_context)
self.assertNotEqual(extraction_worker, None)
extraction_worker.InitializeParserObjects()
# Process a file.
source_path = self._GetTestFilePath([u'syslog'])
path_spec = path_spec_factory.Factory.NewPathSpec(
dfvfs_definitions.TYPE_INDICATOR_OS, location=source_path)
collection_queue.PushItem(path_spec)
extraction_worker.Run()
test_queue_consumer = test_lib.TestQueueConsumer(storage_queue)
test_queue_consumer.ConsumeItems()
self.assertEqual(test_queue_consumer.number_of_items, 16)
# Process a compressed file.
source_path = self._GetTestFilePath([u'syslog.gz'])
path_spec = path_spec_factory.Factory.NewPathSpec(
dfvfs_definitions.TYPE_INDICATOR_OS, location=source_path)
collection_queue.PushItem(path_spec)
extraction_worker.Run()
test_queue_consumer = test_lib.TestQueueConsumer(storage_queue)
test_queue_consumer.ConsumeItems()
self.assertEqual(test_queue_consumer.number_of_items, 16)
source_path = self._GetTestFilePath([u'syslog.bz2'])
path_spec = path_spec_factory.Factory.NewPathSpec(
dfvfs_definitions.TYPE_INDICATOR_OS, location=source_path)
collection_queue.PushItem(path_spec)
extraction_worker.Run()
test_queue_consumer = test_lib.TestQueueConsumer(storage_queue)
test_queue_consumer.ConsumeItems()
self.assertEqual(test_queue_consumer.number_of_items, 15)
# Process a file in an archive.
source_path = self._GetTestFilePath([u'syslog.tar'])
path_spec = path_spec_factory.Factory.NewPathSpec(
dfvfs_definitions.TYPE_INDICATOR_OS, location=source_path)
path_spec = path_spec_factory.Factory.NewPathSpec(
dfvfs_definitions.TYPE_INDICATOR_TAR,
location=u'/syslog',
parent=path_spec)
collection_queue.PushItem(path_spec)
extraction_worker.Run()
test_queue_consumer = test_lib.TestQueueConsumer(storage_queue)
test_queue_consumer.ConsumeItems()
self.assertEqual(test_queue_consumer.number_of_items, 13)
# Process an archive file without "process archive files" mode.
extraction_worker.SetProcessArchiveFiles(False)
source_path = self._GetTestFilePath([u'syslog.tar'])
path_spec = path_spec_factory.Factory.NewPathSpec(
dfvfs_definitions.TYPE_INDICATOR_OS, location=source_path)
collection_queue.PushItem(path_spec)
extraction_worker.Run()
test_queue_consumer = test_lib.TestQueueConsumer(storage_queue)
test_queue_consumer.ConsumeItems()
self.assertEqual(test_queue_consumer.number_of_items, 3)
# Process an archive file with "process archive files" mode.
extraction_worker.SetProcessArchiveFiles(True)
source_path = self._GetTestFilePath([u'syslog.tar'])
path_spec = path_spec_factory.Factory.NewPathSpec(
dfvfs_definitions.TYPE_INDICATOR_OS, location=source_path)
collection_queue.PushItem(path_spec)
extraction_worker.Run()
test_queue_consumer = test_lib.TestQueueConsumer(storage_queue)
test_queue_consumer.ConsumeItems()
self.assertEqual(test_queue_consumer.number_of_items, 16)
# Process a file in a compressed archive.
source_path = self._GetTestFilePath([u'syslog.tgz'])
path_spec = path_spec_factory.Factory.NewPathSpec(
dfvfs_definitions.TYPE_INDICATOR_OS, location=source_path)
path_spec = path_spec_factory.Factory.NewPathSpec(
dfvfs_definitions.TYPE_INDICATOR_GZIP, parent=path_spec)
path_spec = path_spec_factory.Factory.NewPathSpec(
dfvfs_definitions.TYPE_INDICATOR_TAR,
location=u'/syslog',
parent=path_spec)
collection_queue.PushItem(path_spec)
extraction_worker.Run()
test_queue_consumer = test_lib.TestQueueConsumer(storage_queue)
test_queue_consumer.ConsumeItems()
self.assertEqual(test_queue_consumer.number_of_items, 13)
# Process an archive file with "process archive files" mode.
extraction_worker.SetProcessArchiveFiles(True)
source_path = self._GetTestFilePath([u'syslog.tgz'])
path_spec = path_spec_factory.Factory.NewPathSpec(
dfvfs_definitions.TYPE_INDICATOR_OS, location=source_path)
collection_queue.PushItem(path_spec)
extraction_worker.Run()
test_queue_consumer = test_lib.TestQueueConsumer(storage_queue)
test_queue_consumer.ConsumeItems()
self.assertEqual(test_queue_consumer.number_of_items, 17)
def __init__(self):
"""Initializes a fake file system builder."""
super(FakeFileSystemBuilder, self).__init__()
resolver_context = context.Context()
self.file_system = fake_file_system.FakeFileSystem(resolver_context)
def _Main(self):
"""The main loop."""
self._event_queue_producer = queue.ItemQueueProducer(
self._event_object_queue)
self._parse_error_queue_producer = queue.ItemQueueProducer(
self._parse_error_queue)
parser_mediator = parsers_mediator.ParserMediator(
self._event_queue_producer, self._parse_error_queue_producer,
self._knowledge_base)
# We need a resolver context per process to prevent multi processing
# issues with file objects stored in images.
resolver_context = context.Context()
self._extraction_worker = worker.BaseEventExtractionWorker(
self._worker_number,
self._path_spec_queue,
self._event_queue_producer,
self._parse_error_queue_producer,
parser_mediator,
resolver_context=resolver_context)
self._extraction_worker.SetEnableDebugOutput(self._enable_debug_output)
self._extraction_worker.SetEnableProfiling(
self._enable_profiling,
profiling_sample_rate=self._profiling_sample_rate,
profiling_type=self._profiling_type)
self._extraction_worker.SetProcessArchiveFiles(
self._process_archive_files)
if self._filter_object:
self._extraction_worker.SetFilterObject(self._filter_object)
if self._mount_path:
self._extraction_worker.SetMountPath(self._mount_path)
if self._text_prepend:
self._extraction_worker.SetTextPrepend(self._text_prepend)
# We need to initialize the parser and hasher objects after the process
# has forked otherwise on Windows the "fork" will fail with
# a PickleError for Python modules that cannot be pickled.
self._extraction_worker.InitializeParserObjects(
parser_filter_string=self._parser_filter_string)
if self._hasher_names_string:
self._extraction_worker.SetHashers(self._hasher_names_string)
logging.debug(u'Extraction worker: {0!s} (PID: {1:d}) started'.format(
self._name, self._pid))
try:
self._extraction_worker.Run()
except Exception as exception:
logging.warning((u'Unhandled exception in extraction worker {0!s} '
u'(PID: {1:d}).').format(self._name, self._pid))
logging.exception(exception)
logging.debug(u'Extraction worker: {0!s} (PID: {1:d}) stopped'.format(
self._name, self._pid))
self._path_spec_queue.Close(abort=True)
self._event_object_queue.Close(abort=True)
self._parse_error_queue.Close(abort=True)
def _ProcessSourceMultiProcessMode(self, options):
"""Processes the source in a multiple process.
Multiprocessing is used to start up separate processes.
Args:
options: the command line arguments (instance of argparse.Namespace).
"""
# TODO: replace by an option.
start_collection_process = True
self._number_of_worker_processes = getattr(options, 'workers', 0)
logging.info(u'Starting extraction in multi process mode.')
self._engine = multi_process.MultiProcessEngine(
maximum_number_of_queued_items=self._queue_size)
self._engine.SetEnableDebugOutput(self._debug_mode)
self._engine.SetEnableProfiling(
self._enable_profiling,
profiling_sample_rate=self._profiling_sample_rate)
self._engine.SetProcessArchiveFiles(self._process_archive_files)
if self._filter_object:
self._engine.SetFilterObject(self._filter_object)
if self._mount_path:
self._engine.SetMountPath(self._mount_path)
if self._text_prepend:
self._engine.SetTextPrepend(self._text_prepend)
# TODO: add support to handle multiple partitions.
self._engine.SetSource(
self.GetSourcePathSpec(), resolver_context=self._resolver_context)
logging.debug(u'Starting preprocessing.')
pre_obj = self.PreprocessSource(options)
logging.debug(u'Preprocessing done.')
# TODO: make sure parsers option is not set by preprocessing.
parser_filter_string = getattr(options, 'parsers', '')
self._parser_names = []
for _, parser_class in parsers_manager.ParsersManager.GetParsers(
parser_filter_string=parser_filter_string):
self._parser_names.append(parser_class.NAME)
hasher_names_string = getattr(options, u'hashers', u'')
self._hasher_names = []
hasher_manager = hashers_manager.HashersManager
for hasher_name in hasher_manager.GetHasherNamesFromString(
hasher_names_string=hasher_names_string):
self._hasher_names.append(hasher_name)
self._PreprocessSetCollectionInformation(options, pre_obj)
if 'filestat' in self._parser_names:
include_directory_stat = True
else:
include_directory_stat = False
filter_file = getattr(options, 'file_filter', None)
if filter_file:
filter_find_specs = engine_utils.BuildFindSpecsFromFile(
filter_file, pre_obj=pre_obj)
else:
filter_find_specs = None
if start_collection_process:
resolver_context = context.Context()
else:
resolver_context = self._resolver_context
# TODO: create multi process collector.
self._collector = self._engine.CreateCollector(
include_directory_stat, vss_stores=self._vss_stores,
filter_find_specs=filter_find_specs, resolver_context=resolver_context)
self._DebugPrintCollector(options)
if self._output_module:
storage_writer = storage.BypassStorageWriter(
self._engine.storage_queue, self._storage_file_path,
output_module_string=self._output_module, pre_obj=pre_obj)
else:
storage_writer = storage.StorageFileWriter(
self._engine.storage_queue, self._storage_file_path,
buffer_size=self._buffer_size, pre_obj=pre_obj,
serializer_format=self._storage_serializer_format)
try:
self._engine.ProcessSource(
self._collector, storage_writer,
parser_filter_string=parser_filter_string,
hasher_names_string=hasher_names_string,
number_of_extraction_workers=self._number_of_worker_processes,
have_collection_process=start_collection_process,
have_foreman_process=self._run_foreman,
show_memory_usage=self._show_worker_memory_information)
except KeyboardInterrupt:
self._CleanUpAfterAbort()
raise errors.UserAbort(u'Process source aborted.')
def setUp(self): """Sets up the needed objects used throughout the test.""" self._resolver_context = context.Context() test_file = self._GetTestFilePath(['syslog.gz']) path_spec = os_path_spec.OSPathSpec(location=test_file) self._gzip_path_spec = gzip_path_spec.GzipPathSpec(parent=path_spec)
def _Main(self):
"""The main loop."""
self._parser_mediator = parsers_mediator.ParserMediator(
None,
self._knowledge_base,
preferred_year=self._preferred_year,
temporary_directory=self._temporary_directory)
if self._filter_object:
self._parser_mediator.SetFilterObject(self._filter_object)
if self._mount_path:
self._parser_mediator.SetMountPath(self._mount_path)
if self._text_prepend:
self._parser_mediator.SetTextPrepend(self._text_prepend)
# We need a resolver context per process to prevent multi processing
# issues with file objects stored in images.
resolver_context = context.Context()
# We need to initialize the parser and hasher objects after the process
# has forked otherwise on Windows the "fork" will fail with
# a PickleError for Python modules that cannot be pickled.
self._extraction_worker = worker.EventExtractionWorker(
resolver_context,
parser_filter_expression=self._parser_filter_expression,
process_archives=self._process_archives,
process_compressed_streams=self._process_compressed_streams)
if self._hasher_names_string:
self._extraction_worker.SetHashers(self._hasher_names_string)
if self._yara_rules_string:
self._extraction_worker.SetYaraRules(self._yara_rules_string)
self._StartProfiling()
logging.debug(u'Worker: {0!s} (PID: {1:d}) started'.format(
self._name, self._pid))
self._status = definitions.PROCESSING_STATUS_RUNNING
try:
logging.debug(
u'{0!s} (PID: {1:d}) started monitoring task queue.'.format(
self._name, self._pid))
while not self._abort:
try:
task = self._task_queue.PopItem()
except (errors.QueueClose, errors.QueueEmpty) as exception:
logging.debug(
u'ConsumeItems exiting with exception {0:s}.'.format(
type(exception)))
break
if isinstance(task, plaso_queue.QueueAbort):
logging.debug(
u'ConsumeItems exiting, dequeued QueueAbort object.')
break
self._ProcessTask(task)
logging.debug(
u'{0!s} (PID: {1:d}) stopped monitoring task queue.'.format(
self._name, self._pid))
# All exceptions need to be caught here to prevent the process
# from being killed by an uncaught exception.
except Exception as exception: # pylint: disable=broad-except
logging.warning(
u'Unhandled exception in process: {0!s} (PID: {1:d}).'.format(
self._name, self._pid))
logging.exception(exception)
self._abort = True
self._StopProfiling()
self._extraction_worker = None
self._parser_mediator = None
self._storage_writer = None
if self._abort:
self._status = definitions.PROCESSING_STATUS_ABORTED
else:
self._status = definitions.PROCESSING_STATUS_COMPLETED
logging.debug(u'Worker: {0!s} (PID: {1:d}) stopped'.format(
self._name, self._pid))
try:
self._task_queue.Close(abort=self._abort)
except errors.QueueAlreadyClosed:
logging.error(u'Queue for {0:s} was already closed.'.format(
self.name))
def testExtractPathSpecsStorageMediaImageWithPartitions(self):
"""Tests the ExtractPathSpecs function an image file with partitions.
The image file contains 2 partitions, p1 and p2, both with a NFTS
file systems.
"""
# Note that the source file is a RAW (VMDK flat) image.
test_file = self._GetTestFilePath(['multi_partition_image.vmdk'])
image_path_spec = path_spec_factory.Factory.NewPathSpec(
dfvfs_definitions.TYPE_INDICATOR_OS, location=test_file)
p1_path_spec = path_spec_factory.Factory.NewPathSpec(
dfvfs_definitions.TYPE_INDICATOR_TSK_PARTITION,
location='/p1',
part_index=2,
start_offset=0x00010000,
parent=image_path_spec)
p1_file_system_path_spec = path_spec_factory.Factory.NewPathSpec(
dfvfs_definitions.TYPE_INDICATOR_TSK,
location='/',
parent=p1_path_spec)
p2_path_spec = path_spec_factory.Factory.NewPathSpec(
dfvfs_definitions.TYPE_INDICATOR_TSK_PARTITION,
location='/p2',
part_index=3,
start_offset=0x00510000,
parent=image_path_spec)
p2_file_system_path_spec = path_spec_factory.Factory.NewPathSpec(
dfvfs_definitions.TYPE_INDICATOR_TSK,
location='/',
parent=p2_path_spec)
resolver_context = context.Context()
test_extractor = extractors.PathSpecExtractor(resolver_context)
path_specs = list(
test_extractor.ExtractPathSpecs(
[p1_file_system_path_spec, p2_file_system_path_spec],
resolver_context=resolver_context))
expected_paths_p1 = [
'/$AttrDef', '/$BadClus', '/$BadClus:$Bad', '/$Bitmap', '/$Boot',
'/$Extend', '/$Extend/$ObjId', '/$Extend/$Quota',
'/$Extend/$Reparse', '/$Extend/$RmMetadata',
'/$Extend/$RmMetadata/$Repair',
'/$Extend/$RmMetadata/$Repair:$Config',
'/$Extend/$RmMetadata/$TxfLog', '/$LogFile', '/$MFT', '/$MFTMirr',
'/$Secure', '/$Secure:$SDS', '/$UpCase', '/$Volume', '/file1.txt',
'/file2.txt'
]
expected_paths_p2 = [
'/$AttrDef', '/$BadClus', '/$BadClus:$Bad', '/$Bitmap', '/$Boot',
'/$Extend', '/$Extend/$ObjId', '/$Extend/$Quota',
'/$Extend/$Reparse', '/$Extend/$RmMetadata',
'/$Extend/$RmMetadata/$Repair',
'/$Extend/$RmMetadata/$Repair:$Config',
'/$Extend/$RmMetadata/$TxfLog', '/$LogFile', '/$MFT', '/$MFTMirr',
'/$Secure', '/$Secure:$SDS', '/$UpCase', '/$Volume',
'/file1_on_part_2.txt', '/file2_on_part_2.txt'
]
paths = self._GetFilePaths(path_specs)
expected_paths = expected_paths_p1
expected_paths.extend(expected_paths_p2)
self.assertEqual(len(path_specs), len(expected_paths))
self.assertEqual(sorted(paths), sorted(expected_paths))
def GetEntries(self, parser_context, match=None, **unused_kwargs):
"""Extracts relevant user timestamp entries.
Args:
parser_context: A parser context object (instance of ParserContext).
match: Optional dictionary containing keys extracted from PLIST_KEYS.
The default is None.
"""
account = match['name'][0]
uid = match['uid'][0]
cocoa_zero = (
timelib.Timestamp.COCOA_TIME_TO_POSIX_BASE *
timelib.Timestamp.MICRO_SECONDS_PER_SECOND)
# INFO: binplist return a string with the Plist XML.
for policy in match['passwordpolicyoptions']:
xml_policy = ElementTree.fromstring(policy)
for dict_elements in xml_policy.iterfind('dict'):
key_values = [value.text for value in dict_elements.getchildren()]
policy_dict = dict(zip(key_values[0::2], key_values[1::2]))
if policy_dict.get('passwordLastSetTime', 0):
timestamp = timelib.Timestamp.FromTimeString(
policy_dict.get('passwordLastSetTime', '0'))
if timestamp > cocoa_zero:
# Extract the hash password information.
# It is store in the attribure ShadowHasData which is
# a binary plist data; However binplist only extract one
# level of binary plist, then it returns this information
# as a string.
# TODO: change this into a DataRange instead. For this we
# need the file offset and size of the ShadowHashData value data.
resolver_context = context.Context()
fake_file = fake_file_io.FakeFile(
resolver_context, match['ShadowHashData'][0])
fake_file.open(path_spec=fake_path_spec.FakePathSpec(
location=u'ShadowHashData'))
try:
plist_file = binplist.BinaryPlist(file_obj=fake_file)
top_level = plist_file.Parse()
except binplist.FormatError:
top_level = dict()
salted_hash = top_level.get('SALTED-SHA512-PBKDF2', None)
if salted_hash:
password_hash = u'$ml${0:d}${1:s}${2:s}'.format(
salted_hash['iterations'],
binascii.hexlify(salted_hash['salt']),
binascii.hexlify(salted_hash['entropy']))
else:
password_hash = u'N/A'
description = (
u'Last time {0:s} ({1!s}) changed the password: {2!s}').format(
account, uid, password_hash)
event_object = plist_event.PlistTimeEvent(
self._ROOT, u'passwordLastSetTime', timestamp, description)
parser_context.ProduceEvent(event_object, plugin_name=self.NAME)
if policy_dict.get('lastLoginTimestamp', 0):
timestamp = timelib.Timestamp.FromTimeString(
policy_dict.get('lastLoginTimestamp', '0'))
description = u'Last login from {0:s} ({1!s})'.format(account, uid)
if timestamp > cocoa_zero:
event_object = plist_event.PlistTimeEvent(
self._ROOT, u'lastLoginTimestamp', timestamp, description)
parser_context.ProduceEvent(event_object, plugin_name=self.NAME)
if policy_dict.get('failedLoginTimestamp', 0):
timestamp = timelib.Timestamp.FromTimeString(
policy_dict.get('failedLoginTimestamp', '0'))
description = (
u'Last failed login from {0:s} ({1!s}) ({2!s} times)').format(
account, uid, policy_dict['failedLoginCount'])
if timestamp > cocoa_zero:
event_object = plist_event.PlistTimeEvent(
self._ROOT, u'failedLoginTimestamp', timestamp, description)
parser_context.ProduceEvent(event_object, plugin_name=self.NAME)
def _Main(self):
"""The main loop."""
# We need a resolver context per process to prevent multi processing
# issues with file objects stored in images.
resolver_context = context.Context()
for credential_configuration in self._processing_configuration.credentials:
resolver.Resolver.key_chain.SetCredential(
credential_configuration.path_spec,
credential_configuration.credential_type,
credential_configuration.credential_data)
self._parser_mediator = parsers_mediator.ParserMediator(
None, self._knowledge_base,
preferred_year=self._processing_configuration.preferred_year,
resolver_context=resolver_context,
temporary_directory=self._processing_configuration.temporary_directory)
self._parser_mediator.SetEventExtractionConfiguration(
self._processing_configuration.event_extraction)
self._parser_mediator.SetInputSourceConfiguration(
self._processing_configuration.input_source)
# We need to initialize the parser and hasher objects after the process
# has forked otherwise on Windows the "fork" will fail with
# a PickleError for Python modules that cannot be pickled.
self._extraction_worker = worker.EventExtractionWorker(
parser_filter_expression=(
self._processing_configuration.parser_filter_expression))
self._extraction_worker.SetExtractionConfiguration(
self._processing_configuration.extraction)
self._StartProfiling()
logging.debug('Worker: {0!s} (PID: {1:d}) started'.format(
self._name, self._pid))
self._status = definitions.PROCESSING_STATUS_RUNNING
try:
logging.debug('{0!s} (PID: {1:d}) started monitoring task queue.'.format(
self._name, self._pid))
while not self._abort:
try:
task = self._task_queue.PopItem()
except (errors.QueueClose, errors.QueueEmpty) as exception:
logging.debug('ConsumeItems exiting with exception {0:s}.'.format(
type(exception)))
break
if isinstance(task, plaso_queue.QueueAbort):
logging.debug('ConsumeItems exiting, dequeued QueueAbort object.')
break
self._ProcessTask(task)
logging.debug('{0!s} (PID: {1:d}) stopped monitoring task queue.'.format(
self._name, self._pid))
# All exceptions need to be caught here to prevent the process
# from being killed by an uncaught exception.
except Exception as exception: # pylint: disable=broad-except
logging.warning(
'Unhandled exception in process: {0!s} (PID: {1:d}).'.format(
self._name, self._pid))
logging.exception(exception)
self._abort = True
self._StopProfiling()
self._extraction_worker = None
self._parser_mediator = None
self._storage_writer = None
if self._abort:
self._status = definitions.PROCESSING_STATUS_ABORTED
else:
self._status = definitions.PROCESSING_STATUS_COMPLETED
logging.debug('Worker: {0!s} (PID: {1:d}) stopped'.format(
self._name, self._pid))
try:
self._task_queue.Close(abort=self._abort)
except errors.QueueAlreadyClosed:
logging.error('Queue for {0:s} was already closed.'.format(self.name))
def GetEntries(self, parser_mediator, match=None, **unused_kwargs):
"""Extracts relevant user timestamp entries.
Args:
parser_mediator (ParserMediator): mediates interactions between parsers
and other components, such as storage and dfvfs.
match (Optional[dict[str: object]]): keys extracted from PLIST_KEYS.
"""
if 'name' not in match or 'uid' not in match:
return
account = match['name'][0]
uid = match['uid'][0]
# INFO: binplist return a string with the Plist XML.
for policy in match.get('passwordpolicyoptions', []):
try:
xml_policy = ElementTree.fromstring(policy)
except (ElementTree.ParseError, LookupError) as exception:
logging.error((
'Unable to parse XML structure for an user policy, account: '
'{0:s} and uid: {1!s}, with error: {2!s}').format(
account, uid, exception))
continue
for dict_elements in xml_policy.iterfind('dict'):
key_values = [value.text for value in dict_elements.getchildren()]
# Taking a list and converting it to a dict, using every other item
# as the key and the other one as the value.
policy_dict = dict(zip(key_values[0::2], key_values[1::2]))
time_string = policy_dict.get('passwordLastSetTime', None)
if time_string and time_string != '2001-01-01T00:00:00Z':
try:
date_time = dfdatetime_time_elements.TimeElements()
date_time.CopyFromStringISO8601(time_string)
except ValueError:
date_time = None
parser_mediator.ProduceExtractionError(
'unable to parse passworkd last set time string: {0:s}'.format(
time_string))
shadow_hash_data = match.get('ShadowHashData', None)
if date_time and isinstance(shadow_hash_data, (list, tuple)):
# Extract the hash password information.
# It is store in the attribute ShadowHasData which is
# a binary plist data; However binplist only extract one
# level of binary plist, then it returns this information
# as a string.
# TODO: change this into a DataRange instead. For this we
# need the file offset and size of the ShadowHashData value data.
shadow_hash_data = shadow_hash_data[0]
resolver_context = context.Context()
fake_file = fake_file_io.FakeFile(
resolver_context, shadow_hash_data)
shadow_hash_data_path_spec = fake_path_spec.FakePathSpec(
location='ShadowHashData')
fake_file.open(path_spec=shadow_hash_data_path_spec)
try:
plist_file = binplist.BinaryPlist(file_obj=fake_file)
top_level = plist_file.Parse()
except binplist.FormatError:
top_level = dict()
salted_hash = top_level.get('SALTED-SHA512-PBKDF2', None)
if salted_hash:
password_hash = '$ml${0:d}${1:s}${2:s}'.format(
salted_hash['iterations'],
binascii.hexlify(salted_hash['salt']),
binascii.hexlify(salted_hash['entropy']))
else:
password_hash = 'N/A'
event_data = plist_event.PlistTimeEventData()
event_data.desc = (
'Last time {0:s} ({1!s}) changed the password: {2!s}').format(
account, uid, password_hash)
event_data.key = 'passwordLastSetTime'
event_data.root = self._ROOT
event = time_events.DateTimeValuesEvent(
date_time, definitions.TIME_DESCRIPTION_WRITTEN)
parser_mediator.ProduceEventWithEventData(event, event_data)
time_string = policy_dict.get('lastLoginTimestamp', None)
if time_string and time_string != '2001-01-01T00:00:00Z':
try:
date_time = dfdatetime_time_elements.TimeElements()
date_time.CopyFromStringISO8601(time_string)
except ValueError:
date_time = None
parser_mediator.ProduceExtractionError(
'unable to parse last login time string: {0:s}'.format(
time_string))
if date_time:
event_data = plist_event.PlistTimeEventData()
event_data.desc = 'Last login from {0:s} ({1!s})'.format(
account, uid)
event_data.key = 'lastLoginTimestamp'
event_data.root = self._ROOT
event = time_events.DateTimeValuesEvent(
date_time, definitions.TIME_DESCRIPTION_WRITTEN)
parser_mediator.ProduceEventWithEventData(event, event_data)
time_string = policy_dict.get('failedLoginTimestamp', None)
if time_string and time_string != '2001-01-01T00:00:00Z':
try:
date_time = dfdatetime_time_elements.TimeElements()
date_time.CopyFromStringISO8601(time_string)
except ValueError:
date_time = None
parser_mediator.ProduceExtractionError(
'unable to parse failed login time string: {0:s}'.format(
time_string))
if date_time:
event_data = plist_event.PlistTimeEventData()
event_data.desc = (
'Last failed login from {0:s} ({1!s}) ({2!s} times)').format(
account, uid, policy_dict.get('failedLoginCount', 0))
event_data.key = 'failedLoginTimestamp'
event_data.root = self._ROOT
event = time_events.DateTimeValuesEvent(
date_time, definitions.TIME_DESCRIPTION_WRITTEN)
parser_mediator.ProduceEventWithEventData(event, event_data)
def setUp(self):
"""Sets up the needed objects used throughout the test."""
self._resolver_context = context.Context()
def __init__(
self, debug_output=False, enable_profiling=False,
maximum_number_of_tasks=_MAXIMUM_NUMBER_OF_TASKS,
profiling_directory=None, profiling_sample_rate=1000,
profiling_type=u'all', use_zeromq=True):
"""Initializes an engine object.
Args:
debug_output (Optional[bool]): True if debug output should be enabled.
enable_profiling (Optional[bool]): True if profiling should be enabled.
maximum_number_of_tasks (Optional[int]): maximum number of concurrent
tasks, where 0 represents no limit.
profiling_directory (Optional[str]): path to the directory where
the profiling sample files should be stored.
profiling_sample_rate (Optional[int]): the profiling sample rate.
Contains the number of event sources processed.
profiling_type (Optional[str]): type of profiling.
Supported types are:
* 'memory' to profile memory usage;
* 'parsers' to profile CPU time consumed by individual parsers;
* 'processing' to profile CPU time consumed by different parts of
the processing;
* 'serializers' to profile CPU time consumed by individual
serializers.
use_zeromq (Optional[bool]): True if ZeroMQ should be used for queuing
instead of Python's multiprocessing queue.
"""
super(TaskMultiProcessEngine, self).__init__(
debug_output=debug_output, enable_profiling=enable_profiling,
profiling_directory=profiling_directory,
profiling_sample_rate=profiling_sample_rate,
profiling_type=profiling_type)
self._enable_sigsegv_handler = False
self._filter_find_specs = None
self._filter_object = None
self._hasher_names_string = None
self._last_worker_number = 0
self._maximum_number_of_tasks = maximum_number_of_tasks
self._memory_profiler = None
self._merge_task = None
self._merge_task_on_hold = None
self._mount_path = None
self._number_of_consumed_errors = 0
self._number_of_consumed_event_tags = 0
self._number_of_consumed_events = 0
self._number_of_consumed_reports = 0
self._number_of_consumed_sources = 0
self._number_of_produced_errors = 0
self._number_of_produced_event_tags = 0
self._number_of_produced_events = 0
self._number_of_produced_reports = 0
self._number_of_produced_sources = 0
self._number_of_worker_processes = 0
self._parser_filter_expression = None
self._preferred_year = None
self._process_archives = False
self._process_compressed_streams = True
self._processing_profiler = None
self._resolver_context = context.Context()
self._serializers_profiler = None
self._session_identifier = None
self._status = definitions.PROCESSING_STATUS_IDLE
self._storage_merge_reader = None
self._storage_merge_reader_on_hold = None
self._storage_writer = None
self._task_queue = None
self._task_queue_port = None
self._task_manager = task_manager.TaskManager(
maximum_number_of_tasks=maximum_number_of_tasks)
self._temporary_directory = None
self._text_prepend = None
self._use_zeromq = use_zeromq
self._yara_rules_string = None
