def AppendSource(self, type_indicator, attributes):
"""Appends a source.
If you want to implement your own source type you should create a subclass
in source_type.py and change the AppendSource method to handle the new
subclass. This function raises FormatError if an unsupported source type
indicator is encountered.
Args:
type_indicator: the source type indicator.
attributes: a dictionary containing the source attributes.
Returns:
The source type object (instance of SourceType) or None if the type
indicator is not supported.
Raises:
FormatError: if the type indicator is not set or unsupported,
or if required attributes are missing.
"""
if not type_indicator:
raise errors.FormatError(u'Missing type indicator.')
try:
source_object = registry.ArtifactDefinitionsRegistry.CreateSourceType(
type_indicator, attributes)
except (AttributeError, TypeError) as exception:
raise errors.FormatError(
u'Invalid artifact definition for {0}: {1}'.format(
self.name, exception))
self.sources.append(source_object)
return source_object
def _ReadSupportedOS(self, yaml_definition, definition_object, name):
"""Reads the optional artifact or source type supported OS.
Args:
yaml_definition: the YAML artifact definition.
definition_object: the definition object (instance of ArtifactDefinition
or SourceType).
name: string containing the name of the artifact definition.
Raises:
FormatError: if there are undefined supported operating systems.
"""
supported_os = yaml_definition.get(u'supported_os', [])
if not isinstance(supported_os, list):
raise errors.FormatError(
u'supported_os must be a list of strings, got: {0:s}'.format(
supported_os))
undefined_supported_os = [
item for item in supported_os if item not in definitions.SUPPORTED_OS]
if undefined_supported_os:
raise errors.FormatError((
u'Artifact definition: {0:s} supported operating system: {1:s} '
u'not defined.').format(name, u', '.join(undefined_supported_os)))
definition_object.supported_os = supported_os
def _ReadSupportedOS(self, definition_values, definition_object, name):
"""Reads the optional artifact or source type supported OS.
Args:
definition_values (dict[str, object]): artifact definition values.
definition_object (ArtifactDefinition|SourceType): the definition object.
name (str): name of the artifact definition.
Raises:
FormatError: if there are undefined supported operating systems.
"""
supported_os = definition_values.get('supported_os', [])
if not isinstance(supported_os, list):
raise errors.FormatError('Invalid supported_os type: {0!s}'.format(
type(supported_os)))
undefined_supported_os = set(supported_os).difference(
self.supported_os)
if undefined_supported_os:
error_string = (
'Artifact definition: {0:s} undefined supported operating system: '
'{1:s}.').format(name, ', '.join(undefined_supported_os))
raise errors.FormatError(error_string)
definition_object.supported_os = supported_os
def _ReadSupportedOS(self, definition_values, definition_object, name):
"""Reads the optional artifact or source type supported OS.
Args:
definition_values: a dictionary containing the artifact defintion or
source type values.
definition_object: the definition object (instance of ArtifactDefinition
or SourceType).
name: string containing the name of the artifact definition.
Raises:
FormatError: if there are undefined supported operating systems.
"""
supported_os = definition_values.get(u'supported_os', [])
if not isinstance(supported_os, list):
raise errors.FormatError(u'Invalid supported_os type: {0}'.format(
type(supported_os)))
undefined_supported_os = set(supported_os).difference(
self.supported_os)
if undefined_supported_os:
error_string = (
u'Artifact definition: {0} undefined supported operating system: '
u'{1}.').format(name, u', '.join(undefined_supported_os))
raise errors.FormatError(error_string)
definition_object.supported_os = supported_os
def _LoadFieldDefinitions(self, data, field_definitions):
for field in field_definitions:
name = field["name"]
default = field.get("default")
required_type = field.get("type")
if required_type in (str, unicode):
required_type = basestring
if default is None and required_type is not None:
# basestring cant be instantiated.
if required_type is basestring:
default = ""
else:
default = required_type()
if required_type is None and default is not None:
required_type = type(default)
if not field.get("optional"):
if name not in data:
raise errors.FormatError(
u'Missing fields {}.'.format(name))
value = data.get(name, default)
if default is not None and not isinstance(value, required_type):
raise errors.FormatError(
u'field {} has type {} should be {}.'.format(
name, type(data[name]), required_type))
if field.get("checker"):
value = field["checker"](self, data)
setattr(self, name, value)
def BuildSources(self, art_definition):
sources = art_definition["sources"]
result = []
self.unsupported_source_types = []
for source in sources:
if not isinstance(source, dict):
raise errors.FormatError("Source is not a dict.")
source_type_name = source.get("type")
if source_type_name is None:
raise errors.FormatError("Source has no type.")
source_cls = self.source_types.get(source_type_name)
if source_cls:
result.append(source_cls(source, artifact=self))
else:
self.unsupported_source_types.append(source_type_name)
if not result:
if self.unsupported_source_types:
raise errors.FormatError("No supported sources: %s" %
(self.unsupported_source_types, ))
raise errors.FormatError("No available sources.")
return result
def AppendSource(self, type_indicator, attributes):
"""Appends a source.
If you want to implement your own source type you should create a subclass
in source_type.py and change the AppendSource method to handle the new
subclass. This function raises FormatError if an unsupported source type
indicator is encountered.
Args:
type_indicator (str): source type indicator.
attributes (dict[str, object]): source attributes.
Returns:
SourceType: a source type.
Raises:
FormatError: if the type indicator is not set or unsupported,
or if required attributes are missing.
"""
if not type_indicator:
raise errors.FormatError('Missing type indicator.')
try:
source_object = registry.ArtifactDefinitionsRegistry.CreateSourceType(
type_indicator, attributes)
except (AttributeError, TypeError) as exception:
raise errors.FormatError((
'Unable to create source type: {0:s} for artifact definition: {1:s} '
'with error: {2!s}').format(type_indicator, self.name, exception))
self.sources.append(source_object)
return source_object
def __init__(self, key_value_pairs=None):
"""Initializes a source type.
Args:
key_value_pairs (Optional[list[tuple[str, str]]]): key path and value
name pairs, where key paths are relative to the root of the Windows
Registry.
Raises:
FormatError: when key value pairs is not set.
"""
if not key_value_pairs:
raise errors.FormatError('Missing key value pairs value.')
if not isinstance(key_value_pairs, list):
raise errors.FormatError('key_value_pairs must be a list')
for pair in key_value_pairs:
if not isinstance(pair, dict):
raise errors.FormatError('key_value_pair must be a dict')
if set(pair.keys()) != set(['key', 'value']):
key_value_pairs = ', '.join([
'{0:s}: {1:s}'.format(key, value) for key, value in key_value_pairs
])
error_message = (
'key_value_pair missing "key" and "value" keys, got: '
'{0:s}').format(key_value_pairs)
raise errors.FormatError(error_message)
WindowsRegistryKeySourceType.ValidateKey(pair['key'])
super(WindowsRegistryValueSourceType, self).__init__()
self.key_value_pairs = key_value_pairs
def __init__(self, key_value_pairs=None):
"""Initializes the source type object.
Args:
key_value_pairs: optional list of key path and value name pairs.
The key paths are considered relative to the root
of the Windows Registry. The default is None.
Raises:
FormatError: when key value pairs is not set.
"""
if not key_value_pairs:
raise errors.FormatError(u'Missing key value pairs value.')
if not isinstance(key_value_pairs, list):
raise errors.FormatError(u'key_value_pairs must be a list')
for pair in key_value_pairs:
if not isinstance(pair, dict):
raise errors.FormatError(u'key_value_pair must be a dict')
if set(pair.keys()) != set([u'key', u'value']):
error_message = (
u'key_value_pair missing "key" and "value" keys, got: {0}'
).format(key_value_pairs)
raise errors.FormatError(error_message)
WindowsRegistryKeySourceType.ValidateKey(pair['key'])
super(WindowsRegistryValueSourceType, self).__init__()
self.key_value_pairs = key_value_pairs
def AppendSource(self, type_indicator, attributes):
"""Appends a source.
If you want to implement your own source type you should create a subclass
in source_type.py and change the AppendSource method to handle the new
subclass. This function raises FormatError if an unsupported source type
indicator is encountered.
Args:
type_indicator: the source type indicator.
attributes: a dictionary containing the source attributes.
Returns:
The source type object (instance of SourceType) or None if the type
indicator is not supported.
Raises:
FormatError: if the type indicator is not set or unsupported,
or if required attributes are missing.
"""
if not type_indicator:
raise errors.FormatError(u'Missing type indicator.')
source_type_class = None
if type_indicator == definitions.TYPE_INDICATOR_ARTIFACT_GROUP:
source_type_class = source_type.ArtifactGroupSourceType
elif type_indicator == definitions.TYPE_INDICATOR_COMMAND:
source_type_class = source_type.CommandSourceType
elif type_indicator == definitions.TYPE_INDICATOR_COMMAND:
source_type_class = source_type.CommandCollectorDefinition
elif type_indicator == definitions.TYPE_INDICATOR_DIRECTORY:
source_type_class = source_type.DirectorySourceType
elif type_indicator == definitions.TYPE_INDICATOR_FILE:
source_type_class = source_type.FileSourceType
elif type_indicator == definitions.TYPE_INDICATOR_PATH:
source_type_class = source_type.PathSourceType
elif type_indicator == definitions.TYPE_INDICATOR_WINDOWS_REGISTRY_KEY:
source_type_class = source_type.WindowsRegistryKeySourceType
elif type_indicator == definitions.TYPE_INDICATOR_WINDOWS_REGISTRY_VALUE:
source_type_class = source_type.WindowsRegistryValueSourceType
elif type_indicator == definitions.TYPE_INDICATOR_WMI_QUERY:
source_type_class = source_type.WMIQuerySourceType
else:
raise errors.FormatError(
u'Unsupported type indicator: {0}.'.format(type_indicator))
source_object = source_type_class(**attributes)
self.sources.append(source_object)
return source_object
def ReadArtifactDefinitionValues(self, artifact_definition_values):
"""Reads an artifact definition from a dictionary.
Args:
artifact_definition_values (dict[str, object]): artifact definition
values.
Returns:
ArtifactDefinition: an artifact definition.
Raises:
FormatError: if the format of the artifact definition is not set
or incorrect.
"""
if not artifact_definition_values:
raise errors.FormatError('Missing artifact definition values.')
different_keys = (set(artifact_definition_values) -
definitions.TOP_LEVEL_KEYS)
if different_keys:
different_keys = ', '.join(different_keys)
raise errors.FormatError(
'Undefined keys: {0:s}'.format(different_keys))
name = artifact_definition_values.get('name', None)
if not name:
raise errors.FormatError(
'Invalid artifact definition missing name.')
# The description is assumed to be mandatory.
description = artifact_definition_values.get('doc', None)
if not description:
raise errors.FormatError(
'Invalid artifact definition: {0:s} missing description.'.
format(name))
artifact_definition = artifact.ArtifactDefinition(
name, description=description)
if artifact_definition_values.get('collectors', []):
raise errors.FormatError(
'Invalid artifact definition: {0:s} still uses collectors.'.
format(name))
# TODO: check conditions.
artifact_definition.conditions = artifact_definition_values.get(
'conditions', [])
artifact_definition.provides = artifact_definition_values.get(
'provides', [])
self._ReadLabels(artifact_definition_values, artifact_definition, name)
self._ReadSupportedOS(artifact_definition_values, artifact_definition,
name)
artifact_definition.urls = artifact_definition_values.get('urls', [])
self._ReadSources(artifact_definition_values, artifact_definition,
name)
return artifact_definition
def _ReadArtifactDefinition(self, yaml_definition):
"""Reads an artifact definition.
Args:
yaml_definition: the YAML artifact definition.
Returns:
An artifact definition (instance of ArtifactDefinition).
Raises:
FormatError: if the format of the YAML artifact definition is not set
or incorrect.
"""
if not yaml_definition:
raise errors.FormatError(u'Missing YAML definition.')
name = yaml_definition.get('name', None)
if not name:
raise errors.FormatError(
u'Invalid artifact definition missing name.')
# The description is assumed to be mandatory.
description = yaml_definition.get('doc', None)
if not description:
raise errors.FormatError(
u'Invalid artifact definition missing description.')
artifact_definition = artifact.ArtifactDefinition(
name, description=description)
for collector in yaml_definition.get('collectors', []):
type_indicator = collector.get('collector_type', None)
if not type_indicator:
raise errors.FormatError(
u'Invalid artifact definition collector missing type.')
arguments = collector.get('args', None)
collector_definition = artifact_definition.AppendCollector(
type_indicator, arguments)
if collector_definition:
collector_definition.conditions = collector.get(
'conditions', [])
collector_definition.returned_types = collector.get(
'returned_types', [])
self._ReadSupportedOS(yaml_definition, collector_definition,
name)
# TODO: check conditions.
artifact_definition.conditions = yaml_definition.get('conditions', [])
artifact_definition.provides = yaml_definition.get('provides', [])
self._ReadLabels(yaml_definition, artifact_definition, name)
self._ReadSupportedOS(yaml_definition, artifact_definition, name)
artifact_definition.urls = yaml_definition.get('urls', [])
return artifact_definition
def __init__(self, source_definition, **kw):
super(RekallEFilterArtifacts, self).__init__(source_definition, **kw)
for column in self.fields:
if "name" not in column or "type" not in column:
raise errors.FormatError(
u"Field definition should have both name and type.")
mapped_type = column["type"]
if mapped_type not in self.allowed_types:
raise errors.FormatError(u"Unsupported type %s." % mapped_type)
def _LoadDefinition(self, data):
if not isinstance(data, dict):
raise errors.FormatError("Artifact definition must be a dict.")
different_keys = set(data) - definitions.TOP_LEVEL_KEYS
if different_keys:
raise errors.FormatError(
u'Undefined keys: {}'.format(different_keys))
self._LoadFieldDefinitions(data, self._field_definitions)
def _ReadSources(self, artifact_definition_values, artifact_definition,
name):
"""Reads the artifact definition sources.
Args:
artifact_definition_values (dict[str, object]): artifact definition
values.
artifact_definition (ArtifactDefinition): an artifact definition.
name (str): name of the artifact definition.
Raises:
FormatError: if the type indicator is not set or unsupported,
or if required attributes are missing.
"""
sources = artifact_definition_values.get('sources')
if not sources:
raise errors.FormatError(
'Invalid artifact definition: {0:s} missing sources.'.format(
name))
for source in sources:
type_indicator = source.get('type', None)
if not type_indicator:
raise errors.FormatError(
'Invalid artifact definition: {0:s} source type.'.format(
name))
attributes = source.get('attributes', None)
try:
source_type = artifact_definition.AppendSource(
type_indicator, attributes)
except errors.FormatError as exception:
raise errors.FormatError(
'Invalid artifact definition: {0:s}, with error: {1!s}'.
format(name, exception))
# TODO: deprecate these left overs from the collector definition.
if source_type:
if source.get('returned_types', None):
raise errors.FormatError((
'Invalid artifact definition: {0:s} returned_types no longer '
'supported.').format(name))
source_type.conditions = source.get('conditions', [])
self._ReadSupportedOS(source, source_type, name)
if set(source_type.supported_os) - set(
artifact_definition.supported_os):
raise errors.FormatError(
('Invalid artifact definition: {0:s} missing '
'supported_os.').format(name))
def __init__(self, data):
try:
self._LoadDefinition(data)
except Exception as e:
exc_info = sys.exc_info()
raise errors.FormatError(
"Definition %s: %s" % (self.name, e)), None, exc_info[2]
def ReadFileObject(self, file_object):
"""Reads artifact definitions from a file-like object.
Args:
file_object (file): file-like object to read from.
Yields:
ArtifactDefinition: an artifact definition.
Raises:
FormatError: if the format of the YAML artifact definition is not set
or incorrect.
"""
# TODO: add try, except?
yaml_generator = yaml.safe_load_all(file_object)
last_artifact_definition = None
for yaml_definition in yaml_generator:
try:
artifact_definition = self.ReadArtifactDefinitionValues(yaml_definition)
except errors.FormatError as exception:
error_location = 'At start'
if last_artifact_definition:
error_location = 'After: {0:s}'.format(last_artifact_definition.name)
raise errors.FormatError(
'{0:s} {1!s}'.format(error_location, exception))
yield artifact_definition
last_artifact_definition = artifact_definition
def ReadFileObject(self, file_object):
"""Reads artifact definitions from a file-like object.
Args:
file_object (file): file-like object to read from.
Yields:
ArtifactDefinition: an artifact definition.
Raises:
FormatError: if the format of the JSON artifact definition is not set
or incorrect.
"""
# TODO: add try, except?
json_definitions = json.loads(file_object.read())
last_artifact_definition = None
for json_definition in json_definitions:
try:
artifact_definition = self.ReadArtifactDefinitionValues(json_definition)
except errors.FormatError as exception:
error_location = 'At start'
if last_artifact_definition:
error_location = 'After: {0:s}'.format(last_artifact_definition.name)
raise errors.FormatError(
'{0:s} {1!s}'.format(error_location, exception))
yield artifact_definition
last_artifact_definition = artifact_definition
def _ReadSupportedOS(self, yaml_definition, definition_object, name):
"""Reads the optional artifact or collector definition supported OS.
Args:
yaml_definition: the YAML artifact definition.
defintion_object: the definition object (instance of ArtifactDefinition
or CollectorDefinition).
name: string containing the name of the arifact defintion.
Raises:
FormatError: if there are undefined supported operating systems.
"""
supported_os = yaml_definition.get('supported_os', [])
undefined_supported_os = [
item for item in supported_os
if item not in definitions.SUPPORTED_OS
]
if undefined_supported_os:
raise errors.FormatError((
u'Artifact definition: {0:s} supported operating system: {1:s} '
u'not defined.').format(name,
', '.join(undefined_supported_os)))
definition_object.supported_os = supported_os
def _ReadSources(self, artifact_definition_values, artifact_definition,
name):
"""Reads the artifact definition sources.
Args:
artifact_definition_values: dictionary containing the artifact definition
values.
artifact_definition: the artifact definition (instance of
ArtifactDefinition).09
Raises:
FormatError: if the type indicator is not set or unsupported,
or if required attributes are missing.
"""
sources = artifact_definition_values.get(u'sources')
if not sources:
raise errors.FormatError(
u'Invalid artifact definition: {0} missing sources.'.format(
name))
for source in sources:
type_indicator = source.get(u'type', None)
if not type_indicator:
raise errors.FormatError(
u'Invalid artifact definition: {0} source type.'.format(
name))
attributes = source.get(u'attributes', None)
try:
source_type = artifact_definition.AppendSource(
type_indicator, attributes)
except errors.FormatError as exception:
raise errors.FormatError(
u'Invalid artifact definition: {0}. {1}'.format(
name, exception))
# TODO: deprecate these left overs from the collector definition.
if source_type:
source_type.conditions = source.get(u'conditions', [])
source_type.returned_types = source.get(u'returned_types', [])
self._ReadSupportedOS(source, source_type, name)
if set(source_type.supported_os) - set(
artifact_definition.supported_os):
raise errors.FormatError(
(u'Invalid artifact definition: {0} missing '
u'supported_os.').format(name))
def AppendCollector(self, type_indicator, attributes):
"""Appends a collector definition.
If you want to implement your own collector definition you should create
a subclass in collector.py and change the AppendCollector method to
handle the new subclass. This function raises FormatError if an unsupported
collector type indicator is encountered.
Args:
type_indicator: the collector type indicator.
attributes: a dictionary containing the collector attributes.
Returns:
The collector definition object (instance of CollectorDefinition) or
None if the type indicator is not supported.
Raises:
FormatError: if the type indicator is not set or unsupported,
or if required attributes are missing.
"""
if not type_indicator:
raise errors.FormatError(u'Missing type indicator.')
collector_class = None
if type_indicator == definitions.TYPE_INDICATOR_ARTIFACT:
collector_class = collector.ArtifactCollectorDefinition
elif type_indicator == definitions.TYPE_INDICATOR_FILE:
collector_class = collector.FileCollectorDefinition
elif type_indicator == definitions.TYPE_INDICATOR_WINDOWS_REGISTRY_KEY:
collector_class = collector.WindowsRegistryKeyCollectorDefinition
elif type_indicator == definitions.TYPE_INDICATOR_WINDOWS_REGISTRY_VALUE:
collector_class = collector.WindowsRegistryValueCollectorDefinition
elif type_indicator == definitions.TYPE_INDICATOR_WMI_QUERY:
collector_class = collector.WMIQueryCollectorDefinition
else:
raise errors.FormatError(
u'Unsupported type indicator: {0:s}.'.format(type_indicator))
collector_definition = collector_class(**attributes)
self.collectors.append(collector_definition)
return collector_definition
def __init__(self, data, source_types=None):
self.source_types = source_types or SOURCE_TYPES
self.data = data
try:
self._LoadDefinition(data)
except Exception as e:
exc_info = sys.exc_info()
raise errors.FormatError("Definition %s: %s" % (self.name, e))
def __init__(self):
"""Initializes an artifact definition source type.
Raises:
FormatError: if the indicator is not defined.
"""
super(SourceType, self).__init__()
if not self.TYPE_INDICATOR:
raise errors.FormatError('Missing type indicator.')
def CheckKeyValuePairs(self, source):
key_value_pairs = source["key_value_pairs"]
for pair in key_value_pairs:
if (not isinstance(pair, dict) or "key" not in pair
or "value" not in pair):
raise errors.FormatError(
u"key_value_pairs should consist of dicts with key and "
"value items.")
return key_value_pairs
def __init__(self, paths=None, separator='/'):
"""Initializes a source type.
Args:
paths (Optional[str]): paths relative to the root of the file system.
separator (Optional[str]): path segment separator.
Raises:
FormatError: when paths is not set or not a list type.
"""
if not paths:
raise errors.FormatError('Missing paths value.')
if not isinstance(paths, list):
raise errors.FormatError('Invalid paths value, not a list.')
super(PathSourceType, self).__init__()
self.paths = paths
self.separator = separator
def ValidateKey(cls, key_path):
"""Validates this key against supported key names.
Args:
key_path: string containing the path fo the Registry key.
Raises:
FormatError: when key is not supported.
"""
for prefix in cls.VALID_PREFIXES:
if key_path.startswith(prefix):
return
if key_path.startswith(u'HKEY_CURRENT_USER\\'):
raise errors.FormatError(
u'HKEY_CURRENT_USER\\ is not supported instead use: '
u'HKEY_USERS\\%%users.sid%%\\')
raise errors.FormatError(
u'Unupported Registry key path: {0}'.format(key_path))
def __init__(self, keys=None):
"""Initializes a source type.
Args:
keys (Optional[list[str]]): key paths relative to the root of
the Windows Registry.
Raises:
FormatError: when keys is not set.
"""
if not keys:
raise errors.FormatError('Missing keys value.')
if not isinstance(keys, list):
raise errors.FormatError('keys must be a list')
for key in keys:
self.ValidateKey(key)
super(WindowsRegistryKeySourceType, self).__init__()
self.keys = keys
def ValidateKey(cls, key_path):
"""Validates this key against supported key names.
Args:
key_path (str): path of a Windows Registry key.
Raises:
FormatError: when key is not supported.
"""
for prefix in cls.VALID_PREFIXES:
if key_path.startswith(prefix):
return
# TODO: move check to validator.
if key_path.startswith('HKEY_CURRENT_USER\\'):
raise errors.FormatError(
'HKEY_CURRENT_USER\\ is not supported instead use: '
'HKEY_USERS\\%%users.sid%%\\')
raise errors.FormatError(
'Unupported Registry key path: {0:s}'.format(key_path))
def __init__(self, keys=None):
"""Initializes the source type object.
Args:
keys: optional list of key paths. The key paths are considered relative
to the root of the Windows Registry. The default is None.
Raises:
FormatError: when keys is not set.
"""
if not keys:
raise errors.FormatError(u'Missing keys value.')
if not isinstance(keys, list):
raise errors.FormatError(u'keys must be a list')
for key in keys:
self.ValidateKey(key)
super(WindowsRegistryKeySourceType, self).__init__()
self.keys = keys
def SupportedOS(self, art_definition):
supported_os = art_definition.get("supported_os",
definitions.SUPPORTED_OS)
undefined_supported_os = set(supported_os).difference(
definitions.SUPPORTED_OS)
if undefined_supported_os:
raise errors.FormatError(u'supported operating system: {} '
u'not defined.'.format(
u', '.join(undefined_supported_os)))
return supported_os
