def CheckFile(self, path):
"""Validates the definition in a file.
Args:
path (str): path of the definition file.
Returns:
bool: True if the file contains valid definitions.
"""
print('Checking: {0:s}'.format(path))
definitions_registry = registry.DataTypeDefinitionsRegistry()
definitions_reader = reader.YAMLDataTypeDefinitionsFileReader()
result = False
try:
definitions_reader.ReadFile(definitions_registry, path)
result = True
except KeyError as exception:
logging.warning(
('Unable to register data type definition in file: {0:s} with '
'error: {1:s}').format(path, exception))
except errors.FormatError as exception:
logging.warning(
'Unable to validate file: {0:s} with error: {1:s}'.format(
path, exception))
return result
def testReadFileObjectStructureWithSequenceWithExpression(self):
"""Tests the ReadFileObject function of a structure with a sequence."""
definitions_registry = registry.DataTypeDefinitionsRegistry()
definitions_reader = reader.YAMLDataTypeDefinitionsFileReader()
definitions_file = self._GetTestFilePath([u'structure.yaml'])
with open(definitions_file, 'rb') as file_object:
definitions_reader.ReadFileObject(definitions_registry,
file_object)
self.assertEqual(len(definitions_registry._definitions), 5)
data_type_definition = definitions_registry.GetDefinitionByName(
u'sphere3d')
self.assertIsInstance(data_type_definition,
data_types.StructureDefinition)
self.assertEqual(data_type_definition.name, u'sphere3d')
self.assertEqual(data_type_definition.description,
u'Sphere in 3 dimensional space.')
self.assertEqual(len(data_type_definition.members), 2)
member_definition = data_type_definition.members[1]
self.assertIsInstance(member_definition, data_types.SequenceDefinition)
self.assertEqual(member_definition.name, u'triangles')
self.assertEqual(member_definition.element_data_type, u'triangle3d')
self.assertIsNotNone(member_definition.element_data_type_definition)
byte_size = data_type_definition.GetByteSize()
self.assertIsNone(byte_size)
def testReadFileObjectSequence(self):
"""Tests the ReadFileObject function of a sequence data type."""
definitions_registry = registry.DataTypeDefinitionsRegistry()
definitions_reader = reader.YAMLDataTypeDefinitionsFileReader()
definitions_file = self._GetTestFilePath([u'sequence.yaml'])
with open(definitions_file, 'rb') as file_object:
definitions_reader.ReadFileObject(definitions_registry,
file_object)
self.assertEqual(len(definitions_registry._definitions), 2)
data_type_definition = definitions_registry.GetDefinitionByName(
u'vector4')
self.assertIsInstance(data_type_definition,
data_types.SequenceDefinition)
self.assertEqual(data_type_definition.name, u'vector4')
self.assertEqual(data_type_definition.description,
u'vector with 4 elements')
self.assertEqual(data_type_definition.aliases, [u'VECTOR'])
self.assertEqual(data_type_definition.element_data_type, u'int32')
self.assertIsNotNone(data_type_definition.element_data_type_definition)
self.assertEqual(data_type_definition.number_of_elements, 4)
byte_size = data_type_definition.GetByteSize()
self.assertEqual(byte_size, 16)
def testReadFileObjectStructure(self):
"""Tests the ReadFileObject function of a structure data type."""
definitions_registry = registry.DataTypeDefinitionsRegistry()
definitions_reader = reader.YAMLDataTypeDefinitionsFileReader()
definitions_file = self._GetTestFilePath([u'structure.yaml'])
with open(definitions_file, 'rb') as file_object:
definitions_reader.ReadFileObject(definitions_registry,
file_object)
self.assertEqual(len(definitions_registry._definitions), 5)
data_type_definition = definitions_registry.GetDefinitionByName(
u'point3d')
self.assertIsInstance(data_type_definition,
data_types.StructureDefinition)
self.assertEqual(data_type_definition.name, u'point3d')
self.assertEqual(data_type_definition.description,
u'Point in 3 dimensional space.')
self.assertEqual(data_type_definition.aliases, [u'POINT'])
self.assertEqual(len(data_type_definition.members), 3)
member_definition = data_type_definition.members[0]
self.assertIsInstance(member_definition,
data_types.StructureMemberDefinition)
self.assertEqual(member_definition.name, u'x')
self.assertEqual(member_definition.aliases, [u'XCOORD'])
self.assertEqual(member_definition.member_data_type, u'int32')
self.assertIsNotNone(member_definition.member_data_type_definition)
byte_size = data_type_definition.GetByteSize()
self.assertEqual(byte_size, 12)
def testReadFileObjectInteger(self):
"""Tests the ReadFileObject function of an integer data type."""
definitions_registry = registry.DataTypeDefinitionsRegistry()
definitions_reader = reader.YAMLDataTypeDefinitionsFileReader()
definitions_file = self._GetTestFilePath([u'integer.yaml'])
with open(definitions_file, 'rb') as file_object:
definitions_reader.ReadFileObject(definitions_registry,
file_object)
self.assertEqual(len(definitions_registry._definitions), 1)
data_type_definition = definitions_registry.GetDefinitionByName(
u'int32le')
self.assertIsInstance(data_type_definition,
data_types.IntegerDefinition)
self.assertEqual(data_type_definition.name, u'int32le')
self.assertEqual(data_type_definition.byte_order,
definitions.BYTE_ORDER_LITTLE_ENDIAN)
self.assertEqual(data_type_definition.format, u'signed')
self.assertEqual(data_type_definition.size, 4)
self.assertEqual(data_type_definition.units, u'bytes')
byte_size = data_type_definition.GetByteSize()
self.assertEqual(byte_size, 4)
def testReadFileObjectMissingType(self):
"""Tests the ReadFileObject function with a missing type."""
definitions_registry = registry.DataTypeDefinitionsRegistry()
definitions_reader = reader.YAMLDataTypeDefinitionsFileReader()
yaml_data = u'\n'.join([
u'name: int8', u'attributes:', u' format: signed', u' size: 1',
u' units: bytes'
]).encode(u'ascii')
file_object = io.BytesIO(initial_bytes=yaml_data)
with self.assertRaises(errors.FormatError):
definitions_reader.ReadFileObject(definitions_registry,
file_object)
yaml_data = u'\n'.join([
u'name: int8', u'type: integer', u'attributes:',
u' format: signed', u' size: 1', u' units: bytes', u'---',
u'name: int16', u'attributes:', u' format: signed', u' size: 2',
u' units: bytes'
]).encode(u'ascii')
file_object = io.BytesIO(initial_bytes=yaml_data)
with self.assertRaises(errors.FormatError):
definitions_reader.ReadFileObject(definitions_registry,
file_object)
def ReadDefinitions(self, path):
"""Reads the definitions form file or directory.
Args:
path (str): path of the definition file.
"""
definitions_reader = reader.YAMLDataTypeDefinitionsFileReader()
definitions_reader.ReadFile(self._definitions_registry, path)
def testReadDirectory(self):
"""Tests the ReadDirectory function."""
definitions_directory = self._GetTestFilePath([u'definitions'])
definitions_registry = registry.DataTypeDefinitionsRegistry()
definitions_reader = reader.YAMLDataTypeDefinitionsFileReader()
definitions_reader.ReadDirectory(definitions_registry,
definitions_directory)
def _GetDataFormatInformationFromPackage(self, package):
"""Retrieves event data attribute containers from a package.
Args:
package (list[str]): package name segments such as ["plaso", "parsers"].
Returns:
list[DataFormatDescriptor]: data format descriptors.
"""
data_format_descriptors = []
package_path = '/'.join(package)
for _, name, is_package in pkgutil.iter_modules(path=[package_path]):
sub_package = list(package)
sub_package.append(name)
if is_package:
sub_data_format_descriptors = (
self._GetDataFormatInformationFromPackage(sub_package))
data_format_descriptors.extend(sub_data_format_descriptors)
else:
module_path = '.'.join(sub_package)
module_object = importlib.import_module(module_path)
for _, cls in inspect.getmembers(module_object, inspect.isclass):
if issubclass(cls, (
parsers_interface.BaseParser, parsers_plugins.BasePlugin)):
# TODO: detect corresponding dtFabric .yaml file
parser_name = getattr(cls, 'NAME', None)
if not parser_name or parser_name in self._PARSER_NAME_IGNORE_LIST:
continue
category = self._DATA_FORMAT_CATEGORY_PER_PACKAGE_PATH.get(
package_path, 'File formats')
data_format = getattr(cls, 'DATA_FORMAT', None)
url = ''
dtfabric_file = os.path.join(package_path, ''.join([name, '.yaml']))
if os.path.exists(dtfabric_file):
definitions_registry = (
dtfabric_registry.DataTypeDefinitionsRegistry())
definitions_reader = (
dtfabric_reader.YAMLDataTypeDefinitionsFileReader())
try:
definitions_reader.ReadFile(definitions_registry, dtfabric_file)
# TODO: determine the URL using definitions_registry.
except Exception: # pylint: disable=broad-except
pass
data_format_descriptor = DataFormatDescriptor(
category=category, name=data_format, url=url)
data_format_descriptors.append(data_format_descriptor)
return data_format_descriptors
def _FillDefinitionRegistryFromFile(self, definitions_registry, path):
"""Fills a data type definition registry from a file.
Args:
definitions_registry (DataTypeDefinitionsRegistry): data type definitions
registry.
path (str): path to the data definition file.
"""
definitions_reader = reader.YAMLDataTypeDefinitionsFileReader()
with io.open(path, 'r', encoding='utf8') as file_object:
definitions_reader.ReadFileObject(definitions_registry,
file_object)
def ReadDefinitions(self, path):
"""Reads the definitions form file or directory.
Args:
path (str): path of the definition file or directory.
"""
definitions_reader = reader.YAMLDataTypeDefinitionsFileReader()
if os.path.isdir(path):
definitions_reader.ReadDirectory(
self._structure_definitions_registry, path)
else:
definitions_reader.ReadFile(self._structure_definitions_registry,
path)
def __init__(self, yaml_definition=None):
"""Initializes a data type fabric.
Args:
yaml_definition (str): YAML formatted data type definitions.
"""
definitions_registry = registry.DataTypeDefinitionsRegistry()
if yaml_definition:
definitions_reader = reader.YAMLDataTypeDefinitionsFileReader()
file_object = io.BytesIO(yaml_definition)
definitions_reader.ReadFileObject(definitions_registry,
file_object)
super(DataTypeFabric, self).__init__(definitions_registry)
def testReadFileObjectConstant(self):
"""Tests the ReadFileObject function of a constant data type."""
definitions_registry = registry.DataTypeDefinitionsRegistry()
definitions_reader = reader.YAMLDataTypeDefinitionsFileReader()
definitions_file = self._GetTestFilePath([u'constant.yaml'])
with open(definitions_file, 'rb') as file_object:
definitions_reader.ReadFileObject(definitions_registry,
file_object)
self.assertEqual(len(definitions_registry._definitions), 1)
data_type_definition = definitions_registry.GetDefinitionByName(
u'maximum_number_of_back_traces')
self.assertIsInstance(data_type_definition,
data_types.ConstantDefinition)
self.assertEqual(data_type_definition.name,
u'maximum_number_of_back_traces')
self.assertEqual(data_type_definition.value, 32)
def testReadFileObjectCharacter(self):
"""Tests the ReadFileObject function of a character data type."""
definitions_registry = registry.DataTypeDefinitionsRegistry()
definitions_reader = reader.YAMLDataTypeDefinitionsFileReader()
definitions_file = self._GetTestFilePath([u'character.yaml'])
with open(definitions_file, 'rb') as file_object:
definitions_reader.ReadFileObject(definitions_registry,
file_object)
self.assertEqual(len(definitions_registry._definitions), 1)
data_type_definition = definitions_registry.GetDefinitionByName(
u'char')
self.assertIsInstance(data_type_definition,
data_types.CharacterDefinition)
self.assertEqual(data_type_definition.name, u'char')
self.assertEqual(data_type_definition.size, 1)
self.assertEqual(data_type_definition.units, u'bytes')
byte_size = data_type_definition.GetByteSize()
self.assertEqual(byte_size, 1)
def testReadFileObjectEnumeration(self):
"""Tests the ReadFileObject function of an enumeration data type."""
definitions_registry = registry.DataTypeDefinitionsRegistry()
definitions_reader = reader.YAMLDataTypeDefinitionsFileReader()
definitions_file = self._GetTestFilePath([u'enumeration.yaml'])
with open(definitions_file, 'rb') as file_object:
definitions_reader.ReadFileObject(definitions_registry,
file_object)
self.assertEqual(len(definitions_registry._definitions), 1)
data_type_definition = definitions_registry.GetDefinitionByName(
u'object_information_type')
self.assertIsInstance(data_type_definition,
data_types.EnumerationDefinition)
self.assertEqual(data_type_definition.name, u'object_information_type')
self.assertEqual(data_type_definition.size, 4)
self.assertEqual(data_type_definition.units, u'bytes')
self.assertEqual(len(data_type_definition.values), 6)
byte_size = data_type_definition.GetByteSize()
self.assertEqual(byte_size, 4)
