def testReadFileObjectStructureWithSequenceWithExpression(self): """Tests the ReadFileObject function of a structure with a sequence.""" definitions_registry = registry.DataTypeDefinitionsRegistry() definitions_reader = reader.YAMLDataTypeDefinitionsFileReader() definitions_file = self._GetTestFilePath([u'structure.yaml']) with open(definitions_file, 'rb') as file_object: definitions_reader.ReadFileObject(definitions_registry, file_object) self.assertEqual(len(definitions_registry._definitions), 5) data_type_definition = definitions_registry.GetDefinitionByName( u'sphere3d') self.assertIsInstance(data_type_definition, data_types.StructureDefinition) self.assertEqual(data_type_definition.name, u'sphere3d') self.assertEqual(data_type_definition.description, u'Sphere in 3 dimensional space.') self.assertEqual(len(data_type_definition.members), 2) member_definition = data_type_definition.members[1] self.assertIsInstance(member_definition, data_types.SequenceDefinition) self.assertEqual(member_definition.name, u'triangles') self.assertEqual(member_definition.element_data_type, u'triangle3d') self.assertIsNotNone(member_definition.element_data_type_definition) byte_size = data_type_definition.GetByteSize() self.assertIsNone(byte_size)
def _GetDataFormatInformationFromPackage(self, package): """Retrieves event data attribute containers from a package. Args: package (list[str]): package name segments such as ["plaso", "parsers"]. Returns: list[DataFormatDescriptor]: data format descriptors. """ data_format_descriptors = [] package_path = '/'.join(package) for _, name, is_package in pkgutil.iter_modules(path=[package_path]): sub_package = list(package) sub_package.append(name) if is_package: sub_data_format_descriptors = ( self._GetDataFormatInformationFromPackage(sub_package)) data_format_descriptors.extend(sub_data_format_descriptors) else: module_path = '.'.join(sub_package) module_object = importlib.import_module(module_path) for _, cls in inspect.getmembers(module_object, inspect.isclass): if issubclass(cls, (parsers_interface.BaseParser, parsers_plugins.BasePlugin)): # TODO: detect corresponding dtFabric .yaml file parser_name = getattr(cls, 'NAME', None) if not parser_name or parser_name in self._PARSER_NAME_IGNORE_LIST: continue category = self._DATA_FORMAT_CATEGORY_PER_PACKAGE_PATH.get( package_path, 'File formats') data_format = getattr(cls, 'DATA_FORMAT', None) url = '' dtfabric_file = os.path.join(package_path, ''.join([name, '.yaml'])) if os.path.exists(dtfabric_file): definitions_registry = ( dtfabric_registry.DataTypeDefinitionsRegistry( )) definitions_reader = ( dtfabric_reader. YAMLDataTypeDefinitionsFileReader()) try: definitions_reader.ReadFile( definitions_registry, dtfabric_file) # TODO: determine the URL using definitions_registry. except Exception: # pylint: disable=broad-except pass data_format_descriptor = DataFormatDescriptor( category=category, name=data_format, url=url) data_format_descriptors.append(data_format_descriptor) return data_format_descriptors
def testRegistration(self): """Tests the RegisterDefinition and DeregisterDefinition functions.""" definitions_registry = registry.DataTypeDefinitionsRegistry() data_type_definition = data_types.IntegerDefinition( u'int32', aliases=[u'LONG', u'LONG32'], description=u'signed 32-bit integer') definitions_registry.RegisterDefinition(data_type_definition) with self.assertRaises(KeyError): definitions_registry.RegisterDefinition(data_type_definition) test_definition = data_types.IntegerDefinition( u'LONG', description=u'long integer') with self.assertRaises(KeyError): definitions_registry.RegisterDefinition(test_definition) test_definition = data_types.IntegerDefinition( u'test', aliases=[u'LONG'], description=u'long integer') with self.assertRaises(KeyError): definitions_registry.RegisterDefinition(test_definition) definitions_registry.DeregisterDefinition(data_type_definition) with self.assertRaises(KeyError): definitions_registry.DeregisterDefinition(data_type_definition)
def testReadConstantDataTypeDefinition(self): """Tests the _ReadConstantDataTypeDefinition function.""" definition_values = { u'aliases': [u'AVRF_MAX_TRACES'], u'description': (u'Application verifier resource enumeration maximum number of ' u'back traces'), u'value': 32, } definitions_registry = registry.DataTypeDefinitionsRegistry() definitions_reader = reader.DataTypeDefinitionsFileReader() data_type_definition = ( definitions_reader._ReadConstantDataTypeDefinition( definitions_registry, definition_values, u'const')) self.assertIsNotNone(data_type_definition) self.assertIsInstance(data_type_definition, data_types.ConstantDefinition) # Test with missing value definition. del definition_values[u'value'] with self.assertRaises(errors.DefinitionReaderError): definitions_reader._ReadConstantDataTypeDefinition( definitions_registry, definition_values, u'const')
def testReadUUIDDataTypeDefinition(self): """Tests the _ReadUUIDDataTypeDefinition function.""" definition_values = { u'aliases': [u'guid', u'GUID', u'UUID'], u'attributes': { u'byte_order': u'little-endian', }, u'description': (u'Globally or Universal unique identifier (GUID or UUID) type'), } definitions_registry = registry.DataTypeDefinitionsRegistry() definitions_reader = reader.DataTypeDefinitionsFileReader() data_type_definition = definitions_reader._ReadUUIDDataTypeDefinition( definitions_registry, definition_values, u'uuid') self.assertIsNotNone(data_type_definition) self.assertIsInstance(data_type_definition, data_types.UUIDDefinition) # Test with unsupported size. definition_values[u'attributes'][u'size'] = 32 with self.assertRaises(errors.DefinitionReaderError): definitions_reader._ReadUUIDDataTypeDefinition( definitions_registry, definition_values, u'uuid')
def testReadFileObjectMissingType(self): """Tests the ReadFileObject function with a missing type.""" definitions_registry = registry.DataTypeDefinitionsRegistry() definitions_reader = reader.YAMLDataTypeDefinitionsFileReader() yaml_data = u'\n'.join([ u'name: int8', u'attributes:', u' format: signed', u' size: 1', u' units: bytes' ]).encode(u'ascii') file_object = io.BytesIO(initial_bytes=yaml_data) with self.assertRaises(errors.FormatError): definitions_reader.ReadFileObject(definitions_registry, file_object) yaml_data = u'\n'.join([ u'name: int8', u'type: integer', u'attributes:', u' format: signed', u' size: 1', u' units: bytes', u'---', u'name: int16', u'attributes:', u' format: signed', u' size: 2', u' units: bytes' ]).encode(u'ascii') file_object = io.BytesIO(initial_bytes=yaml_data) with self.assertRaises(errors.FormatError): definitions_reader.ReadFileObject(definitions_registry, file_object)
def testReadDefinitionFromDict(self): """Tests the ReadDefinitionFromDict function.""" definition_values = { u'aliases': [u'LONG', u'LONG32'], u'attributes': { u'format': u'signed', u'size': 4, }, u'description': u'signed 32-bit integer type', u'name': u'int32', u'type': u'integer', } definitions_registry = registry.DataTypeDefinitionsRegistry() definitions_reader = reader.DataTypeDefinitionsFileReader() data_type_definition = definitions_reader.ReadDefinitionFromDict( definitions_registry, definition_values) self.assertIsNotNone(data_type_definition) self.assertIsInstance(data_type_definition, data_types.IntegerDefinition) with self.assertRaises(errors.DefinitionReaderError): definitions_reader.ReadDefinitionFromDict(definitions_registry, None) definition_values[u'type'] = u'bogus' with self.assertRaises(errors.DefinitionReaderError): definitions_reader.ReadDefinitionFromDict(definitions_registry, definition_values)
def testReadIntegerDataTypeDefinition(self): """Tests the _ReadIntegerDataTypeDefinition function.""" definition_values = { u'aliases': [u'LONG', u'LONG32'], u'attributes': { u'format': u'signed', u'size': 4, }, u'description': u'signed 32-bit integer type', } definitions_registry = registry.DataTypeDefinitionsRegistry() definitions_reader = reader.DataTypeDefinitionsFileReader() data_type_definition = definitions_reader._ReadIntegerDataTypeDefinition( definitions_registry, definition_values, u'int32') self.assertIsNotNone(data_type_definition) self.assertIsInstance(data_type_definition, data_types.IntegerDefinition) # Test with unsupported format attribute. definition_values[u'attributes'][u'format'] = u'bogus' with self.assertRaises(errors.DefinitionReaderError): definitions_reader._ReadIntegerDataTypeDefinition( definitions_registry, definition_values, u'int32')
def testReadFileObjectSequence(self): """Tests the ReadFileObject function of a sequence data type.""" definitions_registry = registry.DataTypeDefinitionsRegistry() definitions_reader = reader.YAMLDataTypeDefinitionsFileReader() definitions_file = self._GetTestFilePath([u'sequence.yaml']) with open(definitions_file, 'rb') as file_object: definitions_reader.ReadFileObject(definitions_registry, file_object) self.assertEqual(len(definitions_registry._definitions), 2) data_type_definition = definitions_registry.GetDefinitionByName( u'vector4') self.assertIsInstance(data_type_definition, data_types.SequenceDefinition) self.assertEqual(data_type_definition.name, u'vector4') self.assertEqual(data_type_definition.description, u'vector with 4 elements') self.assertEqual(data_type_definition.aliases, [u'VECTOR']) self.assertEqual(data_type_definition.element_data_type, u'int32') self.assertIsNotNone(data_type_definition.element_data_type_definition) self.assertEqual(data_type_definition.number_of_elements, 4) byte_size = data_type_definition.GetByteSize() self.assertEqual(byte_size, 16)
def testReadFileObjectStructure(self): """Tests the ReadFileObject function of a structure data type.""" definitions_registry = registry.DataTypeDefinitionsRegistry() definitions_reader = reader.YAMLDataTypeDefinitionsFileReader() definitions_file = self._GetTestFilePath([u'structure.yaml']) with open(definitions_file, 'rb') as file_object: definitions_reader.ReadFileObject(definitions_registry, file_object) self.assertEqual(len(definitions_registry._definitions), 5) data_type_definition = definitions_registry.GetDefinitionByName( u'point3d') self.assertIsInstance(data_type_definition, data_types.StructureDefinition) self.assertEqual(data_type_definition.name, u'point3d') self.assertEqual(data_type_definition.description, u'Point in 3 dimensional space.') self.assertEqual(data_type_definition.aliases, [u'POINT']) self.assertEqual(len(data_type_definition.members), 3) member_definition = data_type_definition.members[0] self.assertIsInstance(member_definition, data_types.StructureMemberDefinition) self.assertEqual(member_definition.name, u'x') self.assertEqual(member_definition.aliases, [u'XCOORD']) self.assertEqual(member_definition.member_data_type, u'int32') self.assertIsNotNone(member_definition.member_data_type_definition) byte_size = data_type_definition.GetByteSize() self.assertEqual(byte_size, 12)
def testReadFileObjectInteger(self): """Tests the ReadFileObject function of an integer data type.""" definitions_registry = registry.DataTypeDefinitionsRegistry() definitions_reader = reader.YAMLDataTypeDefinitionsFileReader() definitions_file = self._GetTestFilePath([u'integer.yaml']) with open(definitions_file, 'rb') as file_object: definitions_reader.ReadFileObject(definitions_registry, file_object) self.assertEqual(len(definitions_registry._definitions), 1) data_type_definition = definitions_registry.GetDefinitionByName( u'int32le') self.assertIsInstance(data_type_definition, data_types.IntegerDefinition) self.assertEqual(data_type_definition.name, u'int32le') self.assertEqual(data_type_definition.byte_order, definitions.BYTE_ORDER_LITTLE_ENDIAN) self.assertEqual(data_type_definition.format, u'signed') self.assertEqual(data_type_definition.size, 4) self.assertEqual(data_type_definition.units, u'bytes') byte_size = data_type_definition.GetByteSize() self.assertEqual(byte_size, 4)
def CheckFile(self, path): """Validates the definition in a file. Args: path (str): path of the definition file. Returns: bool: True if the file contains valid definitions. """ print('Checking: {0:s}'.format(path)) definitions_registry = registry.DataTypeDefinitionsRegistry() definitions_reader = reader.YAMLDataTypeDefinitionsFileReader() result = False try: definitions_reader.ReadFile(definitions_registry, path) result = True except KeyError as exception: logging.warning( ('Unable to register data type definition in file: {0:s} with ' 'error: {1:s}').format(path, exception)) except errors.FormatError as exception: logging.warning( 'Unable to validate file: {0:s} with error: {1:s}'.format( path, exception)) return result
def testReadFile(self): """Tests the ReadFile function.""" definitions_file = self._GetTestFilePath( [u'definitions', u'integers.yaml']) definitions_registry = registry.DataTypeDefinitionsRegistry() definitions_reader = reader.DataTypeDefinitionsFileReader() definitions_reader.ReadFile(definitions_registry, definitions_file)
def testReadDirectory(self): """Tests the ReadDirectory function.""" definitions_directory = self._GetTestFilePath([u'definitions']) definitions_registry = registry.DataTypeDefinitionsRegistry() definitions_reader = reader.YAMLDataTypeDefinitionsFileReader() definitions_reader.ReadDirectory(definitions_registry, definitions_directory)
def __init__(self, templates_path): """Initializes a generator. Args: templates_path (str): templates path. """ super(AsciidocFormatDocumentGenerator, self).__init__() self._definitions_registry = registry.DataTypeDefinitionsRegistry() self._templates_path = templates_path self._template_string_generator = template_string.TemplateStringGenerator( )
def __init__(self, templates_path): """Initializes a source generator. Args: templates_path (str): templates path. """ super(SourceGenerator, self).__init__() self._definitions_registry = registry.DataTypeDefinitionsRegistry() self._prefix = None self._templates_path = templates_path self._template_string_generator = template_string.TemplateStringGenerator( )
def _CreateDefinitionRegistryFromFile(self, path): """Creates a data type definition registry from a file. Args: path (str): path to the data definition file. Returns: DataTypeDefinitionsRegistry: data type definition registry or None on error. """ definitions_registry = registry.DataTypeDefinitionsRegistry() self._FillDefinitionRegistryFromFile(definitions_registry, path) return definitions_registry
def __init__(self, yaml_definition=None): """Initializes a data type fabric. Args: yaml_definition (str): YAML formatted data type definitions. """ definitions_registry = registry.DataTypeDefinitionsRegistry() if yaml_definition: definitions_reader = reader.YAMLDataTypeDefinitionsFileReader() file_object = io.BytesIO(yaml_definition) definitions_reader.ReadFileObject(definitions_registry, file_object) super(DataTypeFabric, self).__init__(definitions_registry)
def testReadFormatDefinition(self): """Tests the _ReadFormatDefinition function.""" definition_values = { u'description': u'Windows Shortcut (LNK) file format', u'type': u'format', } definitions_registry = registry.DataTypeDefinitionsRegistry() definitions_reader = reader.DataTypeDefinitionsFileReader() # TODO: implement. _ = definitions_registry data_type_definition = definitions_reader._ReadFormatDefinition( definition_values, u'lnk') self.assertIsNotNone(data_type_definition) self.assertIsInstance(data_type_definition, data_types.FormatDefinition)
def testReadCharacterDataTypeDefinition(self): """Tests the _ReadCharacterDataTypeDefinition function.""" definition_values = { u'aliases': [u'CHAR'], u'attributes': { u'size': 1, }, u'description': u'8-bit character type', } definitions_registry = registry.DataTypeDefinitionsRegistry() definitions_reader = reader.DataTypeDefinitionsFileReader() data_type_definition = definitions_reader._ReadCharacterDataTypeDefinition( definitions_registry, definition_values, u'char') self.assertIsNotNone(data_type_definition) self.assertIsInstance(data_type_definition, data_types.CharacterDefinition)
def testReadBooleanDataTypeDefinition(self): """Tests the _ReadBooleanDataTypeDefinition function.""" definition_values = { u'aliases': [u'BOOL'], u'attributes': { u'size': 4, }, u'description': u'32-bit boolean type', } definitions_registry = registry.DataTypeDefinitionsRegistry() definitions_reader = reader.DataTypeDefinitionsFileReader() data_type_definition = definitions_reader._ReadBooleanDataTypeDefinition( definitions_registry, definition_values, u'bool') self.assertIsNotNone(data_type_definition) self.assertIsInstance(data_type_definition, data_types.BooleanDefinition)
def testGetDefinitions(self): """Tests the GetDefinitions function.""" definitions_registry = registry.DataTypeDefinitionsRegistry() test_definitions = definitions_registry.GetDefinitions() self.assertEqual(len(test_definitions), 0) data_type_definition = data_types.IntegerDefinition( u'int32', aliases=[u'LONG', u'LONG32'], description=u'signed 32-bit integer') definitions_registry.RegisterDefinition(data_type_definition) test_definitions = definitions_registry.GetDefinitions() self.assertEqual(len(test_definitions), 1) definitions_registry.DeregisterDefinition(data_type_definition)
def testReadFloatingPointDataTypeDefinition(self): """Tests the _ReadFloatingPointDataTypeDefinition function.""" definition_values = { u'aliases': [u'float', u'FLOAT'], u'attributes': { u'size': 4, }, u'description': u'32-bit floating-point type', } definitions_registry = registry.DataTypeDefinitionsRegistry() definitions_reader = reader.DataTypeDefinitionsFileReader() data_type_definition = ( definitions_reader._ReadFloatingPointDataTypeDefinition( definitions_registry, definition_values, u'float32')) self.assertIsNotNone(data_type_definition) self.assertIsInstance(data_type_definition, data_types.FloatingPointDefinition)
def testReadFixedSizeDataTypeDefinition(self): """Tests the _ReadFixedSizeDataTypeDefinition function.""" definition_values = { u'aliases': [u'LONG', u'LONG32'], u'attributes': { u'byte_order': u'little-endian', u'size': 4, }, u'description': u'signed 32-bit integer type', } definitions_registry = registry.DataTypeDefinitionsRegistry() definitions_reader = reader.DataTypeDefinitionsFileReader() data_type_definition = definitions_reader._ReadFixedSizeDataTypeDefinition( definitions_registry, definition_values, data_types.IntegerDefinition, u'int32') self.assertIsNotNone(data_type_definition) self.assertIsInstance(data_type_definition, data_types.IntegerDefinition) self.assertEqual(data_type_definition.byte_order, definitions.BYTE_ORDER_LITTLE_ENDIAN) self.assertEqual(data_type_definition.size, 4) # Test with incorrect byte-order. definition_values[u'attributes'][u'byte_order'] = u'bogus' with self.assertRaises(errors.DefinitionReaderError): definitions_reader._ReadFixedSizeDataTypeDefinition( definitions_registry, definition_values, data_types.IntegerDefinition, u'int32') definition_values[u'attributes'][u'byte_order'] = u'little-endian' # Test with incorrect size. definition_values[u'attributes'][u'size'] = u'bogus' with self.assertRaises(errors.DefinitionReaderError): definitions_reader._ReadFixedSizeDataTypeDefinition( definitions_registry, definition_values, data_types.IntegerDefinition, u'int32') definition_values[u'attributes'][u'size'] = 4
def testReadFileObjectConstant(self): """Tests the ReadFileObject function of a constant data type.""" definitions_registry = registry.DataTypeDefinitionsRegistry() definitions_reader = reader.YAMLDataTypeDefinitionsFileReader() definitions_file = self._GetTestFilePath([u'constant.yaml']) with open(definitions_file, 'rb') as file_object: definitions_reader.ReadFileObject(definitions_registry, file_object) self.assertEqual(len(definitions_registry._definitions), 1) data_type_definition = definitions_registry.GetDefinitionByName( u'maximum_number_of_back_traces') self.assertIsInstance(data_type_definition, data_types.ConstantDefinition) self.assertEqual(data_type_definition.name, u'maximum_number_of_back_traces') self.assertEqual(data_type_definition.value, 32)
def testReadFileObjectCharacter(self): """Tests the ReadFileObject function of a character data type.""" definitions_registry = registry.DataTypeDefinitionsRegistry() definitions_reader = reader.YAMLDataTypeDefinitionsFileReader() definitions_file = self._GetTestFilePath([u'character.yaml']) with open(definitions_file, 'rb') as file_object: definitions_reader.ReadFileObject(definitions_registry, file_object) self.assertEqual(len(definitions_registry._definitions), 1) data_type_definition = definitions_registry.GetDefinitionByName( u'char') self.assertIsInstance(data_type_definition, data_types.CharacterDefinition) self.assertEqual(data_type_definition.name, u'char') self.assertEqual(data_type_definition.size, 1) self.assertEqual(data_type_definition.units, u'bytes') byte_size = data_type_definition.GetByteSize() self.assertEqual(byte_size, 1)
def testGetDefinitionByName(self): """Tests the GetDefinitionByName function.""" definitions_registry = registry.DataTypeDefinitionsRegistry() data_type_definition = data_types.IntegerDefinition( u'int32', aliases=[u'LONG', u'LONG32'], description=u'signed 32-bit integer') definitions_registry.RegisterDefinition(data_type_definition) test_definition = definitions_registry.GetDefinitionByName(u'int32') self.assertIsNotNone(test_definition) self.assertIsInstance(test_definition, data_types.IntegerDefinition) test_definition = definitions_registry.GetDefinitionByName(u'LONG32') self.assertIsNotNone(test_definition) self.assertIsInstance(test_definition, data_types.IntegerDefinition) test_definition = definitions_registry.GetDefinitionByName(u'bogus') self.assertIsNone(test_definition) definitions_registry.DeregisterDefinition(data_type_definition)
def testReadFileObjectEnumeration(self): """Tests the ReadFileObject function of an enumeration data type.""" definitions_registry = registry.DataTypeDefinitionsRegistry() definitions_reader = reader.YAMLDataTypeDefinitionsFileReader() definitions_file = self._GetTestFilePath([u'enumeration.yaml']) with open(definitions_file, 'rb') as file_object: definitions_reader.ReadFileObject(definitions_registry, file_object) self.assertEqual(len(definitions_registry._definitions), 1) data_type_definition = definitions_registry.GetDefinitionByName( u'object_information_type') self.assertIsInstance(data_type_definition, data_types.EnumerationDefinition) self.assertEqual(data_type_definition.name, u'object_information_type') self.assertEqual(data_type_definition.size, 4) self.assertEqual(data_type_definition.units, u'bytes') self.assertEqual(len(data_type_definition.values), 6) byte_size = data_type_definition.GetByteSize() self.assertEqual(byte_size, 4)
def __init__(self): """Initializes a dtFabric definitions validator.""" super(DefinitionsValidator, self).__init__() self._structure_definitions_registry = ( registry.DataTypeDefinitionsRegistry())
def testReadEnumerationDataTypeDefinition(self): """Tests the _ReadEnumerationDataTypeDefinition function.""" definition_values = { u'description': u'Minidump object information type', u'values': [ { u'description': u'No object-specific information available', u'name': u'MiniHandleObjectInformationNone', u'value': 0 }, { u'description': u'Thread object information', u'name': u'MiniThreadInformation1', u'value': 1 }, ], } definitions_registry = registry.DataTypeDefinitionsRegistry() definitions_reader = reader.DataTypeDefinitionsFileReader() data_type_definition = ( definitions_reader._ReadEnumerationDataTypeDefinition( definitions_registry, definition_values, u'enum')) self.assertIsNotNone(data_type_definition) self.assertIsInstance(data_type_definition, data_types.EnumerationDefinition) # Test with missing name in enumeration value definition. del definition_values[u'values'][-1][u'name'] with self.assertRaises(errors.DefinitionReaderError): definitions_reader._ReadEnumerationDataTypeDefinition( definitions_registry, definition_values, u'enum') definition_values[u'values'][-1][u'name'] = u'MiniThreadInformation1' # Test with missing value in enumeration value definition. del definition_values[u'values'][-1][u'value'] with self.assertRaises(errors.DefinitionReaderError): definitions_reader._ReadEnumerationDataTypeDefinition( definitions_registry, definition_values, u'enum') definition_values[u'values'][-1][u'value'] = 1 # Test with duplicate enumeration value definition. definition_values[u'values'].append({ u'description': u'Thread object information', u'name': u'MiniThreadInformation1', u'value': 1 }) with self.assertRaises(errors.DefinitionReaderError): definitions_reader._ReadEnumerationDataTypeDefinition( definitions_registry, definition_values, u'enum') del definition_values[u'values'][-1] # Test with missing enumeration value definitions. del definition_values[u'values'] with self.assertRaises(errors.DefinitionReaderError): definitions_reader._ReadEnumerationDataTypeDefinition( definitions_registry, definition_values, u'enum')