示例#1
0
    def isValidForField(self, field):
        """Checks if the specified field can be used
        to parse the current message. It returns a boolean
        that indicates this.

        >>> from netzob.all import *
        >>> f0 = Field(Raw(nbBytes=4))
        >>> f1 = Field(", hello ", name="F1")
        >>> f2 = Field(Raw(nbBytes=(2,5)), name="F2")
        >>> symbol = Symbol([f0, f1, f2], name="Symbol")
        >>> m2 = RawMessage("Toto, hello you", source="server", destination="client")
        >>> m2.isValidForField(symbol)
        True
        >>> m1 = RawMessage("Toto, hello !", source="server", destination="client")
        >>> m1.isValidForField(symbol)
        False

        :parameter field: the current message will be parsed with this field
        :type field: :class:`netzob.Common.Models.Vocabulary.AbstractField.AbstractField`
        :return: a boolean indicating if the current message can be parsed with the specified field
        :rtype: :class:`bool`

        """
        if field is None:
            raise TypeError("Field cannot be None")

        try:
            DataAlignment.align([self.data], field)
        except:
            return False
        return True
示例#2
0
    def isValidForField(self, field):
        """Checks if the specified field can be used
        to parse the current message. It returns a boolean
        that indicates this.

        >>> from netzob.all import *
        >>> f0 = Field(Raw(nbBytes=4))
        >>> f1 = Field(b", hello ", name="F1")
        >>> f2 = Field(Raw(nbBytes=(2,5)), name="F2")
        >>> symbol = Symbol([f0, f1, f2], name="Symbol")
        >>> m2 = RawMessage(b"Toto, hello you", source="server", destination="client")
        >>> m2.isValidForField(symbol)
        True
        >>> m1 = RawMessage(b"Toto, hello !", source="server", destination="client")
        >>> m1.isValidForField(symbol)
        False

        :parameter field: the current message will be parsed with this field
        :type field: :class:`netzob.Model.Vocabulary.AbstractField.AbstractField`
        :return: a boolean indicating if the current message can be parsed with the specified field
        :rtype: :class:`bool`

        """
        if field is None:
            raise TypeError("Field cannot be None")

        try:
            DataAlignment.align([self.data], field)
        except:
            return False
        return True
示例#3
0
    def abstract(data, fields):
        """Search in the fields/symbols the first one that can abstract the data.

        >>> from netzob.all import *
        >>> messages = ["{0}, what's up in {1} ?".format(pseudo, city) for pseudo in ['netzob', 'zoby'] for city in ['Paris', 'Berlin']]

        >>> f1a = Field("netzob")
        >>> f2a = Field(", what's up in ")
        >>> f3a = Field(Alt(["Paris", "Berlin"]))
        >>> f4a = Field(" ?")
        >>> s1 = Symbol([f1a, f2a, f3a, f4a], name="Symbol-netzob")

        >>> f1b = Field("zoby")
        >>> f2b = Field(", what's up in ")
        >>> f3b = Field(Alt(["Paris", "Berlin"]))
        >>> f4b = Field(" ?")
        >>> s2 = Symbol([f1b, f2b, f3b, f4b], name="Symbol-zoby")

        >>> for m in messages:
        ...    abstractedSymbol = AbstractField.abstract(m, [s1, s2])
        ...    print(abstractedSymbol.name)
        Symbol-netzob
        Symbol-netzob
        Symbol-zoby
        Symbol-zoby

        :parameter data: the data that should be abstracted in symbol
        :type data: :class:`str`
        :parameter fields: a list of fields/symbols targeted during the abstraction process
        :type fields: :class:`list` of :class:`netzob.Model.Vocabulary.AbstractField`

        :return: a field/symbol
        :rtype: :class:`netzob.Model.Vocabulary.AbstractField`
        :raises: :class:`netzob.Model.Vocabulary.AbstractField.AbstractionException` if an error occurs while abstracting the data
        """
        from netzob.Common.Utils.DataAlignment.DataAlignment import DataAlignment
        for field in fields:
            try:
                DataAlignment.align([data], field, encoded=False)
                return field
            except:
                pass

        from netzob.Model.Vocabulary.UnknownSymbol import UnknownSymbol
        from netzob.Model.Vocabulary.Messages.RawMessage import RawMessage
        unknown_symbol = UnknownSymbol(RawMessage(data))
        logging.error(
            "Impossible to abstract the message in one of the specified symbols, we create an unknown symbol for it: '%s'",
            unknown_symbol)

        return unknown_symbol
示例#4
0
    def abstract(data, fields):
        """Search in the fields/symbols the first one that can abstract the data.

        >>> from netzob.all import *
        >>> messages = ["{0}, what's up in {1} ?".format(pseudo, city) for pseudo in ['netzob', 'zoby'] for city in ['Paris', 'Berlin']]

        >>> f1a = Field("netzob")
        >>> f2a = Field(", what's up in ")
        >>> f3a = Field(Alt(["Paris", "Berlin"]))
        >>> f4a = Field(" ?")
        >>> s1 = Symbol([f1a, f2a, f3a, f4a], name="Symbol-netzob")

        >>> f1b = Field("zoby")
        >>> f2b = Field(", what's up in ")
        >>> f3b = Field(Alt(["Paris", "Berlin"]))
        >>> f4b = Field(" ?")
        >>> s2 = Symbol([f1b, f2b, f3b, f4b], name="Symbol-zoby")

        >>> for m in messages:
        ...    abstractedSymbol = AbstractField.abstract(m, [s1, s2])
        ...    print abstractedSymbol.name
        Symbol-netzob
        Symbol-netzob
        Symbol-zoby
        Symbol-zoby

        :parameter data: the data that should be abstracted in symbol
        :type data: :class:`str`
        :parameter fields: a list of fields/symbols targeted during the abstraction process
        :type fields: :class:`list` of :class:`netzob.Common.Models.Vocabulary.AbstractField`

        :return: a field/symbol
        :rtype: :class:`netzob.Common.Models.Vocabulary.AbstractField`
        :raises: :class:`netzob.Common.Models.Vocabulary.AbstractField.AbstractionException` if an error occurs while abstracting the data
        """
        from netzob.Common.Utils.DataAlignment.DataAlignment import DataAlignment
        for field in fields:
            try:                
                DataAlignment.align([data], field, encoded=False)
                return field
            except:
                pass

        logging.error("Impossible to abstract the message in one of the specified symbols, we create an unknown symbol for it.")
        
        from netzob.Common.Models.Vocabulary.UnknownSymbol import UnknownSymbol
        from netzob.Common.Models.Vocabulary.Messages.RawMessage import RawMessage
        return UnknownSymbol(RawMessage(data))
def _executeDataAlignment(arg, **kwargs):
    """Wrapper used to parallelize the DataAlignment using
    a pool of threads.
    """
    data = arg[0]
    field = arg[1]
    alignedData = DataAlignment.align([data], field)
    return (data, alignedData)
示例#6
0
def _executeDataAlignment(arg, **kwargs):
    """Wrapper used to parallelize the DataAlignment using
    a pool of threads.
    """
    data = arg[0]
    field = arg[1]
    alignedData = DataAlignment.align([data], field)
    return (data, alignedData)
示例#7
0
    def abstract(data, fields):
        """Search in the fields/symbols the first one that can abstract the data.

        >>> from netzob.all import *
        >>> messages = ["{0}, what's up in {1} ?".format(pseudo, city) for pseudo in ['netzob', 'zoby'] for city in ['Paris', 'Berlin']]

        >>> f1a = Field(name="name", domain="netzob")
        >>> f2a = Field(name="question", domain=", what's up in ")
        >>> f3a = Field(name="city", domain=Alt(["Paris", "Berlin"]))
        >>> f4a = Field(name="mark", domain=" ?")
        >>> s1 = Symbol([f1a, f2a, f3a, f4a], name="Symbol-netzob")

        >>> f1b = Field(name="name", domain="zoby")
        >>> f2b = Field(name="question", domain=", what's up in ")
        >>> f3b = Field(name="city", domain=Alt(["Paris", "Berlin"]))
        >>> f4b = Field(name="mark", domain=" ?")
        >>> s2 = Symbol([f1b, f2b, f3b, f4b], name="Symbol-zoby")

        >>> for m in messages:
        ...    (abstractedSymbol, structured_data) = AbstractField.abstract(m, [s1, s2])
        ...    print(structured_data)
        ...    print(abstractedSymbol.name)
        OrderedDict([('name', b'netzob'), ('question', b", what's up in "), ('city', b'Paris'), ('mark', b' ?')])
        Symbol-netzob
        OrderedDict([('name', b'netzob'), ('question', b", what's up in "), ('city', b'Berlin'), ('mark', b' ?')])
        Symbol-netzob
        OrderedDict([('name', b'zoby'), ('question', b", what's up in "), ('city', b'Paris'), ('mark', b' ?')])
        Symbol-zoby
        OrderedDict([('name', b'zoby'), ('question', b", what's up in "), ('city', b'Berlin'), ('mark', b' ?')])
        Symbol-zoby

        :parameter data: the data that should be abstracted in symbol
        :type data: :class:`str`
        :parameter fields: a list of fields/symbols targeted during the abstraction process
        :type fields: :class:`list` of :class:`netzob.Model.Vocabulary.AbstractField`

        :return: a field/symbol and the structured received message
        :rtype: a tuple (:class:`netzob.Model.Vocabulary.AbstractField`, dict)
        :raises: :class:`netzob.Model.Vocabulary.AbstractField.AbstractionException` if an error occurs while abstracting the data
        """
        from netzob.Common.Utils.DataAlignment.DataAlignment import DataAlignment
        for field in fields:
            try:
                # Try to align/parse the data with the current field
                alignedData = DataAlignment.align([data], field, encoded=False)

                # If it matches, we build a dict that contains, for each field, the associated value that was present in the message
                structured_data = OrderedDict()
                for fields_value in alignedData:
                    for i, field_value in enumerate(fields_value):
                        structured_data[alignedData.headers[i]] = field_value
                return (field, structured_data)
            except:
                pass

        from netzob.Model.Vocabulary.UnknownSymbol import UnknownSymbol
        from netzob.Model.Vocabulary.Messages.RawMessage import RawMessage
        unknown_symbol = UnknownSymbol(RawMessage(data))
        structured_data = OrderedDict()
        logging.error(
            "Impossible to abstract the message in one of the specified symbols, we create an unknown symbol for it: '%s'",
            unknown_symbol)

        return (unknown_symbol, structured_data)
示例#8
0
    def getCells(self, encoded=True, styled=True, transposed=False):
        """Returns a matrix with a different line for each messages attached to the symbol of the current element.

        The matrix includes a different column for each leaf children of the current element.
        In each cell, the slices of messages once aligned.
        Attached :class:`EncodingFunction` can also be considered if parameter encoded is set to True.
        In addition, visualizationFunctions are also applied if parameter styled is set to True.
        If parameter Transposed is set to True, the matrix is built with rows for fields and columns for messages.

        >>> from netzob.all import *
        >>> messages = [RawMessage("hello {0}, what's up in {1} ?".format(pseudo, city)) for pseudo in ['netzob', 'zoby', 'lapy'] for city in ['Paris', 'Berlin', 'New-York']]
        >>> fh1 = Field(ASCII("hello "), name="hello")
        >>> fh2 = Field(Alt([ASCII("netzob"), ASCII("zoby"), ASCII("lapy"), ASCII("sygus")]), name="pseudo")
        >>> fheader = Field(name="header")
        >>> fheader.fields = [fh1, fh2]
        >>> fb1 = Field(ASCII(", what's up in "), name="whatsup")
        >>> fb2 = Field(["Paris", "Berlin", "New-York"], name="city")
        >>> fb3 = Field(" ?", name="end")
        >>> fbody = Field(name="body")
        >>> fbody.fields = [fb1, fb2, fb3]
        >>> symbol = Symbol([fheader, fbody], messages=messages)

        >>> print(symbol)
        hello    | pseudo   | whatsup           | city       | end 
        -------- | -------- | ----------------- | ---------- | ----
        'hello ' | 'netzob' | ", what's up in " | 'Paris'    | ' ?'
        'hello ' | 'netzob' | ", what's up in " | 'Berlin'   | ' ?'
        'hello ' | 'netzob' | ", what's up in " | 'New-York' | ' ?'
        'hello ' | 'zoby'   | ", what's up in " | 'Paris'    | ' ?'
        'hello ' | 'zoby'   | ", what's up in " | 'Berlin'   | ' ?'
        'hello ' | 'zoby'   | ", what's up in " | 'New-York' | ' ?'
        'hello ' | 'lapy'   | ", what's up in " | 'Paris'    | ' ?'
        'hello ' | 'lapy'   | ", what's up in " | 'Berlin'   | ' ?'
        'hello ' | 'lapy'   | ", what's up in " | 'New-York' | ' ?'
        -------- | -------- | ----------------- | ---------- | ----

        >>> fh1.addEncodingFunction(TypeEncodingFunction(HexaString))
        >>> fb2.addEncodingFunction(TypeEncodingFunction(HexaString))
        >>> print(symbol)
        hello          | pseudo   | whatsup           | city               | end 
        -------------- | -------- | ----------------- | ------------------ | ----
        '68656c6c6f20' | 'netzob' | ", what's up in " | '5061726973'       | ' ?'
        '68656c6c6f20' | 'netzob' | ", what's up in " | '4265726c696e'     | ' ?'
        '68656c6c6f20' | 'netzob' | ", what's up in " | '4e65772d596f726b' | ' ?'
        '68656c6c6f20' | 'zoby'   | ", what's up in " | '5061726973'       | ' ?'
        '68656c6c6f20' | 'zoby'   | ", what's up in " | '4265726c696e'     | ' ?'
        '68656c6c6f20' | 'zoby'   | ", what's up in " | '4e65772d596f726b' | ' ?'
        '68656c6c6f20' | 'lapy'   | ", what's up in " | '5061726973'       | ' ?'
        '68656c6c6f20' | 'lapy'   | ", what's up in " | '4265726c696e'     | ' ?'
        '68656c6c6f20' | 'lapy'   | ", what's up in " | '4e65772d596f726b' | ' ?'
        -------------- | -------- | ----------------- | ------------------ | ----

        >>> print(fheader.getCells()) 
        Field          | Field   
        -------------- | --------
        '68656c6c6f20' | 'netzob'
        '68656c6c6f20' | 'netzob'
        '68656c6c6f20' | 'netzob'
        '68656c6c6f20' | 'zoby'  
        '68656c6c6f20' | 'zoby'  
        '68656c6c6f20' | 'zoby'  
        '68656c6c6f20' | 'lapy'  
        '68656c6c6f20' | 'lapy'  
        '68656c6c6f20' | 'lapy'  
        -------------- | --------

        >>> print(fh1.getCells())
        Field         
        --------------
        '68656c6c6f20'
        '68656c6c6f20'
        '68656c6c6f20'
        '68656c6c6f20'
        '68656c6c6f20'
        '68656c6c6f20'
        '68656c6c6f20'
        '68656c6c6f20'
        '68656c6c6f20'
        --------------

        >>> print(fh2.getCells())
        Field   
        --------
        'netzob'
        'netzob'
        'netzob'
        'zoby'  
        'zoby'  
        'zoby'  
        'lapy'  
        'lapy'  
        'lapy'  
        --------

        >>> print(fbody.getCells())
        Field             | Field              | Field
        ----------------- | ------------------ | -----
        ", what's up in " | '5061726973'       | ' ?' 
        ", what's up in " | '4265726c696e'     | ' ?' 
        ", what's up in " | '4e65772d596f726b' | ' ?' 
        ", what's up in " | '5061726973'       | ' ?' 
        ", what's up in " | '4265726c696e'     | ' ?' 
        ", what's up in " | '4e65772d596f726b' | ' ?' 
        ", what's up in " | '5061726973'       | ' ?' 
        ", what's up in " | '4265726c696e'     | ' ?' 
        ", what's up in " | '4e65772d596f726b' | ' ?' 
        ----------------- | ------------------ | -----

        >>> print(fb1.getCells())
        Field            
        -----------------
        ", what's up in "
        ", what's up in "
        ", what's up in "
        ", what's up in "
        ", what's up in "
        ", what's up in "
        ", what's up in "
        ", what's up in "
        ", what's up in "
        -----------------

        >>> print(fb2.getCells())
        Field             
        ------------------
        '5061726973'      
        '4265726c696e'    
        '4e65772d596f726b'
        '5061726973'      
        '4265726c696e'    
        '4e65772d596f726b'
        '5061726973'      
        '4265726c696e'    
        '4e65772d596f726b'
        ------------------

        >>> print(fb3.getCells())
        Field
        -----
        ' ?' 
        ' ?' 
        ' ?' 
        ' ?' 
        ' ?' 
        ' ?' 
        ' ?' 
        ' ?' 
        ' ?' 
        -----

        :keyword encoded: if set to True, encoding functions are applied on returned cells
        :type encoded: :class:`bool`
        :keyword styled: if set to True, visualization functions are applied on returned cells
        :type styled: :class:`bool`
        :keyword transposed: is set to True, the returned matrix is transposed (1 line for each field)
        :type transposed: :class:`bool`

        :return: a matrix representing the aligned messages following fields definitions.
        :rtype: a :class:`netzob.Common.Utils.MatrixList.MatrixList`
        :raises: :class:`netzob.Model.Vocabulary.AbstractField.AlignmentException` if an error occurs while aligning messages
        """

        if len(self.messages) < 1:
            raise ValueError("This symbol does not contain any message.")

        # Fetch all the data to align
        data = [message.data for message in self.messages]

        # [DEBUG] set to false for debug only. A sequential alignment is more simple to debug
        useParallelAlignment = False

        if useParallelAlignment:
            # Execute a parallel alignment
            from netzob.Common.Utils.DataAlignment.ParallelDataAlignment import ParallelDataAlignment
            return ParallelDataAlignment.align(data, self, encoded=encoded)
        else:
            # Execute a sequential alignment
            from netzob.Common.Utils.DataAlignment.DataAlignment import DataAlignment
            return DataAlignment.align(data, self, encoded=encoded)
示例#9
0
    def cluster(self, field, keyField):
        """Create and return new symbols according to a specific key
        field.

        >>> import binascii
        >>> from netzob.all import *
        >>> samples = [b"00ff2f000000", b"000020000000", b"00ff2f000000"]
        >>> messages = [RawMessage(data=binascii.unhexlify(sample)) for sample in samples]
        >>> f1 = Field(Raw(nbBytes=1))
        >>> f2 = Field(Raw(nbBytes=2))
        >>> f3 = Field(Raw(nbBytes=3))
        >>> symbol = Symbol([f1, f2, f3], messages=messages)
        >>> symbol.addEncodingFunction(TypeEncodingFunction(HexaString))
        >>> newSymbols = Format.clusterByKeyField(symbol, f2)
        >>> for sym in list(newSymbols.values()):
        ...     sym.addEncodingFunction(TypeEncodingFunction(HexaString))
        ...     print(sym.name + ":")
        ...     print(sym)
        Symbol_ff2f:
        Field | Field  | Field   
        ----- | ------ | --------
        '00'  | 'ff2f' | '000000'
        '00'  | 'ff2f' | '000000'
        ----- | ------ | --------
        Symbol_0020:
        Field | Field  | Field   
        ----- | ------ | --------
        '00'  | '0020' | '000000'
        ----- | ------ | --------


        :param field: the field we want to split in new symbols
        :type field: :class:`netzob.Model.Vocabulary.AbstractField.AbstractField`
        :param keyField: the field used as a key during the splitting operation
        :type field: :class:`netzob.Model.Vocabulary.AbstractField.AbstractField`
        :raise Exception if something bad happens
        """

        # Safe checks
        if field is None:
            raise TypeError("'field' should not be None")
        if keyField is None:
            raise TypeError("'keyField' should not be None")
        if keyField not in field.fields:
            raise TypeError("'keyField' is not a child of 'field'")

        newSymbols = collections.OrderedDict()

        keyFieldMessageValues = keyField.getMessageValues(encoded=False,
                                                          styled=False)
        newSymbolsSplittedMessages = {}

        # we identify what would be the best type of the key field
        keyFieldType = ASCII
        for message, keyFieldValue in list(keyFieldMessageValues.items()):
            # If the value cannot be parsed as ASCII, we convert it to HexaString
            if not ASCII().canParse(
                    TypeConverter.convert(keyFieldValue, Raw, BitArray)):
                keyFieldType = HexaString
                break

            # Even if the value is theoritically parsable as ASCII, some caracters cannot be encoded, so we double check
            tmp_value = TypeConverter.convert(keyFieldValue, Raw, ASCII)
            tmp2_value = TypeConverter.convert(tmp_value, ASCII, Raw)
            if keyFieldValue != tmp2_value:
                # This means we cannot retrieve the original value by encoding and then decoding in ASCII
                keyFieldType = HexaString
                break

        # we create a symbol for each of these uniq values
        for message, keyFieldValue in list(keyFieldMessageValues.items()):
            keyFieldValue = TypeConverter.convert(keyFieldValue, Raw,
                                                  keyFieldType)
            if keyFieldValue not in list(newSymbols.keys()):
                if type(keyFieldValue) is str:
                    symbolName = "Symbol_{0}".format(keyFieldValue)
                else:
                    symbolName = "Symbol_{0}".format(
                        keyFieldValue.decode("utf-8"))
                newSymbols[keyFieldValue] = Symbol(name=symbolName,
                                                   messages=[message])
                splittedMessages = DataAlignment.align([message.data],
                                                       field,
                                                       encoded=False)
                newSymbolsSplittedMessages[keyFieldValue] = [
                    splittedMessages[0]
                ]
            else:
                newSymbols[keyFieldValue].messages.append(message)
                splittedMessages = DataAlignment.align([message.data],
                                                       field,
                                                       encoded=False)
                newSymbolsSplittedMessages[keyFieldValue].append(
                    splittedMessages[0])

        for newSymbolKeyValue, newSymbol in list(newSymbols.items()):
            # we recreate the same fields in this new symbol as the fields that exist in the original symbol
            newSymbol.clearFields()
            for i, f in enumerate(field.fields):
                if f == keyField:
                    newFieldDomain = TypeConverter.convert(
                        newSymbolKeyValue, keyFieldType, Raw)
                else:
                    newFieldDomain = set()
                    for j in range(
                            len(newSymbolsSplittedMessages[newSymbolKeyValue])
                    ):
                        newFieldDomain.add(
                            newSymbolsSplittedMessages[newSymbolKeyValue][j]
                            [i])
                    newFieldDomain = list(newFieldDomain)
                newF = Field(name=f.name, domain=newFieldDomain)
                newF.parent = newSymbol
                newSymbol.fields.append(newF)

            # we remove endless fields that accepts no values
            cells = newSymbol.getCells(encoded=False,
                                       styled=False,
                                       transposed=False)
            max_i_cell_with_value = 0
            for line in cells:
                for i_cell, cell in enumerate(line):
                    if cell != '' and max_i_cell_with_value < i_cell:
                        max_i_cell_with_value = i_cell
            newSymbol.clearFields()
            for i, f in enumerate(field.fields[:max_i_cell_with_value + 1]):
                if f == keyField:
                    newFieldDomain = TypeConverter.convert(
                        newSymbolKeyValue, keyFieldType, Raw)
                else:
                    newFieldDomain = set()
                    for j in range(
                            len(newSymbolsSplittedMessages[newSymbolKeyValue])
                    ):
                        newFieldDomain.add(
                            newSymbolsSplittedMessages[newSymbolKeyValue][j]
                            [i])
                    newFieldDomain = list(newFieldDomain)
                newF = Field(name=f.name, domain=newFieldDomain)
                newF.parent = newSymbol
                newSymbol.fields.append(newF)

        return newSymbols
示例#10
0
    def cluster(self, field, keyField):
        """Create and return new symbols according to a specific key
        field.

        >>> import binascii
        >>> from netzob.all import *
        >>> samples = ["00ff2f000000",	"000020000000",	"00ff2f000000"]
        >>> messages = [RawMessage(data=binascii.unhexlify(sample)) for sample in samples]
        >>> f1 = Field(Raw(nbBytes=1))
        >>> f2 = Field(Raw(nbBytes=2))
        >>> f3 = Field(Raw(nbBytes=3))
        >>> symbol = Symbol([f1, f2, f3], messages=messages)
        >>> symbol.addEncodingFunction(TypeEncodingFunction(HexaString))
        >>> newSymbols = Format.clusterByKeyField(symbol, f2)
        >>> for sym in newSymbols.values():
        ...     sym.addEncodingFunction(TypeEncodingFunction(HexaString))
        ...     print sym.name + ":"
        ...     print sym
        Symbol_ff2f:
        Field | Field  | Field   
        ----- | ------ | --------
        '00'  | 'ff2f' | '000000'
        '00'  | 'ff2f' | '000000'
        ----- | ------ | --------
        Symbol_0020:
        Field | Field  | Field   
        ----- | ------ | --------
        '00'  | '0020' | '000000'
        ----- | ------ | --------

        :param field: the field we want to split in new symbols
        :type field: :class:`netzob.Common.Models.Vocabulary.AbstractField.AbstractField`
        :param keyField: the field used as a key during the splitting operation
        :type field: :class:`netzob.Common.Models.Vocabulary.AbstractField.AbstractField`
        :raise Exception if something bad happens
        """

        # Safe checks
        if field is None:
            raise TypeError("'field' should not be None")
        if keyField is None:
            raise TypeError("'keyField' should not be None")
        if keyField not in field.fields:
            raise TypeError("'keyField' is not a child of 'field'")

        newSymbols = {}

        keyFieldMessageValues = keyField.getMessageValues(encoded=False, styled=False)
        newSymbolsSplittedMessages = {}

        # we identify what would be the best type of the key field
        keyFieldType = ASCII
        for message, keyFieldValue in keyFieldMessageValues.iteritems():
            # If the value cannot be parsed as ASCII, we convert it to HexaString
            if not ASCII().canParse(TypeConverter.convert(keyFieldValue, Raw, BitArray)):
                keyFieldType = HexaString
                break

            # Even if the value is theoritically parsable as ASCII, some caracters cannot be encoded, so we double check
            tmp_value = TypeConverter.convert(keyFieldValue, Raw, ASCII)
            tmp2_value = TypeConverter.convert(tmp_value, ASCII, Raw)
            if keyFieldValue != tmp2_value:
                # This means we cannot retrieve the original value by encoding and then decoding in ASCII
                keyFieldType = HexaString
                break

        # we create a symbol for each of these uniq values
        for message, keyFieldValue in keyFieldMessageValues.iteritems():
            keyFieldValue = TypeConverter.convert(keyFieldValue, Raw, keyFieldType)
            if keyFieldValue not in newSymbols.keys():
                symbolName = "Symbol_{0}".format(keyFieldValue)
                newSymbols[keyFieldValue] = Symbol(name=symbolName, messages=[message])
                splittedMessages = DataAlignment.align([message.data], field, encoded=False)
                newSymbolsSplittedMessages[keyFieldValue] = [splittedMessages[0]]
            else:
                newSymbols[keyFieldValue].messages.append(message)
                splittedMessages = DataAlignment.align([message.data], field, encoded=False)
                newSymbolsSplittedMessages[keyFieldValue].append(splittedMessages[0])

        for newSymbolKeyValue, newSymbol in newSymbols.iteritems():
            # we recreate the same fields in this new symbol as the fields that exist in the original symbol
            newSymbol.clearFields()
            for i, f in enumerate(field.fields):
                if f == keyField:
                    newFieldDomain = TypeConverter.convert(newSymbolKeyValue, keyFieldType, Raw)
                else:
                    newFieldDomain = set()
                    for j in range(len(newSymbolsSplittedMessages[newSymbolKeyValue])):
                        newFieldDomain.add(newSymbolsSplittedMessages[newSymbolKeyValue][j][i])
                    newFieldDomain = list(newFieldDomain)
                newF = Field(name=f.name, domain=newFieldDomain)
                newF.parent = newSymbol
                newSymbol.fields.append(newF)

            # we remove endless fields that accepts no values
            cells = newSymbol.getCells(encoded=False, styled=False, transposed=False)
            max_i_cell_with_value = 0
            for line in cells:
                for i_cell, cell in enumerate(line):
                    if cell != '' and max_i_cell_with_value < i_cell:
                        max_i_cell_with_value = i_cell
            newSymbol.clearFields()
            for i, f in enumerate(field.fields[:max_i_cell_with_value + 1]):
                if f == keyField:
                    newFieldDomain = TypeConverter.convert(newSymbolKeyValue, keyFieldType, Raw)
                else:
                    newFieldDomain = set()
                    for j in range(len(newSymbolsSplittedMessages[newSymbolKeyValue])):
                        newFieldDomain.add(newSymbolsSplittedMessages[newSymbolKeyValue][j][i])
                    newFieldDomain = list(newFieldDomain)
                newF = Field(name=f.name, domain=newFieldDomain)
                newF.parent = newSymbol
                newSymbol.fields.append(newF)

        return newSymbols
示例#11
0
    def abstract(data, fields):
        """Search in the fields/symbols the first one that can abstract the data.

        >>> from netzob.all import *
        >>> messages = ["{0}, what's up in {1} ?".format(pseudo, city) for pseudo in ['netzob', 'zoby'] for city in ['Paris', 'Berlin']]

        >>> f1a = Field(name="name", domain="netzob")
        >>> f2a = Field(name="question", domain=", what's up in ")
        >>> f3a = Field(name="city", domain=Alt(["Paris", "Berlin"]))
        >>> f4a = Field(name="mark", domain=" ?")
        >>> s1 = Symbol([f1a, f2a, f3a, f4a], name="Symbol-netzob")

        >>> f1b = Field(name="name", domain="zoby")
        >>> f2b = Field(name="question", domain=", what's up in ")
        >>> f3b = Field(name="city", domain=Alt(["Paris", "Berlin"]))
        >>> f4b = Field(name="mark", domain=" ?")
        >>> s2 = Symbol([f1b, f2b, f3b, f4b], name="Symbol-zoby")

        >>> for m in messages:
        ...    (abstractedSymbol, structured_data) = AbstractField.abstract(m, [s1, s2])
        ...    print(structured_data)
        ...    print(abstractedSymbol.name)
        OrderedDict([('name', b'netzob'), ('question', b", what's up in "), ('city', b'Paris'), ('mark', b' ?')])
        Symbol-netzob
        OrderedDict([('name', b'netzob'), ('question', b", what's up in "), ('city', b'Berlin'), ('mark', b' ?')])
        Symbol-netzob
        OrderedDict([('name', b'zoby'), ('question', b", what's up in "), ('city', b'Paris'), ('mark', b' ?')])
        Symbol-zoby
        OrderedDict([('name', b'zoby'), ('question', b", what's up in "), ('city', b'Berlin'), ('mark', b' ?')])
        Symbol-zoby

        :parameter data: the data that should be abstracted in symbol
        :type data: :class:`str`
        :parameter fields: a list of fields/symbols targeted during the abstraction process
        :type fields: :class:`list` of :class:`netzob.Model.Vocabulary.AbstractField`

        :return: a field/symbol and the structured received message
        :rtype: a tuple (:class:`netzob.Model.Vocabulary.AbstractField`, dict)
        :raises: :class:`netzob.Model.Vocabulary.AbstractField.AbstractionException` if an error occurs while abstracting the data
        """
        from netzob.Common.Utils.DataAlignment.DataAlignment import DataAlignment
        for field in fields:
            try:
                # Try to align/parse the data with the current field
                alignedData = DataAlignment.align([data], field, encoded=False)

                # If it matches, we build a dict that contains, for each field, the associated value that was present in the message
                structured_data = OrderedDict()
                for fields_value in alignedData:
                    for i, field_value in enumerate(fields_value):
                        structured_data[alignedData.headers[i]] = field_value
                return (field, structured_data)
            except:
                pass

        from netzob.Model.Vocabulary.UnknownSymbol import UnknownSymbol
        from netzob.Model.Vocabulary.Messages.RawMessage import RawMessage
        unknown_symbol = UnknownSymbol(RawMessage(data))
        structured_data = OrderedDict()
        logging.error("Impossible to abstract the message in one of the specified symbols, we create an unknown symbol for it: '%s'", unknown_symbol)

        return (unknown_symbol, structured_data)
示例#12
0
    def getCells(self, encoded=True, styled=True, transposed=False):
        """Returns a matrix with a different line for each messages attached to the symbol of the current element.

        The matrix includes a different column for each leaf children of the current element.
        In each cell, the slices of messages once aligned.
        Attached :class:`EncodingFunction` can also be considered if parameter encoded is set to True.
        In addition, visualizationFunctions are also applied if parameter styled is set to True.
        If parameter Transposed is set to True, the matrix is built with rows for fields and columns for messages.

        >>> from netzob.all import *
        >>> messages = [RawMessage("hello {0}, what's up in {1} ?".format(pseudo, city)) for pseudo in ['netzob', 'zoby', 'lapy'] for city in ['Paris', 'Berlin', 'New-York']]
        >>> fh1 = Field(ASCII("hello "), name="hello")
        >>> fh2 = Field(Alt([ASCII("netzob"), ASCII("zoby"), ASCII("lapy"), ASCII("sygus")]), name="pseudo")
        >>> fheader = Field(name="header")
        >>> fheader.fields = [fh1, fh2]
        >>> fb1 = Field(ASCII(", what's up in "), name="whatsup")
        >>> fb2 = Field(["Paris", "Berlin", "New-York"], name="city")
        >>> fb3 = Field(" ?", name="end")
        >>> fbody = Field(name="body")
        >>> fbody.fields = [fb1, fb2, fb3]
        >>> symbol = Symbol([fheader, fbody], messages=messages)

        >>> print(symbol)
        hello    | pseudo   | whatsup           | city       | end 
        -------- | -------- | ----------------- | ---------- | ----
        'hello ' | 'netzob' | ", what's up in " | 'Paris'    | ' ?'
        'hello ' | 'netzob' | ", what's up in " | 'Berlin'   | ' ?'
        'hello ' | 'netzob' | ", what's up in " | 'New-York' | ' ?'
        'hello ' | 'zoby'   | ", what's up in " | 'Paris'    | ' ?'
        'hello ' | 'zoby'   | ", what's up in " | 'Berlin'   | ' ?'
        'hello ' | 'zoby'   | ", what's up in " | 'New-York' | ' ?'
        'hello ' | 'lapy'   | ", what's up in " | 'Paris'    | ' ?'
        'hello ' | 'lapy'   | ", what's up in " | 'Berlin'   | ' ?'
        'hello ' | 'lapy'   | ", what's up in " | 'New-York' | ' ?'
        -------- | -------- | ----------------- | ---------- | ----

        >>> fh1.addEncodingFunction(TypeEncodingFunction(HexaString))
        >>> fb2.addEncodingFunction(TypeEncodingFunction(HexaString))
        >>> print(symbol)
        hello          | pseudo   | whatsup           | city               | end 
        -------------- | -------- | ----------------- | ------------------ | ----
        '68656c6c6f20' | 'netzob' | ", what's up in " | '5061726973'       | ' ?'
        '68656c6c6f20' | 'netzob' | ", what's up in " | '4265726c696e'     | ' ?'
        '68656c6c6f20' | 'netzob' | ", what's up in " | '4e65772d596f726b' | ' ?'
        '68656c6c6f20' | 'zoby'   | ", what's up in " | '5061726973'       | ' ?'
        '68656c6c6f20' | 'zoby'   | ", what's up in " | '4265726c696e'     | ' ?'
        '68656c6c6f20' | 'zoby'   | ", what's up in " | '4e65772d596f726b' | ' ?'
        '68656c6c6f20' | 'lapy'   | ", what's up in " | '5061726973'       | ' ?'
        '68656c6c6f20' | 'lapy'   | ", what's up in " | '4265726c696e'     | ' ?'
        '68656c6c6f20' | 'lapy'   | ", what's up in " | '4e65772d596f726b' | ' ?'
        -------------- | -------- | ----------------- | ------------------ | ----

        >>> print(fheader.getCells()) 
        Field          | Field   
        -------------- | --------
        '68656c6c6f20' | 'netzob'
        '68656c6c6f20' | 'netzob'
        '68656c6c6f20' | 'netzob'
        '68656c6c6f20' | 'zoby'  
        '68656c6c6f20' | 'zoby'  
        '68656c6c6f20' | 'zoby'  
        '68656c6c6f20' | 'lapy'  
        '68656c6c6f20' | 'lapy'  
        '68656c6c6f20' | 'lapy'  
        -------------- | --------

        >>> print(fh1.getCells())
        Field         
        --------------
        '68656c6c6f20'
        '68656c6c6f20'
        '68656c6c6f20'
        '68656c6c6f20'
        '68656c6c6f20'
        '68656c6c6f20'
        '68656c6c6f20'
        '68656c6c6f20'
        '68656c6c6f20'
        --------------

        >>> print(fh2.getCells())
        Field   
        --------
        'netzob'
        'netzob'
        'netzob'
        'zoby'  
        'zoby'  
        'zoby'  
        'lapy'  
        'lapy'  
        'lapy'  
        --------

        >>> print(fbody.getCells())
        Field             | Field              | Field
        ----------------- | ------------------ | -----
        ", what's up in " | '5061726973'       | ' ?' 
        ", what's up in " | '4265726c696e'     | ' ?' 
        ", what's up in " | '4e65772d596f726b' | ' ?' 
        ", what's up in " | '5061726973'       | ' ?' 
        ", what's up in " | '4265726c696e'     | ' ?' 
        ", what's up in " | '4e65772d596f726b' | ' ?' 
        ", what's up in " | '5061726973'       | ' ?' 
        ", what's up in " | '4265726c696e'     | ' ?' 
        ", what's up in " | '4e65772d596f726b' | ' ?' 
        ----------------- | ------------------ | -----

        >>> print(fb1.getCells())
        Field            
        -----------------
        ", what's up in "
        ", what's up in "
        ", what's up in "
        ", what's up in "
        ", what's up in "
        ", what's up in "
        ", what's up in "
        ", what's up in "
        ", what's up in "
        -----------------

        >>> print(fb2.getCells())
        Field             
        ------------------
        '5061726973'      
        '4265726c696e'    
        '4e65772d596f726b'
        '5061726973'      
        '4265726c696e'    
        '4e65772d596f726b'
        '5061726973'      
        '4265726c696e'    
        '4e65772d596f726b'
        ------------------

        >>> print(fb3.getCells())
        Field
        -----
        ' ?' 
        ' ?' 
        ' ?' 
        ' ?' 
        ' ?' 
        ' ?' 
        ' ?' 
        ' ?' 
        ' ?' 
        -----

        :keyword encoded: if set to True, encoding functions are applied on returned cells
        :type encoded: :class:`bool`
        :keyword styled: if set to True, visualization functions are applied on returned cells
        :type styled: :class:`bool`
        :keyword transposed: is set to True, the returned matrix is transposed (1 line for each field)
        :type transposed: :class:`bool`

        :return: a matrix representing the aligned messages following fields definitions.
        :rtype: a :class:`netzob.Common.Utils.MatrixList.MatrixList`
        :raises: :class:`netzob.Model.Vocabulary.AbstractField.AlignmentException` if an error occurs while aligning messages
        """

        if len(self.messages) < 1:
            raise ValueError("This symbol does not contain any message.")

        # Fetch all the data to align
        data = [message.data for message in self.messages]

        # [DEBUG] set to false for debug only. A sequential alignment is more simple to debug
        useParallelAlignment = False

        if useParallelAlignment:
            # Execute a parallel alignment
            from netzob.Common.Utils.DataAlignment.ParallelDataAlignment import ParallelDataAlignment
            return ParallelDataAlignment.align(data, self, encoded=encoded)
        else:
            # Execute a sequential alignment
            from netzob.Common.Utils.DataAlignment.DataAlignment import DataAlignment
            return DataAlignment.align(data, self, encoded=encoded)