def isValidForField(self, field):
"""Checks if the specified field can be used
to parse the current message. It returns a boolean
that indicates this.
>>> from netzob.all import *
>>> f0 = Field(Raw(nbBytes=4))
>>> f1 = Field(", hello ", name="F1")
>>> f2 = Field(Raw(nbBytes=(2,5)), name="F2")
>>> symbol = Symbol([f0, f1, f2], name="Symbol")
>>> m2 = RawMessage("Toto, hello you", source="server", destination="client")
>>> m2.isValidForField(symbol)
True
>>> m1 = RawMessage("Toto, hello !", source="server", destination="client")
>>> m1.isValidForField(symbol)
False
:parameter field: the current message will be parsed with this field
:type field: :class:`netzob.Common.Models.Vocabulary.AbstractField.AbstractField`
:return: a boolean indicating if the current message can be parsed with the specified field
:rtype: :class:`bool`
"""
if field is None:
raise TypeError("Field cannot be None")
try:
DataAlignment.align([self.data], field)
except:
return False
return True
def isValidForField(self, field):
"""Checks if the specified field can be used
to parse the current message. It returns a boolean
that indicates this.
>>> from netzob.all import *
>>> f0 = Field(Raw(nbBytes=4))
>>> f1 = Field(b", hello ", name="F1")
>>> f2 = Field(Raw(nbBytes=(2,5)), name="F2")
>>> symbol = Symbol([f0, f1, f2], name="Symbol")
>>> m2 = RawMessage(b"Toto, hello you", source="server", destination="client")
>>> m2.isValidForField(symbol)
True
>>> m1 = RawMessage(b"Toto, hello !", source="server", destination="client")
>>> m1.isValidForField(symbol)
False
:parameter field: the current message will be parsed with this field
:type field: :class:`netzob.Model.Vocabulary.AbstractField.AbstractField`
:return: a boolean indicating if the current message can be parsed with the specified field
:rtype: :class:`bool`
"""
if field is None:
raise TypeError("Field cannot be None")
try:
DataAlignment.align([self.data], field)
except:
return False
return True
def abstract(data, fields):
"""Search in the fields/symbols the first one that can abstract the data.
>>> from netzob.all import *
>>> messages = ["{0}, what's up in {1} ?".format(pseudo, city) for pseudo in ['netzob', 'zoby'] for city in ['Paris', 'Berlin']]
>>> f1a = Field("netzob")
>>> f2a = Field(", what's up in ")
>>> f3a = Field(Alt(["Paris", "Berlin"]))
>>> f4a = Field(" ?")
>>> s1 = Symbol([f1a, f2a, f3a, f4a], name="Symbol-netzob")
>>> f1b = Field("zoby")
>>> f2b = Field(", what's up in ")
>>> f3b = Field(Alt(["Paris", "Berlin"]))
>>> f4b = Field(" ?")
>>> s2 = Symbol([f1b, f2b, f3b, f4b], name="Symbol-zoby")
>>> for m in messages:
... abstractedSymbol = AbstractField.abstract(m, [s1, s2])
... print(abstractedSymbol.name)
Symbol-netzob
Symbol-netzob
Symbol-zoby
Symbol-zoby
:parameter data: the data that should be abstracted in symbol
:type data: :class:`str`
:parameter fields: a list of fields/symbols targeted during the abstraction process
:type fields: :class:`list` of :class:`netzob.Model.Vocabulary.AbstractField`
:return: a field/symbol
:rtype: :class:`netzob.Model.Vocabulary.AbstractField`
:raises: :class:`netzob.Model.Vocabulary.AbstractField.AbstractionException` if an error occurs while abstracting the data
"""
from netzob.Common.Utils.DataAlignment.DataAlignment import DataAlignment
for field in fields:
try:
DataAlignment.align([data], field, encoded=False)
return field
except:
pass
from netzob.Model.Vocabulary.UnknownSymbol import UnknownSymbol
from netzob.Model.Vocabulary.Messages.RawMessage import RawMessage
unknown_symbol = UnknownSymbol(RawMessage(data))
logging.error(
"Impossible to abstract the message in one of the specified symbols, we create an unknown symbol for it: '%s'",
unknown_symbol)
return unknown_symbol
def abstract(data, fields):
"""Search in the fields/symbols the first one that can abstract the data.
>>> from netzob.all import *
>>> messages = ["{0}, what's up in {1} ?".format(pseudo, city) for pseudo in ['netzob', 'zoby'] for city in ['Paris', 'Berlin']]
>>> f1a = Field("netzob")
>>> f2a = Field(", what's up in ")
>>> f3a = Field(Alt(["Paris", "Berlin"]))
>>> f4a = Field(" ?")
>>> s1 = Symbol([f1a, f2a, f3a, f4a], name="Symbol-netzob")
>>> f1b = Field("zoby")
>>> f2b = Field(", what's up in ")
>>> f3b = Field(Alt(["Paris", "Berlin"]))
>>> f4b = Field(" ?")
>>> s2 = Symbol([f1b, f2b, f3b, f4b], name="Symbol-zoby")
>>> for m in messages:
... abstractedSymbol = AbstractField.abstract(m, [s1, s2])
... print abstractedSymbol.name
Symbol-netzob
Symbol-netzob
Symbol-zoby
Symbol-zoby
:parameter data: the data that should be abstracted in symbol
:type data: :class:`str`
:parameter fields: a list of fields/symbols targeted during the abstraction process
:type fields: :class:`list` of :class:`netzob.Common.Models.Vocabulary.AbstractField`
:return: a field/symbol
:rtype: :class:`netzob.Common.Models.Vocabulary.AbstractField`
:raises: :class:`netzob.Common.Models.Vocabulary.AbstractField.AbstractionException` if an error occurs while abstracting the data
"""
from netzob.Common.Utils.DataAlignment.DataAlignment import DataAlignment
for field in fields:
try:
DataAlignment.align([data], field, encoded=False)
return field
except:
pass
logging.error("Impossible to abstract the message in one of the specified symbols, we create an unknown symbol for it.")
from netzob.Common.Models.Vocabulary.UnknownSymbol import UnknownSymbol
from netzob.Common.Models.Vocabulary.Messages.RawMessage import RawMessage
return UnknownSymbol(RawMessage(data))
def _executeDataAlignment(arg, **kwargs):
"""Wrapper used to parallelize the DataAlignment using
a pool of threads.
"""
data = arg[0]
field = arg[1]
alignedData = DataAlignment.align([data], field)
return (data, alignedData)
def _executeDataAlignment(arg, **kwargs):
"""Wrapper used to parallelize the DataAlignment using
a pool of threads.
"""
data = arg[0]
field = arg[1]
alignedData = DataAlignment.align([data], field)
return (data, alignedData)
def abstract(data, fields):
"""Search in the fields/symbols the first one that can abstract the data.
>>> from netzob.all import *
>>> messages = ["{0}, what's up in {1} ?".format(pseudo, city) for pseudo in ['netzob', 'zoby'] for city in ['Paris', 'Berlin']]
>>> f1a = Field(name="name", domain="netzob")
>>> f2a = Field(name="question", domain=", what's up in ")
>>> f3a = Field(name="city", domain=Alt(["Paris", "Berlin"]))
>>> f4a = Field(name="mark", domain=" ?")
>>> s1 = Symbol([f1a, f2a, f3a, f4a], name="Symbol-netzob")
>>> f1b = Field(name="name", domain="zoby")
>>> f2b = Field(name="question", domain=", what's up in ")
>>> f3b = Field(name="city", domain=Alt(["Paris", "Berlin"]))
>>> f4b = Field(name="mark", domain=" ?")
>>> s2 = Symbol([f1b, f2b, f3b, f4b], name="Symbol-zoby")
>>> for m in messages:
... (abstractedSymbol, structured_data) = AbstractField.abstract(m, [s1, s2])
... print(structured_data)
... print(abstractedSymbol.name)
OrderedDict([('name', b'netzob'), ('question', b", what's up in "), ('city', b'Paris'), ('mark', b' ?')])
Symbol-netzob
OrderedDict([('name', b'netzob'), ('question', b", what's up in "), ('city', b'Berlin'), ('mark', b' ?')])
Symbol-netzob
OrderedDict([('name', b'zoby'), ('question', b", what's up in "), ('city', b'Paris'), ('mark', b' ?')])
Symbol-zoby
OrderedDict([('name', b'zoby'), ('question', b", what's up in "), ('city', b'Berlin'), ('mark', b' ?')])
Symbol-zoby
:parameter data: the data that should be abstracted in symbol
:type data: :class:`str`
:parameter fields: a list of fields/symbols targeted during the abstraction process
:type fields: :class:`list` of :class:`netzob.Model.Vocabulary.AbstractField`
:return: a field/symbol and the structured received message
:rtype: a tuple (:class:`netzob.Model.Vocabulary.AbstractField`, dict)
:raises: :class:`netzob.Model.Vocabulary.AbstractField.AbstractionException` if an error occurs while abstracting the data
"""
from netzob.Common.Utils.DataAlignment.DataAlignment import DataAlignment
for field in fields:
try:
# Try to align/parse the data with the current field
alignedData = DataAlignment.align([data], field, encoded=False)
# If it matches, we build a dict that contains, for each field, the associated value that was present in the message
structured_data = OrderedDict()
for fields_value in alignedData:
for i, field_value in enumerate(fields_value):
structured_data[alignedData.headers[i]] = field_value
return (field, structured_data)
except:
pass
from netzob.Model.Vocabulary.UnknownSymbol import UnknownSymbol
from netzob.Model.Vocabulary.Messages.RawMessage import RawMessage
unknown_symbol = UnknownSymbol(RawMessage(data))
structured_data = OrderedDict()
logging.error(
"Impossible to abstract the message in one of the specified symbols, we create an unknown symbol for it: '%s'",
unknown_symbol)
return (unknown_symbol, structured_data)
def getCells(self, encoded=True, styled=True, transposed=False):
"""Returns a matrix with a different line for each messages attached to the symbol of the current element.
The matrix includes a different column for each leaf children of the current element.
In each cell, the slices of messages once aligned.
Attached :class:`EncodingFunction` can also be considered if parameter encoded is set to True.
In addition, visualizationFunctions are also applied if parameter styled is set to True.
If parameter Transposed is set to True, the matrix is built with rows for fields and columns for messages.
>>> from netzob.all import *
>>> messages = [RawMessage("hello {0}, what's up in {1} ?".format(pseudo, city)) for pseudo in ['netzob', 'zoby', 'lapy'] for city in ['Paris', 'Berlin', 'New-York']]
>>> fh1 = Field(ASCII("hello "), name="hello")
>>> fh2 = Field(Alt([ASCII("netzob"), ASCII("zoby"), ASCII("lapy"), ASCII("sygus")]), name="pseudo")
>>> fheader = Field(name="header")
>>> fheader.fields = [fh1, fh2]
>>> fb1 = Field(ASCII(", what's up in "), name="whatsup")
>>> fb2 = Field(["Paris", "Berlin", "New-York"], name="city")
>>> fb3 = Field(" ?", name="end")
>>> fbody = Field(name="body")
>>> fbody.fields = [fb1, fb2, fb3]
>>> symbol = Symbol([fheader, fbody], messages=messages)
>>> print(symbol)
hello | pseudo | whatsup | city | end
-------- | -------- | ----------------- | ---------- | ----
'hello ' | 'netzob' | ", what's up in " | 'Paris' | ' ?'
'hello ' | 'netzob' | ", what's up in " | 'Berlin' | ' ?'
'hello ' | 'netzob' | ", what's up in " | 'New-York' | ' ?'
'hello ' | 'zoby' | ", what's up in " | 'Paris' | ' ?'
'hello ' | 'zoby' | ", what's up in " | 'Berlin' | ' ?'
'hello ' | 'zoby' | ", what's up in " | 'New-York' | ' ?'
'hello ' | 'lapy' | ", what's up in " | 'Paris' | ' ?'
'hello ' | 'lapy' | ", what's up in " | 'Berlin' | ' ?'
'hello ' | 'lapy' | ", what's up in " | 'New-York' | ' ?'
-------- | -------- | ----------------- | ---------- | ----
>>> fh1.addEncodingFunction(TypeEncodingFunction(HexaString))
>>> fb2.addEncodingFunction(TypeEncodingFunction(HexaString))
>>> print(symbol)
hello | pseudo | whatsup | city | end
-------------- | -------- | ----------------- | ------------------ | ----
'68656c6c6f20' | 'netzob' | ", what's up in " | '5061726973' | ' ?'
'68656c6c6f20' | 'netzob' | ", what's up in " | '4265726c696e' | ' ?'
'68656c6c6f20' | 'netzob' | ", what's up in " | '4e65772d596f726b' | ' ?'
'68656c6c6f20' | 'zoby' | ", what's up in " | '5061726973' | ' ?'
'68656c6c6f20' | 'zoby' | ", what's up in " | '4265726c696e' | ' ?'
'68656c6c6f20' | 'zoby' | ", what's up in " | '4e65772d596f726b' | ' ?'
'68656c6c6f20' | 'lapy' | ", what's up in " | '5061726973' | ' ?'
'68656c6c6f20' | 'lapy' | ", what's up in " | '4265726c696e' | ' ?'
'68656c6c6f20' | 'lapy' | ", what's up in " | '4e65772d596f726b' | ' ?'
-------------- | -------- | ----------------- | ------------------ | ----
>>> print(fheader.getCells())
Field | Field
-------------- | --------
'68656c6c6f20' | 'netzob'
'68656c6c6f20' | 'netzob'
'68656c6c6f20' | 'netzob'
'68656c6c6f20' | 'zoby'
'68656c6c6f20' | 'zoby'
'68656c6c6f20' | 'zoby'
'68656c6c6f20' | 'lapy'
'68656c6c6f20' | 'lapy'
'68656c6c6f20' | 'lapy'
-------------- | --------
>>> print(fh1.getCells())
Field
--------------
'68656c6c6f20'
'68656c6c6f20'
'68656c6c6f20'
'68656c6c6f20'
'68656c6c6f20'
'68656c6c6f20'
'68656c6c6f20'
'68656c6c6f20'
'68656c6c6f20'
--------------
>>> print(fh2.getCells())
Field
--------
'netzob'
'netzob'
'netzob'
'zoby'
'zoby'
'zoby'
'lapy'
'lapy'
'lapy'
--------
>>> print(fbody.getCells())
Field | Field | Field
----------------- | ------------------ | -----
", what's up in " | '5061726973' | ' ?'
", what's up in " | '4265726c696e' | ' ?'
", what's up in " | '4e65772d596f726b' | ' ?'
", what's up in " | '5061726973' | ' ?'
", what's up in " | '4265726c696e' | ' ?'
", what's up in " | '4e65772d596f726b' | ' ?'
", what's up in " | '5061726973' | ' ?'
", what's up in " | '4265726c696e' | ' ?'
", what's up in " | '4e65772d596f726b' | ' ?'
----------------- | ------------------ | -----
>>> print(fb1.getCells())
Field
-----------------
", what's up in "
", what's up in "
", what's up in "
", what's up in "
", what's up in "
", what's up in "
", what's up in "
", what's up in "
", what's up in "
-----------------
>>> print(fb2.getCells())
Field
------------------
'5061726973'
'4265726c696e'
'4e65772d596f726b'
'5061726973'
'4265726c696e'
'4e65772d596f726b'
'5061726973'
'4265726c696e'
'4e65772d596f726b'
------------------
>>> print(fb3.getCells())
Field
-----
' ?'
' ?'
' ?'
' ?'
' ?'
' ?'
' ?'
' ?'
' ?'
-----
:keyword encoded: if set to True, encoding functions are applied on returned cells
:type encoded: :class:`bool`
:keyword styled: if set to True, visualization functions are applied on returned cells
:type styled: :class:`bool`
:keyword transposed: is set to True, the returned matrix is transposed (1 line for each field)
:type transposed: :class:`bool`
:return: a matrix representing the aligned messages following fields definitions.
:rtype: a :class:`netzob.Common.Utils.MatrixList.MatrixList`
:raises: :class:`netzob.Model.Vocabulary.AbstractField.AlignmentException` if an error occurs while aligning messages
"""
if len(self.messages) < 1:
raise ValueError("This symbol does not contain any message.")
# Fetch all the data to align
data = [message.data for message in self.messages]
# [DEBUG] set to false for debug only. A sequential alignment is more simple to debug
useParallelAlignment = False
if useParallelAlignment:
# Execute a parallel alignment
from netzob.Common.Utils.DataAlignment.ParallelDataAlignment import ParallelDataAlignment
return ParallelDataAlignment.align(data, self, encoded=encoded)
else:
# Execute a sequential alignment
from netzob.Common.Utils.DataAlignment.DataAlignment import DataAlignment
return DataAlignment.align(data, self, encoded=encoded)
def cluster(self, field, keyField):
"""Create and return new symbols according to a specific key
field.
>>> import binascii
>>> from netzob.all import *
>>> samples = [b"00ff2f000000", b"000020000000", b"00ff2f000000"]
>>> messages = [RawMessage(data=binascii.unhexlify(sample)) for sample in samples]
>>> f1 = Field(Raw(nbBytes=1))
>>> f2 = Field(Raw(nbBytes=2))
>>> f3 = Field(Raw(nbBytes=3))
>>> symbol = Symbol([f1, f2, f3], messages=messages)
>>> symbol.addEncodingFunction(TypeEncodingFunction(HexaString))
>>> newSymbols = Format.clusterByKeyField(symbol, f2)
>>> for sym in list(newSymbols.values()):
... sym.addEncodingFunction(TypeEncodingFunction(HexaString))
... print(sym.name + ":")
... print(sym)
Symbol_ff2f:
Field | Field | Field
----- | ------ | --------
'00' | 'ff2f' | '000000'
'00' | 'ff2f' | '000000'
----- | ------ | --------
Symbol_0020:
Field | Field | Field
----- | ------ | --------
'00' | '0020' | '000000'
----- | ------ | --------
:param field: the field we want to split in new symbols
:type field: :class:`netzob.Model.Vocabulary.AbstractField.AbstractField`
:param keyField: the field used as a key during the splitting operation
:type field: :class:`netzob.Model.Vocabulary.AbstractField.AbstractField`
:raise Exception if something bad happens
"""
# Safe checks
if field is None:
raise TypeError("'field' should not be None")
if keyField is None:
raise TypeError("'keyField' should not be None")
if keyField not in field.fields:
raise TypeError("'keyField' is not a child of 'field'")
newSymbols = collections.OrderedDict()
keyFieldMessageValues = keyField.getMessageValues(encoded=False,
styled=False)
newSymbolsSplittedMessages = {}
# we identify what would be the best type of the key field
keyFieldType = ASCII
for message, keyFieldValue in list(keyFieldMessageValues.items()):
# If the value cannot be parsed as ASCII, we convert it to HexaString
if not ASCII().canParse(
TypeConverter.convert(keyFieldValue, Raw, BitArray)):
keyFieldType = HexaString
break
# Even if the value is theoritically parsable as ASCII, some caracters cannot be encoded, so we double check
tmp_value = TypeConverter.convert(keyFieldValue, Raw, ASCII)
tmp2_value = TypeConverter.convert(tmp_value, ASCII, Raw)
if keyFieldValue != tmp2_value:
# This means we cannot retrieve the original value by encoding and then decoding in ASCII
keyFieldType = HexaString
break
# we create a symbol for each of these uniq values
for message, keyFieldValue in list(keyFieldMessageValues.items()):
keyFieldValue = TypeConverter.convert(keyFieldValue, Raw,
keyFieldType)
if keyFieldValue not in list(newSymbols.keys()):
if type(keyFieldValue) is str:
symbolName = "Symbol_{0}".format(keyFieldValue)
else:
symbolName = "Symbol_{0}".format(
keyFieldValue.decode("utf-8"))
newSymbols[keyFieldValue] = Symbol(name=symbolName,
messages=[message])
splittedMessages = DataAlignment.align([message.data],
field,
encoded=False)
newSymbolsSplittedMessages[keyFieldValue] = [
splittedMessages[0]
]
else:
newSymbols[keyFieldValue].messages.append(message)
splittedMessages = DataAlignment.align([message.data],
field,
encoded=False)
newSymbolsSplittedMessages[keyFieldValue].append(
splittedMessages[0])
for newSymbolKeyValue, newSymbol in list(newSymbols.items()):
# we recreate the same fields in this new symbol as the fields that exist in the original symbol
newSymbol.clearFields()
for i, f in enumerate(field.fields):
if f == keyField:
newFieldDomain = TypeConverter.convert(
newSymbolKeyValue, keyFieldType, Raw)
else:
newFieldDomain = set()
for j in range(
len(newSymbolsSplittedMessages[newSymbolKeyValue])
):
newFieldDomain.add(
newSymbolsSplittedMessages[newSymbolKeyValue][j]
[i])
newFieldDomain = list(newFieldDomain)
newF = Field(name=f.name, domain=newFieldDomain)
newF.parent = newSymbol
newSymbol.fields.append(newF)
# we remove endless fields that accepts no values
cells = newSymbol.getCells(encoded=False,
styled=False,
transposed=False)
max_i_cell_with_value = 0
for line in cells:
for i_cell, cell in enumerate(line):
if cell != '' and max_i_cell_with_value < i_cell:
max_i_cell_with_value = i_cell
newSymbol.clearFields()
for i, f in enumerate(field.fields[:max_i_cell_with_value + 1]):
if f == keyField:
newFieldDomain = TypeConverter.convert(
newSymbolKeyValue, keyFieldType, Raw)
else:
newFieldDomain = set()
for j in range(
len(newSymbolsSplittedMessages[newSymbolKeyValue])
):
newFieldDomain.add(
newSymbolsSplittedMessages[newSymbolKeyValue][j]
[i])
newFieldDomain = list(newFieldDomain)
newF = Field(name=f.name, domain=newFieldDomain)
newF.parent = newSymbol
newSymbol.fields.append(newF)
return newSymbols
def cluster(self, field, keyField):
"""Create and return new symbols according to a specific key
field.
>>> import binascii
>>> from netzob.all import *
>>> samples = ["00ff2f000000", "000020000000", "00ff2f000000"]
>>> messages = [RawMessage(data=binascii.unhexlify(sample)) for sample in samples]
>>> f1 = Field(Raw(nbBytes=1))
>>> f2 = Field(Raw(nbBytes=2))
>>> f3 = Field(Raw(nbBytes=3))
>>> symbol = Symbol([f1, f2, f3], messages=messages)
>>> symbol.addEncodingFunction(TypeEncodingFunction(HexaString))
>>> newSymbols = Format.clusterByKeyField(symbol, f2)
>>> for sym in newSymbols.values():
... sym.addEncodingFunction(TypeEncodingFunction(HexaString))
... print sym.name + ":"
... print sym
Symbol_ff2f:
Field | Field | Field
----- | ------ | --------
'00' | 'ff2f' | '000000'
'00' | 'ff2f' | '000000'
----- | ------ | --------
Symbol_0020:
Field | Field | Field
----- | ------ | --------
'00' | '0020' | '000000'
----- | ------ | --------
:param field: the field we want to split in new symbols
:type field: :class:`netzob.Common.Models.Vocabulary.AbstractField.AbstractField`
:param keyField: the field used as a key during the splitting operation
:type field: :class:`netzob.Common.Models.Vocabulary.AbstractField.AbstractField`
:raise Exception if something bad happens
"""
# Safe checks
if field is None:
raise TypeError("'field' should not be None")
if keyField is None:
raise TypeError("'keyField' should not be None")
if keyField not in field.fields:
raise TypeError("'keyField' is not a child of 'field'")
newSymbols = {}
keyFieldMessageValues = keyField.getMessageValues(encoded=False, styled=False)
newSymbolsSplittedMessages = {}
# we identify what would be the best type of the key field
keyFieldType = ASCII
for message, keyFieldValue in keyFieldMessageValues.iteritems():
# If the value cannot be parsed as ASCII, we convert it to HexaString
if not ASCII().canParse(TypeConverter.convert(keyFieldValue, Raw, BitArray)):
keyFieldType = HexaString
break
# Even if the value is theoritically parsable as ASCII, some caracters cannot be encoded, so we double check
tmp_value = TypeConverter.convert(keyFieldValue, Raw, ASCII)
tmp2_value = TypeConverter.convert(tmp_value, ASCII, Raw)
if keyFieldValue != tmp2_value:
# This means we cannot retrieve the original value by encoding and then decoding in ASCII
keyFieldType = HexaString
break
# we create a symbol for each of these uniq values
for message, keyFieldValue in keyFieldMessageValues.iteritems():
keyFieldValue = TypeConverter.convert(keyFieldValue, Raw, keyFieldType)
if keyFieldValue not in newSymbols.keys():
symbolName = "Symbol_{0}".format(keyFieldValue)
newSymbols[keyFieldValue] = Symbol(name=symbolName, messages=[message])
splittedMessages = DataAlignment.align([message.data], field, encoded=False)
newSymbolsSplittedMessages[keyFieldValue] = [splittedMessages[0]]
else:
newSymbols[keyFieldValue].messages.append(message)
splittedMessages = DataAlignment.align([message.data], field, encoded=False)
newSymbolsSplittedMessages[keyFieldValue].append(splittedMessages[0])
for newSymbolKeyValue, newSymbol in newSymbols.iteritems():
# we recreate the same fields in this new symbol as the fields that exist in the original symbol
newSymbol.clearFields()
for i, f in enumerate(field.fields):
if f == keyField:
newFieldDomain = TypeConverter.convert(newSymbolKeyValue, keyFieldType, Raw)
else:
newFieldDomain = set()
for j in range(len(newSymbolsSplittedMessages[newSymbolKeyValue])):
newFieldDomain.add(newSymbolsSplittedMessages[newSymbolKeyValue][j][i])
newFieldDomain = list(newFieldDomain)
newF = Field(name=f.name, domain=newFieldDomain)
newF.parent = newSymbol
newSymbol.fields.append(newF)
# we remove endless fields that accepts no values
cells = newSymbol.getCells(encoded=False, styled=False, transposed=False)
max_i_cell_with_value = 0
for line in cells:
for i_cell, cell in enumerate(line):
if cell != '' and max_i_cell_with_value < i_cell:
max_i_cell_with_value = i_cell
newSymbol.clearFields()
for i, f in enumerate(field.fields[:max_i_cell_with_value + 1]):
if f == keyField:
newFieldDomain = TypeConverter.convert(newSymbolKeyValue, keyFieldType, Raw)
else:
newFieldDomain = set()
for j in range(len(newSymbolsSplittedMessages[newSymbolKeyValue])):
newFieldDomain.add(newSymbolsSplittedMessages[newSymbolKeyValue][j][i])
newFieldDomain = list(newFieldDomain)
newF = Field(name=f.name, domain=newFieldDomain)
newF.parent = newSymbol
newSymbol.fields.append(newF)
return newSymbols
def abstract(data, fields):
"""Search in the fields/symbols the first one that can abstract the data.
>>> from netzob.all import *
>>> messages = ["{0}, what's up in {1} ?".format(pseudo, city) for pseudo in ['netzob', 'zoby'] for city in ['Paris', 'Berlin']]
>>> f1a = Field(name="name", domain="netzob")
>>> f2a = Field(name="question", domain=", what's up in ")
>>> f3a = Field(name="city", domain=Alt(["Paris", "Berlin"]))
>>> f4a = Field(name="mark", domain=" ?")
>>> s1 = Symbol([f1a, f2a, f3a, f4a], name="Symbol-netzob")
>>> f1b = Field(name="name", domain="zoby")
>>> f2b = Field(name="question", domain=", what's up in ")
>>> f3b = Field(name="city", domain=Alt(["Paris", "Berlin"]))
>>> f4b = Field(name="mark", domain=" ?")
>>> s2 = Symbol([f1b, f2b, f3b, f4b], name="Symbol-zoby")
>>> for m in messages:
... (abstractedSymbol, structured_data) = AbstractField.abstract(m, [s1, s2])
... print(structured_data)
... print(abstractedSymbol.name)
OrderedDict([('name', b'netzob'), ('question', b", what's up in "), ('city', b'Paris'), ('mark', b' ?')])
Symbol-netzob
OrderedDict([('name', b'netzob'), ('question', b", what's up in "), ('city', b'Berlin'), ('mark', b' ?')])
Symbol-netzob
OrderedDict([('name', b'zoby'), ('question', b", what's up in "), ('city', b'Paris'), ('mark', b' ?')])
Symbol-zoby
OrderedDict([('name', b'zoby'), ('question', b", what's up in "), ('city', b'Berlin'), ('mark', b' ?')])
Symbol-zoby
:parameter data: the data that should be abstracted in symbol
:type data: :class:`str`
:parameter fields: a list of fields/symbols targeted during the abstraction process
:type fields: :class:`list` of :class:`netzob.Model.Vocabulary.AbstractField`
:return: a field/symbol and the structured received message
:rtype: a tuple (:class:`netzob.Model.Vocabulary.AbstractField`, dict)
:raises: :class:`netzob.Model.Vocabulary.AbstractField.AbstractionException` if an error occurs while abstracting the data
"""
from netzob.Common.Utils.DataAlignment.DataAlignment import DataAlignment
for field in fields:
try:
# Try to align/parse the data with the current field
alignedData = DataAlignment.align([data], field, encoded=False)
# If it matches, we build a dict that contains, for each field, the associated value that was present in the message
structured_data = OrderedDict()
for fields_value in alignedData:
for i, field_value in enumerate(fields_value):
structured_data[alignedData.headers[i]] = field_value
return (field, structured_data)
except:
pass
from netzob.Model.Vocabulary.UnknownSymbol import UnknownSymbol
from netzob.Model.Vocabulary.Messages.RawMessage import RawMessage
unknown_symbol = UnknownSymbol(RawMessage(data))
structured_data = OrderedDict()
logging.error("Impossible to abstract the message in one of the specified symbols, we create an unknown symbol for it: '%s'", unknown_symbol)
return (unknown_symbol, structured_data)
def getCells(self, encoded=True, styled=True, transposed=False):
"""Returns a matrix with a different line for each messages attached to the symbol of the current element.
The matrix includes a different column for each leaf children of the current element.
In each cell, the slices of messages once aligned.
Attached :class:`EncodingFunction` can also be considered if parameter encoded is set to True.
In addition, visualizationFunctions are also applied if parameter styled is set to True.
If parameter Transposed is set to True, the matrix is built with rows for fields and columns for messages.
>>> from netzob.all import *
>>> messages = [RawMessage("hello {0}, what's up in {1} ?".format(pseudo, city)) for pseudo in ['netzob', 'zoby', 'lapy'] for city in ['Paris', 'Berlin', 'New-York']]
>>> fh1 = Field(ASCII("hello "), name="hello")
>>> fh2 = Field(Alt([ASCII("netzob"), ASCII("zoby"), ASCII("lapy"), ASCII("sygus")]), name="pseudo")
>>> fheader = Field(name="header")
>>> fheader.fields = [fh1, fh2]
>>> fb1 = Field(ASCII(", what's up in "), name="whatsup")
>>> fb2 = Field(["Paris", "Berlin", "New-York"], name="city")
>>> fb3 = Field(" ?", name="end")
>>> fbody = Field(name="body")
>>> fbody.fields = [fb1, fb2, fb3]
>>> symbol = Symbol([fheader, fbody], messages=messages)
>>> print(symbol)
hello | pseudo | whatsup | city | end
-------- | -------- | ----------------- | ---------- | ----
'hello ' | 'netzob' | ", what's up in " | 'Paris' | ' ?'
'hello ' | 'netzob' | ", what's up in " | 'Berlin' | ' ?'
'hello ' | 'netzob' | ", what's up in " | 'New-York' | ' ?'
'hello ' | 'zoby' | ", what's up in " | 'Paris' | ' ?'
'hello ' | 'zoby' | ", what's up in " | 'Berlin' | ' ?'
'hello ' | 'zoby' | ", what's up in " | 'New-York' | ' ?'
'hello ' | 'lapy' | ", what's up in " | 'Paris' | ' ?'
'hello ' | 'lapy' | ", what's up in " | 'Berlin' | ' ?'
'hello ' | 'lapy' | ", what's up in " | 'New-York' | ' ?'
-------- | -------- | ----------------- | ---------- | ----
>>> fh1.addEncodingFunction(TypeEncodingFunction(HexaString))
>>> fb2.addEncodingFunction(TypeEncodingFunction(HexaString))
>>> print(symbol)
hello | pseudo | whatsup | city | end
-------------- | -------- | ----------------- | ------------------ | ----
'68656c6c6f20' | 'netzob' | ", what's up in " | '5061726973' | ' ?'
'68656c6c6f20' | 'netzob' | ", what's up in " | '4265726c696e' | ' ?'
'68656c6c6f20' | 'netzob' | ", what's up in " | '4e65772d596f726b' | ' ?'
'68656c6c6f20' | 'zoby' | ", what's up in " | '5061726973' | ' ?'
'68656c6c6f20' | 'zoby' | ", what's up in " | '4265726c696e' | ' ?'
'68656c6c6f20' | 'zoby' | ", what's up in " | '4e65772d596f726b' | ' ?'
'68656c6c6f20' | 'lapy' | ", what's up in " | '5061726973' | ' ?'
'68656c6c6f20' | 'lapy' | ", what's up in " | '4265726c696e' | ' ?'
'68656c6c6f20' | 'lapy' | ", what's up in " | '4e65772d596f726b' | ' ?'
-------------- | -------- | ----------------- | ------------------ | ----
>>> print(fheader.getCells())
Field | Field
-------------- | --------
'68656c6c6f20' | 'netzob'
'68656c6c6f20' | 'netzob'
'68656c6c6f20' | 'netzob'
'68656c6c6f20' | 'zoby'
'68656c6c6f20' | 'zoby'
'68656c6c6f20' | 'zoby'
'68656c6c6f20' | 'lapy'
'68656c6c6f20' | 'lapy'
'68656c6c6f20' | 'lapy'
-------------- | --------
>>> print(fh1.getCells())
Field
--------------
'68656c6c6f20'
'68656c6c6f20'
'68656c6c6f20'
'68656c6c6f20'
'68656c6c6f20'
'68656c6c6f20'
'68656c6c6f20'
'68656c6c6f20'
'68656c6c6f20'
--------------
>>> print(fh2.getCells())
Field
--------
'netzob'
'netzob'
'netzob'
'zoby'
'zoby'
'zoby'
'lapy'
'lapy'
'lapy'
--------
>>> print(fbody.getCells())
Field | Field | Field
----------------- | ------------------ | -----
", what's up in " | '5061726973' | ' ?'
", what's up in " | '4265726c696e' | ' ?'
", what's up in " | '4e65772d596f726b' | ' ?'
", what's up in " | '5061726973' | ' ?'
", what's up in " | '4265726c696e' | ' ?'
", what's up in " | '4e65772d596f726b' | ' ?'
", what's up in " | '5061726973' | ' ?'
", what's up in " | '4265726c696e' | ' ?'
", what's up in " | '4e65772d596f726b' | ' ?'
----------------- | ------------------ | -----
>>> print(fb1.getCells())
Field
-----------------
", what's up in "
", what's up in "
", what's up in "
", what's up in "
", what's up in "
", what's up in "
", what's up in "
", what's up in "
", what's up in "
-----------------
>>> print(fb2.getCells())
Field
------------------
'5061726973'
'4265726c696e'
'4e65772d596f726b'
'5061726973'
'4265726c696e'
'4e65772d596f726b'
'5061726973'
'4265726c696e'
'4e65772d596f726b'
------------------
>>> print(fb3.getCells())
Field
-----
' ?'
' ?'
' ?'
' ?'
' ?'
' ?'
' ?'
' ?'
' ?'
-----
:keyword encoded: if set to True, encoding functions are applied on returned cells
:type encoded: :class:`bool`
:keyword styled: if set to True, visualization functions are applied on returned cells
:type styled: :class:`bool`
:keyword transposed: is set to True, the returned matrix is transposed (1 line for each field)
:type transposed: :class:`bool`
:return: a matrix representing the aligned messages following fields definitions.
:rtype: a :class:`netzob.Common.Utils.MatrixList.MatrixList`
:raises: :class:`netzob.Model.Vocabulary.AbstractField.AlignmentException` if an error occurs while aligning messages
"""
if len(self.messages) < 1:
raise ValueError("This symbol does not contain any message.")
# Fetch all the data to align
data = [message.data for message in self.messages]
# [DEBUG] set to false for debug only. A sequential alignment is more simple to debug
useParallelAlignment = False
if useParallelAlignment:
# Execute a parallel alignment
from netzob.Common.Utils.DataAlignment.ParallelDataAlignment import ParallelDataAlignment
return ParallelDataAlignment.align(data, self, encoded=encoded)
else:
# Execute a sequential alignment
from netzob.Common.Utils.DataAlignment.DataAlignment import DataAlignment
return DataAlignment.align(data, self, encoded=encoded)
