def merge_nontest_fields(self, fields_origin, fid):
logging.debug("[+] Merge Fields")
fields_merged = list()
fields = copy.deepcopy(fields_origin)
fsize_total = 0
for i in range(len(fields)):
typename = fields[i].domain.dataType.typeName
if typename != "Raw":
logging.error("Field type is not Raw")
typesize = fields[i].domain.dataType.size[1]
# print(i, fid, typesize, type(typesize))
if i == fid:
if fsize_total > 0:
field = Field(Raw(nbBytes=fsize_total // 8))
fields_merged.append(field)
fsize_total = 0
if fid != (len(fields) - 1):
field = Field(Raw(nbBytes=typesize // 8))
fields_merged.append(field)
field = Field()
fields_merged.append(field)
else:
if type(typesize).__name__ == 'NoneType':
field = Field()
else:
field = Field(Raw(nbBytes=typesize // 8))
fields_merged.append(field)
break
else:
fsize_total += typesize
return fields_merged
def __init__(self,
domain=None,
name="Field",
isPseudoField=False,
layer=False,
messages=None,
specializingPaths=None):
"""
:keyword domain: the definition domain of the field (see domain property to get more information)
:type domain: a :class:`list` of :class:`object`, default is Raw(None)
:keyword name: the name of the field
:type name: :class:`str`
:keyword isPseudoField: a flag indicating if field is a pseudo field, meaning it is used internally but does not produce data
:type isPseudoField: :class:`bool`
"""
super(Field, self).__init__(name)
if domain is None:
domain = Raw(None)
self.domain = domain
if messages is None:
messages = []
self.messages = messages
self.specializingPaths = specializingPaths
self.isPseudoField = isPseudoField
def __define_field(self,val_set,field):
"""
:param val_set: A set of values in the field
:param field: The field containing the CRC
:return: new_field: A new field (Alt or Raw)
"""
if len(val_set) > 1:
# More than one value, create Alt Fields
domain_list = []
for value in val_set:
domain_list.append(Raw(value))
new_field = Field(name=field.name, domain=Alt(domain_list))
else:
# Only one value, create static Fields
new_field = Field(name=field.name, domain=Raw(val_set.pop()))
return new_field
def __init__(self, fields, dataType=None, name=None):
if isinstance(fields, AbstractField):
fields = [fields]
super(InternetChecksum, self).__init__("InternetChecksum",
fieldDependencies=fields,
name=name)
if dataType is None:
dataType = Raw(nbBytes=2)
self.dataType = dataType
def test_split_delimiter():
BPF_FILTER = "!(arp) and !(len == 96)"
messages = PCAPImporter.readFile(
"/home/research/Downloads/hunter_no_vlan.pcap",
nbPackets=10,
bpfFilter=BPF_FILTER).values()
symbol = Symbol(messages=messages)
Format.splitDelimiter(symbol.fields[0], Raw(b'\05'))
def generate_fields(self, fields_result):
fields = list()
for typeinfo in fields_result:
if typeinfo[0] == "Raw":
field = Field(Raw(nbBytes=(typeinfo[1] // 8,
typeinfo[2] // 8)))
fields.append(field)
else:
logging.error("Field type is not Raw")
return fields
def _splitFieldFollowingAlignment(self, field, align):
"""Update the field definition with new fields following the
specified align."""
# STEP 1 : Create a field separation based on static and dynamic fields
leftAlign, rightAlign = self._splitAlignment(align)
splited = self._mergeAlign(leftAlign, rightAlign)
step1Fields = []
for (entryVal, entryDyn) in splited:
if entryDyn:
newField = Field(Raw(nbBytes=(0, int(len(entryVal) / 2))))
else:
newField = Field(
Raw(TypeConverter.convert(entryVal, HexaString, Raw)))
step1Fields.append(newField)
for f in step1Fields:
f.encodingFunctions = list(field.encodingFunctions.values())
field.fields = step1Fields
def __init__(self,
fields,
dataType=None,
name=None,
endianness=AbstractType.defaultEndianness()):
if isinstance(fields, AbstractField):
fields = [fields]
super(CRC32, self).__init__(varType="CRC32",
fieldDependencies=fields,
name=name)
if dataType is None:
dataType = Raw(nbBytes=4)
self.dataType = dataType
self.endianness = endianness
def __init__(self,
fields,
dataType=None,
factor=1 / float(8),
offset=0,
name=None):
if isinstance(fields, AbstractField):
fields = [fields]
super(Size, self).__init__("Size", fieldDependencies=fields, name=name)
self.fields = fields
if dataType is None:
dataType = Raw(nbBytes=1)
self.dataType = dataType
self.factor = factor
self.offset = offset
def normalize(data):
"""Given the specified data, this static methods normalize its representation
using Netzob types.
:parameter data: the data to normalize
:type data: :class:`object`
:return: an abstractType which value is data
:rtype: :class:`netzob.Model.Vocabulary.Types.AbstractType.AbstractType`
>>> from netzob.all import *
>>> normalizedData = AbstractType.normalize("netzob")
>>> print(normalizedData.__class__)
<class 'netzob.Model.Vocabulary.Types.ASCII.ASCII'>
>>> print(normalizedData.value)
bitarray('011011100110010101110100011110100110111101100010')
"""
if data is None:
raise TypeError("Cannot normalize None data")
normalizedData = None
if isinstance(data, AbstractType):
return data
elif isinstance(data, int):
from netzob.Model.Vocabulary.Types.Integer import Integer
return Integer(value=data)
elif isinstance(data, bytes):
from netzob.Model.Vocabulary.Types.Raw import Raw
normalizedData = Raw(value=data)
elif isinstance(data, str):
from netzob.Model.Vocabulary.Types.ASCII import ASCII
normalizedData = ASCII(value=data)
if normalizedData is None:
raise TypeError(
"Not a valid data ({0}), impossible to normalize it.",
type(data))
return normalizedData
def reset(self, field):
"""Resets the format (field hierarchy and definition domain) of
the specified field.
:param field: the field we want to reset
:type field: :class:`netzob.Model.Vocabulary.AbstractField.AbstractField`
:raise Exception if something bad happens
"""
if field is None:
raise TypeError(
"The field to reset must be specified and cannot be None")
self._logger.debug("Reset the definition of field {0} ({1})".format(
field.name, field.id))
field.clearFields()
if isinstance(field, Symbol):
field.fields = [Field()]
if isinstance(field, Field):
field.domain = Raw(None)
def __seek_ip(self, ip, symbol, create_fields=False, two_terms=False):
#Convert IP to hex value:
hexipstring = binascii.hexlify(socket.inet_aton(ip))
hexipstring = binascii.unhexlify(hexipstring)
index_list = []
for message in symbol.messages:
results = self.__core_find(message.data, hexipstring, index_list,
two_terms)
#Change stdout to get message print in buffer
old_stdout = sys.stdout
sys.stdout = buffer1 = io.StringIO()
print(message)
sys.stdout = old_stdout
#Print results using click
self._logger.warning("Results for [Message] : \n" +
buffer1.getvalue() + "\n")
self._logger.warning("[Number of results found] : " +
str(results.total_length) + "\n")
self._logger.warning("Result indexes in message: \n ")
self._logger.warning("[Whole IP Big Endian] : " +
str(results.full_be) + "\n")
self._logger.warning("[Three last terms of IP Big Endian] : " +
str(results.one_less_be) + "\n")
self._logger.warning("[Two last terms of IP Big Endian] : " +
str(results.two_less_be) + "\n")
self._logger.warning("[Whole IP Little Endian] : " +
str(results.full_le) + "\n")
self._logger.warning("[Three last terms of IP Little Endian] : " +
str(results.one_less_le) + "\n")
self._logger.warning("[Two last terms of IP Little Endian] : " +
str(results.two_less_le) + "\n")
if create_fields:
self._logger.warning("Attempting to create new fields")
if symbol.fields:
self._logger.warning("Refining search to fields...")
for field in symbol.fields:
subfield_index_list = []
field_values = field.getValues()
number_of_values = len(set(field_values))
# Get field length
max_length = max([len(i) for i in field_values])
# Check if values in messages are different for need to create an alternative field or a simple raw field (above 1 if several values):
if number_of_values > 1:
# More than one value => MUST CREATE ALT FIELD
# TODO
pass
else:
mess = field_values[0]
field_result = self.__core_find(
mess, hexipstring, index_list, two_terms)
if field_result.full_be or field_result.one_less_be or field_result.two_less_be or field_result.full_le or field_result.one_less_le or field_result.two_less_le:
# Searchstring not always split in between fields => Need to create subfields
self._logger.warning(
"Searchstring inside fields, creating subfields..."
)
# Create field dict which contains fields and index
fields_dict = dict()
if number_of_values > 1:
#More than one value => MUST CREATE ALT FIELD
#TODO
pass
else:
#Can create simple static subfield
for i in field_result.full_be:
fields_dict[i] = Field(
name='FullIpBe' + str(i),
domain=Raw(hexipstring))
subfield_index_list.append(i)
subfield_index_list.append(i + 4)
for i in field_result.one_less_be:
fields_dict[i] = Field(
name='ThreeTIpBe' + str(i),
domain=Raw(hexipstring[1:]))
subfield_index_list.append(i)
subfield_index_list.append(i + 3)
for i in field_result.two_less_be:
fields_dict[i] = Field(
name='TwoTIpBe' + str(i),
domain=Raw(hexipstring[2:]))
subfield_index_list.append(i)
subfield_index_list.append(i + 2)
for i in field_result.full_le:
fields_dict[i] = Field(
name='FullIpLe' + str(i),
domain=Raw(hexipstring[::-1]))
subfield_index_list.append(i)
subfield_index_list.append(i + 4)
for i in field_result.one_less_le:
fields_dict[i] = Field(
name='ThreeTIpLe' + str(i),
domain=Raw(hexipstring[1:][::-1]))
subfield_index_list.append(i)
subfield_index_list.append(i + 3)
fields_dict[i] = Field(
name='TwoTIpLe' + str(i),
domain=Raw(hexipstring[2:][::-1]))
subfield_index_list.append(i)
subfield_index_list.append(i + 2)
#Sort list in order
subfield_index_list.append(max_length)
subfield_index_list.insert(0, 0)
#Remove duplicates (there shouldn't be any!)
subfield_index_list = list(
set(subfield_index_list))
subfield_index_list = sorted(
subfield_index_list, key=int)
#Create static field for every two elements of the subfield index_list as they represent beginning and end
for i, x in enumerate(subfield_index_list):
#Check if index is pair
if (i % 2 == 0):
#Create static field and store it's beginning index in the structure
try:
fields_dict[x] = Field(
name='Field' + str(i),
domain=Raw(field_values[0][
subfield_index_list[i]:
subfield_index_list[i +
1]]
))
except:
fields_dict[x] = Field(
name='Field' + str(i),
domain=Raw(
field_values[0]
[subfield_index_list[
i]:]))
else:
# Don't do shit
pass
#Create a list of all subfields in order
od = collections.OrderedDict(
sorted(fields_dict.items()))
field_list = list(od.values())
if not field.fields:
#No subfields
field.fields = field_list
else:
#Already contains subfields. We need to just insert our IP subfields in it.
#Get the indexes of Fields which have a name other than FieldX (where X is a number)
only_Ip_fields_dict = dict()
for index, element in fields_dict.items():
if not element.name[:5] == "Field":
only_Ip_fields_dict[
index] = element
#Find to which field these indexes belong to in the old symbol
for index, element in only_Ip_fields_dict.items(
):
new_field_dict = dict()
field_to_split, maxSize, indexInField = self.__getFieldFromIndex(
index, field)
indexInField = int(indexInField / 8)
new_field_dict[indexInField] = element
if field_to_split.getValues(
)[0][:indexInField]:
new_field_dict[0] = Field(
name='FieldBeforeIp' +
str(index),
domain=Raw(
field_to_split.getValues()
[0][:indexInField]))
new_field_dict[indexInField] = element
#Get Size of the IP Field
minSize, sizeIp = element.domain.dataType.size
sizeIp = int(sizeIp / 8)
if field_to_split.getValues(
)[0][indexInField + sizeIp:]:
new_field_dict[
indexInField + sizeIp] = Field(
name='FieldAfterIp' +
str(index),
domain=Raw(field_to_split.
getValues()[0]
[indexInField +
sizeIp:]))
od = collections.OrderedDict(
sorted(new_field_dict.items()))
field_list = list(od.values())
#DELETE EMPTY DOMAIN FIELDS
field_to_split.fields = field_list
else:
#Searchstring always split in between other fields => Delete fields and create new ones Or just return indexes, and let user redefine fields manually
# symbol.fields = []
self._logger.warning(
"Searchstring in between fields...")
self._logger.warning(
"[1] : Delete all fields and replace by IPFIELDS"
+ "\n")
self._logger.warning(
"[ANY] : Just print the index in the message"
+ "\n")
return
def execute(self, field):
"""Executes the field edition following the specified messages.
Children of the specified field will be replaced with new fields.
:param field: the format definition that will be user
:type field: :class:`netzob.Model.Vocabulary.AbstractField.AbstractField`
:raise Exception: if something bad happens
"""
if field is None:
raise TypeError("The field cannot be None")
fieldValues = [
TypeConverter.convert(data, Raw, HexaString)
for data in field.getValues(encoded=False)
]
if len(fieldValues) == 0:
raise Exception("No value found in the field.")
# Retrieve longuest field value
maxLengthFieldValue = len(max(fieldValues, key=len))
# definies the step following specified unitsize
stepUnitsize = self.__computeStepForUnitsize()
# Vertical identification of variation
indexedValues = []
for i in range(0, maxLengthFieldValue, stepUnitsize):
currentIndexValue = []
for fieldValue in fieldValues:
if i < len(fieldValue):
currentIndexValue.append(
fieldValue[i:min(len(fieldValue), i + stepUnitsize)])
else:
currentIndexValue.append(b'')
indexedValues.append(currentIndexValue)
# If requested, merges the adjacent static fields
if self.mergeAdjacentStaticFields:
result = []
staticSequences = []
for values in indexedValues:
if len(set(values)) == 1:
# static
staticSequences.append(values[0])
else:
# dynamic
if len(staticSequences) > 0:
result.append([b''.join(staticSequences)])
staticSequences = []
result.append(values)
if len(staticSequences) > 0:
result.append([b''.join(staticSequences)])
indexedValues = result
# If requested, merges the adjacent dynamic fields
if self.mergeAdjacentDynamicFields:
result = []
dynamicSequences = []
for values in indexedValues:
if len(set(values)) > 1:
# dynamic
dynamicSequences.append(values)
else:
# static
if len(dynamicSequences) > 0:
dynValues = zip(*dynamicSequences)
tmp_result = []
for d in dynValues:
tmp_result.append(b''.join(
[x if x is not None else b'' for x in d]))
result.append(tmp_result)
dynamicSequences = []
result.append(values)
if len(dynamicSequences) > 0:
dynValues = zip(*dynamicSequences)
tmp_result = []
for d in dynValues:
tmp_result.append(b''.join(
[x if x is not None else b'' for x in d]))
result.append(tmp_result)
indexedValues = result
# Create a field for each entry
newFields = []
for (i, val) in enumerate(indexedValues):
fName = "Field-{0}".format(i)
fDomain = DomainFactory.normalizeDomain([
Raw(TypeConverter.convert(v, HexaString, BitArray))
for v in set(val)
])
newFields.append(Field(domain=fDomain, name=fName))
# attach encoding functions
for newField in newFields:
newField.encodingFunctions = list(field.encodingFunctions.values())
field.fields = newFields
def initHeader(self):
"""Initialize the IP header according to the IP format definition.
"""
# Ethernet header
# Retrieve remote MAC address
dstMacAddr = get_mac_address(ip=self.remoteIP)
if dstMacAddr is not None:
dstMacAddr = dstMacAddr.replace(':', '')
dstMacAddr = binascii.unhexlify(dstMacAddr)
else:
# Force ARP resolution
p = subprocess.Popen(["/bin/ping", "-c1", self.remoteIP])
p.wait()
time.sleep(0.1)
dstMacAddr = get_mac_address(ip=self.remoteIP)
if dstMacAddr is not None:
dstMacAddr = dstMacAddr.replace(':', '')
dstMacAddr = binascii.unhexlify(dstMacAddr)
else:
raise Exception(
"Cannot resolve IP address to a MAC address for IP: '{}'".
format(self.remoteIP))
# Retrieve local MAC address
srcMacAddr = self.get_interface_addr(bytes(self.interface, 'utf-8'))[1]
eth_dst = Field(name='eth.dst', domain=Raw(dstMacAddr))
eth_src = Field(name='eth.src', domain=Raw(srcMacAddr))
eth_type = Field(name='eth.type', domain=Raw(b"\x08\x00"))
# IP header
ip_ver = Field(name='ip.version',
domain=BitArray(value=bitarray('0100'))) # IP Version 4
ip_ihl = Field(name='ip.hdr_len', domain=BitArray(bitarray('0000')))
ip_tos = Field(name='ip.tos',
domain=Data(dataType=BitArray(nbBits=8),
originalValue=bitarray('00000000'),
svas=SVAS.PERSISTENT))
ip_tot_len = Field(name='ip.len',
domain=BitArray(bitarray('0000000000000000')))
ip_id = Field(name='ip.id', domain=BitArray(nbBits=16))
ip_flags = Field(name='ip.flags',
domain=Data(dataType=BitArray(nbBits=3),
originalValue=bitarray('000'),
svas=SVAS.PERSISTENT))
ip_frag_off = Field(name='ip.fragment',
domain=Data(
dataType=BitArray(nbBits=13),
originalValue=bitarray('0000000000000'),
svas=SVAS.PERSISTENT))
ip_ttl = Field(name='ip.ttl',
domain=Data(dataType=BitArray(nbBits=8),
originalValue=bitarray('01000000'),
svas=SVAS.PERSISTENT))
ip_proto = Field(name='ip.proto',
domain=Integer(value=self.upperProtocol,
unitSize=AbstractType.UNITSIZE_8,
endianness=AbstractType.ENDIAN_BIG,
sign=AbstractType.SIGN_UNSIGNED))
ip_checksum = Field(name='ip.checksum',
domain=BitArray(bitarray('0000000000000000')))
ip_saddr = Field(name='ip.src', domain=IPv4(self.localIP))
ip_daddr = Field(name='ip.dst', domain=IPv4(self.remoteIP))
ip_payload = Field(name='ip.payload', domain=Raw())
ip_ihl.domain = Size([
ip_ver, ip_ihl, ip_tos, ip_tot_len, ip_id, ip_flags, ip_frag_off,
ip_ttl, ip_proto, ip_checksum, ip_saddr, ip_daddr
],
dataType=BitArray(nbBits=4),
factor=1 / float(32))
ip_tot_len.domain = Size([
ip_ver, ip_ihl, ip_tos, ip_tot_len, ip_id, ip_flags, ip_frag_off,
ip_ttl, ip_proto, ip_checksum, ip_saddr, ip_daddr, ip_payload
],
dataType=Integer(
unitSize=AbstractType.UNITSIZE_16,
sign=AbstractType.SIGN_UNSIGNED),
factor=1 / float(8))
ip_checksum.domain = InternetChecksum(
fields=[
ip_ver, ip_ihl, ip_tos, ip_tot_len, ip_id, ip_flags,
ip_frag_off, ip_ttl, ip_proto, ip_checksum, ip_saddr, ip_daddr
],
dataType=Raw(nbBytes=2, unitSize=AbstractType.UNITSIZE_16))
self.header = Symbol(name='Ethernet layer',
fields=[
eth_dst, eth_src, eth_type, ip_ver, ip_ihl,
ip_tos, ip_tot_len, ip_id, ip_flags,
ip_frag_off, ip_ttl, ip_proto, ip_checksum,
ip_saddr, ip_daddr, ip_payload
])
def __create_fields(self, symbol, results):
size_of_size = 1
for result in results:
field_to_split, maxSize, indexInField = self.__getFieldFromIndex(
result[0], symbol)
self._logger.warning("[Field to split] : " + field_to_split.name +
"\n")
first_field_dep, maxSizefd, indexInFieldfd = self.__getFieldFromIndex(
result[1], symbol)
if indexInFieldfd != 0:
#Create a subfield in the first_field_dependency
values_first_field_dep = first_field_dep.getValues()
if len(set(values_first_field_dep)) > 1:
#Alt field
values_first_field_dep_before = []
values_first_field_dep_after = []
for value in values_first_field_dep:
values_first_field_dep_before.append(
value[:int(indexInFieldfd / 8)])
values_first_field_dep_after.append(
value[int(indexInFieldfd / 8):])
first_field_dep_before = Field(
domain=Alt(values_first_field_dep_before),
name=first_field_dep.name + "-0")
first_field_dep_after = Field(
domain=Alt(values_first_field_dep_after),
name=first_field_dep.name + "-1")
else:
#Static field
value = values_first_field_dep[0]
value_first_field_dep_before = value[:indexInFieldfd]
value_first_field_dep_after = value[indexInFieldfd:]
first_field_dep_before = Field(
domain=Raw(value_first_field_dep_before),
name=first_field_dep.name + "-0")
first_field_dep_after = Field(
domain=Raw(value_first_field_dep_after),
name=first_field_dep.name + "-1")
first_field_dep.fields = [
first_field_dep_before, first_field_dep_after
]
self._logger.warning("[First field dependency] : " +
first_field_dep.name + "\n")
field_dep = self.__get_field_dep(symbol, first_field_dep)
#Check if static or Alt:
values_before = []
values_after = []
field_to_split_values = field_to_split.getValues()
if len(set(field_to_split_values)) > 1:
#Alt field
for value in field_to_split_values:
values_before.append(value[:int(indexInField / 8)])
values_after.append(value[int(indexInField / 8) +
size_of_size:])
#AltField before =>
altField_before_Size = Field(domain=Alt(values_before),
name=field_to_split.name +
"-Before_SizeF")
altField_after_Size = Field(domain=Alt(values_after),
name=field_to_split.name +
"-After_Size")
sizeField = Field(domain=Size(field_dep),
name=field_to_split.name + "-Size")
field_to_split.fields = [
altField_before_Size, sizeField, altField_after_Size
]
else:
#Static field
value_before = field_to_split_values[0][:int(indexInField / 8)]
value_after = field_to_split_values[0][int(indexInField / 8) +
size_of_size:]
staticField_before_Size = Field(domain=Raw(value_before),
name=field_to_split.name +
"-Before_SizeF")
staticField_after_Size = Field(domain=Raw(value_after),
name=field_to_split.name +
"-After_Size")
sizeField = Field(domain=Size(field_dep),
name=field_to_split.name + "-Size")
field_to_split.fields = [
staticField_before_Size, sizeField, staticField_after_Size
]
def split(field, delimiter):
"""Split a field (or symbol) with a specific delimiter. The
delimiter can be passed either as an ASCII, a Raw, an
HexaString, or any objects that inherit from AbstractType.
>>> from netzob.all import *
>>> samples = [b"aaaaff000000ff10", b"bbff110010ff00000011", b"ccccccccfffe1f000000ff12"]
>>> messages = [RawMessage(data=sample) for sample in samples]
>>> symbol = Symbol(messages=messages[:3])
>>> Format.splitDelimiter(symbol, ASCII("ff"))
>>> print(symbol)
Field-0 | Field-sep-6666 | Field-2 | Field-sep-6666 | Field-4
---------- | -------------- | ------------ | -------------- | ----------
'aaaa' | 'ff' | '000000' | 'ff' | '10'
'bb' | 'ff' | '110010' | 'ff' | '00000011'
'cccccccc' | 'ff' | 'fe1f000000' | 'ff' | '12'
---------- | -------------- | ------------ | -------------- | ----------
>>> samples = [b"434d446964656e74696679230400000066726564", b"5245536964656e74696679230000000000000000", b"434d44696e666f2300000000", b"524553696e666f230000000004000000696e666f", b"434d4473746174732300000000", b"52455373746174732300000000050000007374617473", b"434d4461757468656e7469667923090000006d7950617373776421", b"52455361757468656e74696679230000000000000000", b"434d44656e6372797074230a00000031323334353674657374", b"524553656e637279707423000000000a00000073707176777436273136", b"434d4464656372797074230a00000073707176777436273136", b"5245536465637279707423000000000a00000031323334353674657374", b"434d446279652300000000", b"524553627965230000000000000000", b"434d446964656e746966792307000000526f626572746f", b"5245536964656e74696679230000000000000000", b"434d44696e666f2300000000", b"524553696e666f230000000004000000696e666f", b"434d4473746174732300000000", b"52455373746174732300000000050000007374617473", b"434d4461757468656e74696679230a000000615374726f6e67507764", b"52455361757468656e74696679230000000000000000", b"434d44656e63727970742306000000616263646566", b"524553656e6372797074230000000006000000232021262724", b"434d44646563727970742306000000232021262724", b"52455364656372797074230000000006000000616263646566", b"434d446279652300000000", b"524553627965230000000000000000"]
>>> messages = [RawMessage(data=TypeConverter.convert(sample, HexaString, Raw)) for sample in samples]
>>> symbol = Symbol(messages=messages)
>>> symbol.encodingFunctions.add(TypeEncodingFunction(ASCII)) # Change visualization to hexastring
>>> Format.splitDelimiter(symbol, ASCII("#"))
>>> print(symbol)
Field-0 | Field-sep-23 | Field-2 | Field-sep-23 | Field-4
--------------- | ------------ | -------------------- | ------------ | -------
'CMDidentify' | '#' | '....fred' | '' | ''
'RESidentify' | '#' | '........' | '' | ''
'CMDinfo' | '#' | '....' | '' | ''
'RESinfo' | '#' | '........info' | '' | ''
'CMDstats' | '#' | '....' | '' | ''
'RESstats' | '#' | '........stats' | '' | ''
'CMDauthentify' | '#' | '....myPasswd!' | '' | ''
'RESauthentify' | '#' | '........' | '' | ''
'CMDencrypt' | '#' | '....123456test' | '' | ''
'RESencrypt' | '#' | "........spqvwt6'16" | '' | ''
'CMDdecrypt' | '#' | "....spqvwt6'16" | '' | ''
'RESdecrypt' | '#' | '........123456test' | '' | ''
'CMDbye' | '#' | '....' | '' | ''
'RESbye' | '#' | '........' | '' | ''
'CMDidentify' | '#' | '....Roberto' | '' | ''
'RESidentify' | '#' | '........' | '' | ''
'CMDinfo' | '#' | '....' | '' | ''
'RESinfo' | '#' | '........info' | '' | ''
'CMDstats' | '#' | '....' | '' | ''
'RESstats' | '#' | '........stats' | '' | ''
'CMDauthentify' | '#' | '....aStrongPwd' | '' | ''
'RESauthentify' | '#' | '........' | '' | ''
'CMDencrypt' | '#' | '....abcdef' | '' | ''
'RESencrypt' | '#' | '........' | '#' | " !&'$"
'CMDdecrypt' | '#' | '....' | '#' | " !&'$"
'RESdecrypt' | '#' | '........abcdef' | '' | ''
'CMDbye' | '#' | '....' | '' | ''
'RESbye' | '#' | '........' | '' | ''
--------------- | ------------ | -------------------- | ------------ | -------
>>> print(symbol.fields[0]._str_debug())
Field-0
|-- Alt
|-- Data (Raw=b'CMDidentify' ((0, 88)))
|-- Data (Raw=b'RESidentify' ((0, 88)))
|-- Data (Raw=b'CMDinfo' ((0, 56)))
|-- Data (Raw=b'RESinfo' ((0, 56)))
|-- Data (Raw=b'CMDstats' ((0, 64)))
|-- Data (Raw=b'RESstats' ((0, 64)))
|-- Data (Raw=b'CMDauthentify' ((0, 104)))
|-- Data (Raw=b'RESauthentify' ((0, 104)))
|-- Data (Raw=b'CMDencrypt' ((0, 80)))
|-- Data (Raw=b'RESencrypt' ((0, 80)))
|-- Data (Raw=b'CMDdecrypt' ((0, 80)))
|-- Data (Raw=b'RESdecrypt' ((0, 80)))
|-- Data (Raw=b'CMDbye' ((0, 48)))
|-- Data (Raw=b'RESbye' ((0, 48)))
Below is another example of the FieldSplitDelimiter usage: it splits fields based on a Raw string.
>>> from netzob.all import *
>>> samples = [b"\\x01\\x02\\x03\\xff\\x04\\x05\\xff\\x06\\x07", b"\\x01\\x02\\xff\\x03\\x04\\x05\\x06\\xff\\x07", b"\\x01\\xff\\x02\\x03\\x04\\x05\\x06"]
>>> messages = [RawMessage(data=sample) for sample in samples]
>>> symbol = Symbol(messages=messages)
>>> Format.splitDelimiter(symbol, Raw(b"\\xff"))
>>> print(symbol)
Field-0 | Field-sep-ff | Field-2 | Field-sep-ff | Field-4
-------------- | ------------ | ---------------------- | ------------ | ----------
'\\x01\\x02\\x03' | b'\\xff' | '\\x04\\x05' | b'\\xff' | '\\x06\\x07'
'\\x01\\x02' | b'\\xff' | '\\x03\\x04\\x05\\x06' | b'\\xff' | '\\x07'
'\\x01' | b'\\xff' | '\\x02\\x03\\x04\\x05\\x06' | '' | ''
-------------- | ------------ | ---------------------- | ------------ | ----------
:param field : the field to consider when spliting
:type: :class:`netzob.Model.Vocabulary.AbstractField.AbstractField`
:param delimiter : the delimiter used to split messages of the field
:type: :class:`netzob.Model.Vocabulary.Types.AbstractType.AbstractType`
"""
if delimiter is None:
raise TypeError("Delimiter cannot be None.")
if field is None:
raise TypeError("Field cannot be None.")
if len(field.messages) < 1:
raise ValueError(
"The associated symbol does not contain any message.")
# Find message substrings after applying delimiter
splittedMessages = []
for cell in field.getValues(encoded=False, styled=False):
splittedMessage = cell.split(delimiter.value.tobytes())
splittedMessages.append(splittedMessage)
import itertools
# Inverse the array, so that columns contains observed values for each field
splittedMessages = list(itertools.zip_longest(*splittedMessages))
# If the delimiter does not create splitted fields
if len(splittedMessages) <= 1:
return
# Else, we add (2*len(splittedMessages)-1) fields
newFields = []
iField = -1
for i in range(len(splittedMessages)):
iField += 1
fieldDomain = list()
# temporary set that hosts all the observed values to prevent useless duplicate ones
observedValues = set()
has_inserted_empty_value = False
isEmptyField = True # To avoid adding an empty field
for v in splittedMessages[i]:
if v != "" and v is not None:
isEmptyField = False
if v not in observedValues:
fieldDomain.append(Raw(v))
observedValues.add(v)
else:
if not has_inserted_empty_value:
fieldDomain.append(Raw(nbBytes=0))
has_inserted_empty_value = True
if not isEmptyField:
newField = Field(
domain=DomainFactory.normalizeDomain(fieldDomain),
name="Field-" + str(iField))
newField.encodingFunctions = list(
field.encodingFunctions.values())
newFields.append(newField)
iField += 1
str_delimiter = TypeConverter.convert(delimiter.value, BitArray,
HexaString).decode('utf-8')
fieldName = "Field-sep-{}".format(str_delimiter)
newFields.append(
Field(domain=Alt([delimiter, Raw(nbBytes=0)]), name=fieldName))
newFields.pop()
# Reset the field
from netzob.Inference.Vocabulary.Format import Format
Format.resetFormat(field)
# Create a field for each entry
field.fields = newFields
def __automate_mid_field_creation(self, results, symbol, endianness):
"""
This method attempts to create a CRC field. It creates either a big or little endian CRC relation field using
the value b'\xca\xfe\x00\x00\x00\x00\xba\xbe' to compute the CRC
, the b'\x00\x00\x00\x00' is then replaced by the computed CRC in the message.
:param results: list of CRC indexes
:param symbol: The symbol in wich we want to define CRC fields
:param endianness: A string stating the endianness of the CRC, should either be "big" or "little"
"""
domain_field_list = []
subfields = []
for crcindex in results:
searchedfield, field_size, field_index = self.__getFieldFromIndex(crcindex, symbol)
# Get all the fields after the current field (to define the CRC domain Relation
afterField = False
if field_size / 8 == 4:
for i,f in enumerate(symbol.fields):
if f.name == searchedfield.name:
domain_field_list.append(symbol.fields[i-1])
domain_field_list.append(symbol.fields[i+1])
# Is the field just CRC or more?
if endianness == 'little':
searchedfield.name = "CRC32_mid_LE" + str(crcindex)
else:
searchedfield.name = "CRC32_mid_BE" + str(crcindex)
searchedfield.domain = CRC32(domain_field_list, endianness=endianness)
elif field_size > 4:
# The Field is the CRC and something else.
if field_index == 0:
# CRC at beginning of field. Create new fields after CRC
# Create a list of possible values
val_list = []
for val in searchedfield.getValues():
val_list.append(val[4:])
# Make a set
val_set = set(val_list)
newf = self.__define_field(val_set,searchedfield)
#Compute second field (the one before the CRC
val_list = [] #List containing the first four bytes used to compute CRC32 (the 4 bytes prior to the CRC)
temp_field_list = []
for prevalue in symbol.getValues():
prevalue = prevalue[crcindex - 4:crcindex]
val_list.append(prevalue)
val_list = set(val_list)
for i, f in enumerate(symbol.fields):
if f.name == searchedfield.name:
prevField,minSize,maxSize = self.__getFieldFromIndex(crcindex - 1,symbol)
if maxSize >= 4:
#The four bytes are contained in only one field. We create a subfield
newf2 = self.__define_field(val_list, temp)
else:
#The four bytes are split into several fields. We delete all thes fields and create a new field and insert it at the index of the first of these fields in symbol.fields()
for j in range(1,4):
try:
prevField, minSize, maxSize = self.__getFieldFromIndex(crcindex - j, symbol)
except:
pass
temp_field_list.append(prevField)
temp_field_set = set(temp_field_list) #All the fields containing the four bytes
for fiel in temp_field_set:
miny = 99999
stuffToDel = []
for y,fol in enumerate(symbol.fields):
if fol == fiel:
if y < miny:
miny = y
stuffToDel.append(y)
temp = symbol.fields[y]
newf2 = self.__define_field(val_list,temp)
domain_field_list.append(newf2)
domain_field_list.append(Field(domain=Raw(b'\x00\x00\x00\x00')))
domain_field_list.append(newf)
if endianness == 'little':
crc32field = Field(name="CRC32_mid_LE" + str(crcindex),
domain=CRC32(domain_field_list, endianness=endianness))
else:
crc32field = Field(name="CRC32_mid_BE" + str(crcindex),
domain=CRC32(domain_field_list, endianness=endianness))
subfields.append(crc32field)
subfields.append(newf)
elif field_index / 8 + 4 == field_size / 8:
# CRC at end of field. Create new fields before CRC
# Create a list of possible values
val_list = []
for val in searchedfield.getValues():
val_list.append(val[:field_index])
# Make a set
val_set = set(val_list)
newf = self.__define_field(val_set,searchedfield)
#Compute second field (the one after the CRC
val_list = [] #List containing the last four bytes used to compute CRC32 (the 4 bytes after the CRC)
temp_field_list = []
for prevalue in symbol.getValues():
prevalue = prevalue[crcindex:crcindex+4]
val_list.append(prevalue)
val_list = set(val_list)
for i, f in enumerate(symbol.fields):
if f.name == searchedfield.name:
prevField,minSize,maxSize = self.__getFieldFromIndex(crcindex + 1,symbol)
if maxSize >= 4:
#The four bytes are contained in only one field. We create a subfield
newf2 = self.__define_field(val_list,temp)
else:
#The four bytes are split into several fields. We delete all thes fields and create a new field and insert it at the index of the first of these fields in symbol.fields()
for j in range(1,4):
prevField, minSize, maxSize = self.__getFieldFromIndex(crcindex + j, symbol)
temp_field_list.append(prevField)
temp_field_list = set(temp_field_list) #All the fields containing the four bytes
for fiel in set(temp_field_list):
miny = 99999
for y,fol in enumerate(symbol.fields):
if fol == fiel:
if y < miny:
miny = y
temp = symbol.fields[y]
del symbol.fields[y]
symbol.fields[miny] = self.__define_field(val_list,temp)
newf2 = self.__define_field(val_list,temp)
domain_field_list.append(newf)
domain_field_list.append(Field(domain=Raw(b'\x00\x00\x00\x00')))
domain_field_list.append(newf2)
if endianness == 'little':
crc32field = Field(name="CRC32_mid_LE" + str(crcindex),
domain=CRC32(domain_field_list, endianness=endianness))
else:
crc32field = Field(name="CRC32_mid_BE" + str(crcindex),
domain=CRC32(domain_field_list, endianness=endianness))
subfields.append(newf)
subfields.append(crc32field)
else:
# CRC in the middle. Create new fields before and after CRC
val_list1 = []
val_list2 = []
for val in searchedfield.getValues():
val_list1.append(val[4:])
val_list2.append(val[:field_index])
# Make a set
val_set1 = set(val_list1)
val_set2 = set(val_list2)
newf1 = self.__define_field(val_set1,searchedfield)
newf2 = self.__define_field(val_set2,searchedfield)
domain_field_list.insert(0, newf2)
if endianness == 'little':
crc32field = Field(name="CRC32_LE" + str(crcindex),
domain=CRC32(domain_field_list, endianness=endianness))
else:
crc32field = Field(name="CRC32_BE" + str(crcindex),
domain=CRC32(domain_field_list, endianness=endianness))
subfields.append(newf1)
subfields.append(crc32field)
subfields.append(newf2)
searchedfield.fields = subfields
def initHeader(self):
"""Initialize the IP header according to the IP format definition.
"""
ip_ver = Field(name='ip.version',
domain=BitArray(value=bitarray('0100'))) # IP Version 4
ip_ihl = Field(name='ip.hdr_len', domain=BitArray(bitarray('0000')))
ip_tos = Field(name='ip.tos',
domain=Data(dataType=BitArray(nbBits=8),
originalValue=bitarray('00000000'),
svas=SVAS.PERSISTENT))
ip_tot_len = Field(name='ip.len',
domain=BitArray(bitarray('0000000000000000')))
ip_id = Field(name='ip.id', domain=BitArray(nbBits=16))
ip_flags = Field(name='ip.flags',
domain=Data(dataType=BitArray(nbBits=3),
originalValue=bitarray('000'),
svas=SVAS.PERSISTENT))
ip_frag_off = Field(name='ip.fragment',
domain=Data(
dataType=BitArray(nbBits=13),
originalValue=bitarray('0000000000000'),
svas=SVAS.PERSISTENT))
ip_ttl = Field(name='ip.ttl',
domain=Data(dataType=BitArray(nbBits=8),
originalValue=bitarray('01000000'),
svas=SVAS.PERSISTENT))
ip_proto = Field(name='ip.proto',
domain=Integer(value=self.upperProtocol,
unitSize=AbstractType.UNITSIZE_8,
endianness=AbstractType.ENDIAN_BIG,
sign=AbstractType.SIGN_UNSIGNED))
ip_checksum = Field(name='ip.checksum',
domain=BitArray(bitarray('0000000000000000')))
ip_saddr = Field(name='ip.src', domain=IPv4(self.localIP))
ip_daddr = Field(name='ip.dst', domain=IPv4(self.remoteIP))
ip_payload = Field(name='ip.payload', domain=Raw())
ip_ihl.domain = Size([
ip_ver, ip_ihl, ip_tos, ip_tot_len, ip_id, ip_flags, ip_frag_off,
ip_ttl, ip_proto, ip_checksum, ip_saddr, ip_daddr
],
dataType=BitArray(nbBits=4),
factor=1 / float(32))
ip_tot_len.domain = Size([
ip_ver, ip_ihl, ip_tos, ip_tot_len, ip_id, ip_flags, ip_frag_off,
ip_ttl, ip_proto, ip_checksum, ip_saddr, ip_daddr, ip_payload
],
dataType=Integer(
unitSize=AbstractType.UNITSIZE_16,
sign=AbstractType.SIGN_UNSIGNED),
factor=1 / float(8))
ip_checksum.domain = InternetChecksum(
fields=[
ip_ver, ip_ihl, ip_tos, ip_tot_len, ip_id, ip_flags,
ip_frag_off, ip_ttl, ip_proto, ip_checksum, ip_saddr, ip_daddr
],
dataType=Raw(nbBytes=2, unitSize=AbstractType.UNITSIZE_16))
self.header = Symbol(name='IP layer',
fields=[
ip_ver, ip_ihl, ip_tos, ip_tot_len, ip_id,
ip_flags, ip_frag_off, ip_ttl, ip_proto,
ip_checksum, ip_saddr, ip_daddr, ip_payload
])
def create_type_raw(self, value=None, nb_byte_min=None, nb_byte_max=None):
data_type = Raw(value=value, nbBytes=(nb_byte_min, nb_byte_max))
self.__datatypes[str(data_type.id)] = data_type
return data_type