def merge_nontest_fields(self, fields_origin, fid):
logging.debug("[+] Merge Fields")
fields_merged = list()
fields = copy.deepcopy(fields_origin)
fsize_total = 0
for i in range(len(fields)):
typename = fields[i].domain.dataType.typeName
if typename != "Raw":
logging.error("Field type is not Raw")
typesize = fields[i].domain.dataType.size[1]
# print(i, fid, typesize, type(typesize))
if i == fid:
if fsize_total > 0:
field = Field(Raw(nbBytes=fsize_total // 8))
fields_merged.append(field)
fsize_total = 0
if fid != (len(fields) - 1):
field = Field(Raw(nbBytes=typesize // 8))
fields_merged.append(field)
field = Field()
fields_merged.append(field)
else:
if type(typesize).__name__ == 'NoneType':
field = Field()
else:
field = Field(Raw(nbBytes=typesize // 8))
fields_merged.append(field)
break
else:
fsize_total += typesize
return fields_merged
def __define_field(self,val_set,field):
"""
:param val_set: A set of values in the field
:param field: The field containing the CRC
:return: new_field: A new field (Alt or Raw)
"""
if len(val_set) > 1:
# More than one value, create Alt Fields
domain_list = []
for value in val_set:
domain_list.append(Raw(value))
new_field = Field(name=field.name, domain=Alt(domain_list))
else:
# Only one value, create static Fields
new_field = Field(name=field.name, domain=Raw(val_set.pop()))
return new_field
def generate_fields(self, fields_result):
fields = list()
for typeinfo in fields_result:
if typeinfo[0] == "Raw":
field = Field(Raw(nbBytes=(typeinfo[1] // 8,
typeinfo[2] // 8)))
fields.append(field)
else:
logging.error("Field type is not Raw")
return fields
def _splitFieldFollowingAlignment(self, field, align):
"""Update the field definition with new fields following the
specified align."""
# STEP 1 : Create a field separation based on static and dynamic fields
leftAlign, rightAlign = self._splitAlignment(align)
splited = self._mergeAlign(leftAlign, rightAlign)
step1Fields = []
for (entryVal, entryDyn) in splited:
if entryDyn:
newField = Field(Raw(nbBytes=(0, int(len(entryVal) / 2))))
else:
newField = Field(
Raw(TypeConverter.convert(entryVal, HexaString, Raw)))
step1Fields.append(newField)
for f in step1Fields:
f.encodingFunctions = list(field.encodingFunctions.values())
field.fields = step1Fields
def __init__(self, fields=None, messages=None, name="Symbol", meta=False):
"""
:keyword fields: the fields which participate in symbol definition
:type fields: a :class:`list` of :class:`netzob.Model.Vocabulary.Field`
:keyword messages: the message that represent the symbol
:type messages: a :class:`list` of :class:`netzob.Model.Vocabulary.Messages.AbstractMessage.AbstractMessage`
:keyword name: the name of the symbol
:type name: :class:`str`
"""
super(Symbol, self).__init__(name, meta)
self.__messages = TypedList(AbstractMessage)
if messages is None:
messages = []
self.messages = messages
if fields is None:
# create a default empty field
fields = [Field()]
self.fields = fields
def createSubFieldsForAStaticField(self, field, align, semanticTags):
"""createSubFieldsForAStaticField:
Analyzes the static field provided and create sub fields following
the provided semantic tags."""
self._logger.debug(
"Create subfields for static field {0} : {1}".format(
field.getName(), align))
if len(field.getLocalFields()) > 0:
self._logger.warning(
"Impossible to create sub fields for this field since its not cleaned"
)
return
subFields = []
currentTag = None
currentTagLength = 0
for index, tag in list(semanticTags.items()):
if tag != currentTag:
# Create a sub field
subFieldValue = align[index - currentTagLength:index]
if len(subFieldValue) > 0:
subFields.append(subFieldValue)
currentTagLength = 0
currentTag = tag
currentTagLength += 1
if currentTagLength > 0:
subFieldValue = align[-currentTagLength:]
if len(subFieldValue) > 0:
subFields.append(subFieldValue)
if len(subFields) > 1:
for iSubField, subFieldValue in enumerate(subFields):
subField = Field(b"{0}_{1}".format(field.getName(), iSubField),
b"({0})".format(subFieldValue),
field.getSymbol())
field.addLocalField(subField)
def reset(self, field):
"""Resets the format (field hierarchy and definition domain) of
the specified field.
:param field: the field we want to reset
:type field: :class:`netzob.Model.Vocabulary.AbstractField.AbstractField`
:raise Exception if something bad happens
"""
if field is None:
raise TypeError(
"The field to reset must be specified and cannot be None")
self._logger.debug("Reset the definition of field {0} ({1})".format(
field.name, field.id))
field.clearFields()
if isinstance(field, Symbol):
field.fields = [Field()]
if isinstance(field, Field):
field.domain = Raw(None)
def mergeFields(self, field1, field2):
"""Merge specified fields.
>>> import binascii
>>> from netzob.all import *
>>> samples = ["00ff2f000000", "000010000000", "00fe1f000000"]
>>> messages = [RawMessage(data=binascii.unhexlify(sample)) for sample in samples]
>>> f1 = Field(Raw(nbBytes=1), name="f1")
>>> f2 = Field(Raw(nbBytes=2), name="f2")
>>> f3 = Field(Raw(nbBytes=2), name="f3")
>>> f4 = Field(Raw(nbBytes=1), name="f4")
>>> symbol = Symbol([f1, f2, f3, f4], messages=messages)
>>> symbol.addEncodingFunction(TypeEncodingFunction(HexaString))
>>> print(symbol)
f1 | f2 | f3 | f4
---- | ------ | ------ | ----
'00' | 'ff2f' | '0000' | '00'
'00' | '0010' | '0000' | '00'
'00' | 'fe1f' | '0000' | '00'
---- | ------ | ------ | ----
>>> fo = FieldOperations()
>>> fo.mergeFields(f2, f3)
>>> print(symbol)
f1 | Merge | f4
---- | ---------- | ----
'00' | 'ff2f0000' | '00'
'00' | '00100000' | '00'
'00' | 'fe1f0000' | '00'
---- | ---------- | ----
>>> fo.mergeFields(symbol.fields[0], symbol.fields[1])
>>> print(symbol)
Merge | f4
------------ | ----
'00ff2f0000' | '00'
'0000100000' | '00'
'00fe1f0000' | '00'
------------ | ----
>>> fo.mergeFields(symbol.fields[0], symbol.fields[1])
>>> print(symbol)
Merge
--------------
'00ff2f000000'
'000010000000'
'00fe1f000000'
--------------
:param field1: the left field to merge
:type field1: :class:`netzob.Model.Vocabulary.AbstractField.AbstractField`
:param field2: the right field to merge
:type field2: :class:`netzob.Model.Vocabulary.AbstractField.AbstractField`
:raise Exception if something bad happens
"""
if field1 is None or field2 is None:
raise TypeError("Fields cannot be None")
if field1 == field2:
raise ValueError("Cannot merge a unique field (field1 == field2)")
self._logger.debug("Merging field {0} with field {1}".format(field1.name, field2.name))
if field1.parent is not field2.parent:
raise ValueError("Specified fields don't have the same parent, only fields with same parents can be merged.")
# retrieve indexes of specified fields
iField1 = None
iField2 = None
for iField, field in enumerate(field1.parent.fields):
if field == field1:
iField1 = iField
elif field == field2:
iField2 = iField
if iField1 is None:
raise ValueError("Cannot retrieve position of field1 in its parent fields")
if iField2 is None:
raise ValueError("Cannot retrieve position of field2 in its parent fields")
if iField2 != iField1 + 1:
raise ValueError("Field1 must be directly on the left of field2 (iField1={0}, iField2={1})".format(iField1, iField2))
# build a new field domain
newDomain = Agg([field1.domain, field2.domain])
newField = Field(domain=newDomain, name="Merge")
newField.encodingFunctions = list(field1.encodingFunctions.values())
parent = field1.parent
before = parent.fields[:iField1]
after = parent.fields[iField2 + 1:]
parent.fields = before + [newField] + after
def _blendFields(field1, field2):
"""
Completely blend two fields without using Agg.
This requires that the two fields are of the same domain and its parameters.
The returned new field still needs to be placed into the symbol.
>>> from netzob.all import *
>>> samples = [ b'\\x00\\xff/BPz', b'\\x00\\x00 CQ~', b'\\x00\\xff/Gf/' ]
>>> messages = [RawMessage(data=sample) for sample in samples]
>>> f1 = Field(Data(Integer(unitSize=AbstractType.UNITSIZE_8)))
>>> f2 = Field(Raw(nbBytes=2))
>>> f3 = Field(Raw(nbBytes=3))
>>> symbol = Symbol([f1, f2, f3], messages=messages)
>>> print(f2.domain.dataType.size)
(16, 16)
>>> print(f3.domain.dataType.size)
(24, 24)
>>> nf = FieldOperations._blendFields(f2,f3)
>>> print(nf.domain.currentValue)
None
>>> print(nf.domain.dataType.endianness)
big
>>> print(nf.domain.dataType.size)
(40, 40)
>>> nf2 = FieldOperations._blendFields(f1,f2)
Traceback (most recent call last):
...
NotImplementedError: The datatype Integer is not yet supported.
:param field1: the left field
:param field2: the right field
:return: a combination of field1 and field2
:raises TypeError: if domains are not compatible
:raises NotImplementedError: for datatypes which still need to be implemented
"""
for d in [field1.domain, field2.domain]:
if len(d._AbstractVariable__boundedVariables) > 0 \
or len(d._AbstractVariable__fathers) > 0 \
or len(d._AbstractVariable__tokenChoppedIndexes) > 0:
raise TypeError(
"Blending does not support __boundedVariables, __fathers, or __tokenChoppedIndexes."
)
if not isinstance(d.dataType, Raw):
raise NotImplementedError(
"The datatype {} is not yet supported.".format(
d.dataType.typeName))
if not field1.domain.svas == field2.domain.svas:
raise TypeError(
"The SVAS-values of both fields to merge are not the same.")
if not field1.domain.dataType.endianness == field2.domain.dataType.endianness:
raise TypeError(
"The endianness of both fields to merge are not the same.")
if not field1.domain.dataType.sign == field2.domain.dataType.sign:
raise TypeError(
"The signedness of both fields to merge are not the same.")
if not field1.domain.dataType.unitSize == field2.domain.dataType.unitSize:
raise TypeError(
"The unitSize of both fields to merge are not the same.")
# '_ASCII__nbChars': (None, None)
newDomain = field1.domain.__class__(field1.domain.dataType.__class__())
newDomain.svas = field1.domain.svas
newDomain.dataType.endianness = field1.domain.dataType.endianness
newDomain.dataType.sign = field1.domain.dataType.sign
newDomain.dataType.unitSize = field1.domain.dataType.unitSize
newDomain.dataType.size = field1.domain.dataType.size
minsizes = []
maxsizes = []
for f in [field1, field2]:
if not f.domain.dataType.size is None:
sizes = []
for s in f.domain.dataType.size:
if s is not None:
sizes.append(s)
minsizes.append(min(sizes))
maxsizes.append(max(sizes))
newDomain.dataType.size = (sum(minsizes), sum(maxsizes))
newField = Field(domain=newDomain, name="Merge")
newField.encodingFunctions = list(field1.encodingFunctions.values())
# Position the new field in correct positions with correct dataType size
if field1.domain.currentValue is None or field2.domain.currentValue is None:
newField.domain.currentValue = None
else:
newField.domain.currentValue = (field1.domain.currentValue +
field2.domain.currentValue)
return newField
def buildPacket(self, payload):
"""Build a raw IP packet including the IP layer and its payload.
:parameter payload: the payload to write on the channel
:type payload: binary object
"""
ip_ver = Field(name='Version', domain=BitArray(value=bitarray('0100'))) # IP Version 4
ip_ihl = Field(name='Header length', domain=BitArray(bitarray('0000')))
ip_tos = Field(name='TOS', domain=Data(dataType=BitArray(nbBits=8), originalValue=bitarray('00000000'), svas=SVAS.PERSISTENT))
ip_tot_len = Field(name='Total length', domain=BitArray(bitarray('0000000000000000')))
ip_id = Field(name='Identification number', domain=BitArray(nbBits=16))
ip_flags = Field(name='Flags', domain=Data(dataType=BitArray(nbBits=3), originalValue=bitarray('000'), svas=SVAS.PERSISTENT))
ip_frag_off = Field(name='Fragment offset', domain=Data(dataType=BitArray(nbBits=13), originalValue=bitarray('0000000000000'), svas=SVAS.PERSISTENT))
ip_ttl = Field(name='TTL', domain=Data(dataType=BitArray(nbBits=8), originalValue=bitarray('10000000'), svas=SVAS.PERSISTENT))
ip_proto = Field(name='Protocol', domain=Integer(value=self.upperProtocol, unitSize=AbstractType.UNITSIZE_8, endianness=AbstractType.ENDIAN_BIG, sign=AbstractType.SIGN_UNSIGNED))
ip_checksum = Field(name='Checksum', domain=BitArray(bitarray('0000000000000000')))
ip_saddr = Field(name='Source address', domain=IPv4(self.localIP))
ip_daddr = Field(name='Destination address', domain=IPv4(self.remoteIP))
ip_payload = Field(name='Payload', domain=payload)
ip_ihl.domain = Size([ip_ver,
ip_ihl,
ip_tos,
ip_tot_len,
ip_id, ip_flags,
ip_frag_off,
ip_ttl, ip_proto,
ip_checksum,
ip_saddr,
ip_daddr], dataType=BitArray(nbBits=4), factor=1/float(32))
ip_tot_len.domain = Size([ip_ver,
ip_ihl,
ip_tos,
ip_tot_len,
ip_id,
ip_flags,
ip_frag_off,
ip_ttl,
ip_proto,
ip_checksum,
ip_saddr,
ip_daddr,
ip_payload], dataType=Raw(nbBytes=2), factor=1/float(8))
ip_checksum.domain = InternetChecksum(fields=[ip_ver,
ip_ihl,
ip_tos,
ip_tot_len,
ip_id,
ip_flags,
ip_frag_off,
ip_ttl,
ip_proto,
ip_checksum,
ip_saddr,
ip_daddr], dataType=Raw(nbBytes=2))
packet = Symbol(name='IP layer', fields=[ip_ver,
ip_ihl,
ip_tos,
ip_tot_len,
ip_id,
ip_flags,
ip_frag_off,
ip_ttl,
ip_proto,
ip_checksum,
ip_saddr,
ip_daddr,
ip_payload])
return packet.specialize()
def initHeader(self):
"""Initialize the IP header according to the IP format definition.
"""
# Ethernet header
# Retrieve remote MAC address
dstMacAddr = get_mac_address(ip=self.remoteIP)
if dstMacAddr is not None:
dstMacAddr = dstMacAddr.replace(':', '')
dstMacAddr = binascii.unhexlify(dstMacAddr)
else:
# Force ARP resolution
p = subprocess.Popen(["/bin/ping", "-c1", self.remoteIP])
p.wait()
time.sleep(0.1)
dstMacAddr = get_mac_address(ip=self.remoteIP)
if dstMacAddr is not None:
dstMacAddr = dstMacAddr.replace(':', '')
dstMacAddr = binascii.unhexlify(dstMacAddr)
else:
raise Exception(
"Cannot resolve IP address to a MAC address for IP: '{}'".
format(self.remoteIP))
# Retrieve local MAC address
srcMacAddr = self.get_interface_addr(bytes(self.interface, 'utf-8'))[1]
eth_dst = Field(name='eth.dst', domain=Raw(dstMacAddr))
eth_src = Field(name='eth.src', domain=Raw(srcMacAddr))
eth_type = Field(name='eth.type', domain=Raw(b"\x08\x00"))
# IP header
ip_ver = Field(name='ip.version',
domain=BitArray(value=bitarray('0100'))) # IP Version 4
ip_ihl = Field(name='ip.hdr_len', domain=BitArray(bitarray('0000')))
ip_tos = Field(name='ip.tos',
domain=Data(dataType=BitArray(nbBits=8),
originalValue=bitarray('00000000'),
svas=SVAS.PERSISTENT))
ip_tot_len = Field(name='ip.len',
domain=BitArray(bitarray('0000000000000000')))
ip_id = Field(name='ip.id', domain=BitArray(nbBits=16))
ip_flags = Field(name='ip.flags',
domain=Data(dataType=BitArray(nbBits=3),
originalValue=bitarray('000'),
svas=SVAS.PERSISTENT))
ip_frag_off = Field(name='ip.fragment',
domain=Data(
dataType=BitArray(nbBits=13),
originalValue=bitarray('0000000000000'),
svas=SVAS.PERSISTENT))
ip_ttl = Field(name='ip.ttl',
domain=Data(dataType=BitArray(nbBits=8),
originalValue=bitarray('01000000'),
svas=SVAS.PERSISTENT))
ip_proto = Field(name='ip.proto',
domain=Integer(value=self.upperProtocol,
unitSize=AbstractType.UNITSIZE_8,
endianness=AbstractType.ENDIAN_BIG,
sign=AbstractType.SIGN_UNSIGNED))
ip_checksum = Field(name='ip.checksum',
domain=BitArray(bitarray('0000000000000000')))
ip_saddr = Field(name='ip.src', domain=IPv4(self.localIP))
ip_daddr = Field(name='ip.dst', domain=IPv4(self.remoteIP))
ip_payload = Field(name='ip.payload', domain=Raw())
ip_ihl.domain = Size([
ip_ver, ip_ihl, ip_tos, ip_tot_len, ip_id, ip_flags, ip_frag_off,
ip_ttl, ip_proto, ip_checksum, ip_saddr, ip_daddr
],
dataType=BitArray(nbBits=4),
factor=1 / float(32))
ip_tot_len.domain = Size([
ip_ver, ip_ihl, ip_tos, ip_tot_len, ip_id, ip_flags, ip_frag_off,
ip_ttl, ip_proto, ip_checksum, ip_saddr, ip_daddr, ip_payload
],
dataType=Integer(
unitSize=AbstractType.UNITSIZE_16,
sign=AbstractType.SIGN_UNSIGNED),
factor=1 / float(8))
ip_checksum.domain = InternetChecksum(
fields=[
ip_ver, ip_ihl, ip_tos, ip_tot_len, ip_id, ip_flags,
ip_frag_off, ip_ttl, ip_proto, ip_checksum, ip_saddr, ip_daddr
],
dataType=Raw(nbBytes=2, unitSize=AbstractType.UNITSIZE_16))
self.header = Symbol(name='Ethernet layer',
fields=[
eth_dst, eth_src, eth_type, ip_ver, ip_ihl,
ip_tos, ip_tot_len, ip_id, ip_flags,
ip_frag_off, ip_ttl, ip_proto, ip_checksum,
ip_saddr, ip_daddr, ip_payload
])
def initHeader(self):
"""Initialize the IP header according to the IP format definition.
"""
ip_ver = Field(
name='ip.version', domain=BitArray(
value=bitarray('0100'))) # IP Version 4
ip_ihl = Field(name='ip.hdr_len', domain=BitArray(bitarray('0000')))
ip_tos = Field(
name='ip.tos',
domain=Data(
dataType=BitArray(nbBits=8),
originalValue=bitarray('00000000'),
svas=SVAS.PERSISTENT))
ip_tot_len = Field(
name='ip.len', domain=BitArray(bitarray('0000000000000000')))
ip_id = Field(name='ip.id', domain=BitArray(nbBits=16))
ip_flags = Field(name='ip.flags', domain=Data(dataType=BitArray(nbBits=3), originalValue=bitarray('000'), svas=SVAS.PERSISTENT))
ip_frag_off = Field(name='ip.fragment', domain=Data(dataType=BitArray(nbBits=13), originalValue=bitarray('0000000000000'), svas=SVAS.PERSISTENT))
ip_ttl = Field(name='ip.ttl', domain=Data(dataType=BitArray(nbBits=8), originalValue=bitarray('01000000'), svas=SVAS.PERSISTENT))
ip_proto = Field(name='ip.proto', domain=Integer(value=self.upperProtocol, unitSize=AbstractType.UNITSIZE_8, endianness=AbstractType.ENDIAN_BIG, sign=AbstractType.SIGN_UNSIGNED))
ip_checksum = Field(name='ip.checksum', domain=BitArray(bitarray('0000000000000000')))
ip_saddr = Field(name='ip.src', domain=IPv4(self.localIP))
ip_daddr = Field(
name='ip.dst', domain=IPv4(self.remoteIP))
ip_payload = Field(name='ip.payload', domain=Raw())
ip_ihl.domain = Size([ip_ver,
ip_ihl,
ip_tos,
ip_tot_len,
ip_id, ip_flags,
ip_frag_off,
ip_ttl, ip_proto,
ip_checksum,
ip_saddr,
ip_daddr], dataType=BitArray(nbBits=4), factor=1/float(32))
ip_tot_len.domain = Size([ip_ver,
ip_ihl,
ip_tos,
ip_tot_len,
ip_id,
ip_flags,
ip_frag_off,
ip_ttl,
ip_proto,
ip_checksum,
ip_saddr,
ip_daddr,
ip_payload], dataType=Integer(unitSize=AbstractType.UNITSIZE_16, sign=AbstractType.SIGN_UNSIGNED), factor=1/float(8))
ip_checksum.domain = InternetChecksum(fields=[ip_ver,
ip_ihl,
ip_tos,
ip_tot_len,
ip_id,
ip_flags,
ip_frag_off,
ip_ttl,
ip_proto,
ip_checksum,
ip_saddr,
ip_daddr], dataType=Raw(nbBytes=2, unitSize=AbstractType.UNITSIZE_16))
self.header = Symbol(name='IP layer', fields=[ip_ver,
ip_ihl,
ip_tos,
ip_tot_len,
ip_id,
ip_flags,
ip_frag_off,
ip_ttl,
ip_proto,
ip_checksum,
ip_saddr,
ip_daddr,
ip_payload])
def mergeFields(self, field1, field2):
"""Merge specified fields.
>>> import binascii
>>> from netzob.all import *
>>> samples = ["00ff2f000000", "000010000000", "00fe1f000000"]
>>> messages = [RawMessage(data=binascii.unhexlify(sample)) for sample in samples]
>>> f1 = Field(Raw(nbBytes=1), name="f1")
>>> f2 = Field(Raw(nbBytes=2), name="f2")
>>> f3 = Field(Raw(nbBytes=2), name="f3")
>>> f4 = Field(Raw(nbBytes=1), name="f4")
>>> symbol = Symbol([f1, f2, f3, f4], messages=messages)
>>> symbol.addEncodingFunction(TypeEncodingFunction(HexaString))
>>> print(symbol)
f1 | f2 | f3 | f4
---- | ------ | ------ | ----
'00' | 'ff2f' | '0000' | '00'
'00' | '0010' | '0000' | '00'
'00' | 'fe1f' | '0000' | '00'
---- | ------ | ------ | ----
>>> fo = FieldOperations()
>>> fo.mergeFields(f2, f3)
>>> print(symbol)
f1 | Merge | f4
---- | ---------- | ----
'00' | 'ff2f0000' | '00'
'00' | '00100000' | '00'
'00' | 'fe1f0000' | '00'
---- | ---------- | ----
>>> fo.mergeFields(symbol.fields[0], symbol.fields[1])
>>> print(symbol)
Merge | f4
------------ | ----
'00ff2f0000' | '00'
'0000100000' | '00'
'00fe1f0000' | '00'
------------ | ----
>>> fo.mergeFields(symbol.fields[0], symbol.fields[1])
>>> print(symbol)
Merge
--------------
'00ff2f000000'
'000010000000'
'00fe1f000000'
--------------
:param field1: the left field to merge
:type field1: :class:`netzob.Model.Vocabulary.AbstractField.AbstractField`
:param field2: the right field to merge
:type field2: :class:`netzob.Model.Vocabulary.AbstractField.AbstractField`
:raise Exception if something bad happens
"""
if field1 is None or field2 is None:
raise TypeError("Fields cannot be None")
if field1 == field2:
raise ValueError("Cannot merge a unique field (field1 == field2)")
self._logger.debug("Merging field {0} with field {1}".format(
field1.name, field2.name))
if field1.parent is not field2.parent:
raise ValueError(
"Specified fields don't have the same parent, only fields with same parents can be merged."
)
# retrieve indexes of specified fields
iField1 = None
iField2 = None
for iField, field in enumerate(field1.parent.fields):
if field == field1:
iField1 = iField
elif field == field2:
iField2 = iField
if iField1 is None:
raise ValueError(
"Cannot retrieve position of field1 in its parent fields")
if iField2 is None:
raise ValueError(
"Cannot retrieve position of field2 in its parent fields")
if iField2 != iField1 + 1:
raise ValueError(
"Field1 must be directly on the left of field2 (iField1={0}, iField2={1})".
format(iField1, iField2))
# build a new field domain
newDomain = Agg([field1.domain, field2.domain])
newField = Field(domain=newDomain, name="Merge")
newField.encodingFunctions = list(field1.encodingFunctions.values())
parent = field1.parent
before = parent.fields[:iField1]
after = parent.fields[iField2 + 1:]
parent.fields = before + [newField] + after
def __create_fields(self, symbol, results):
size_of_size = 1
for result in results:
field_to_split, maxSize, indexInField = self.__getFieldFromIndex(
result[0], symbol)
self._logger.warning("[Field to split] : " + field_to_split.name +
"\n")
first_field_dep, maxSizefd, indexInFieldfd = self.__getFieldFromIndex(
result[1], symbol)
if indexInFieldfd != 0:
#Create a subfield in the first_field_dependency
values_first_field_dep = first_field_dep.getValues()
if len(set(values_first_field_dep)) > 1:
#Alt field
values_first_field_dep_before = []
values_first_field_dep_after = []
for value in values_first_field_dep:
values_first_field_dep_before.append(
value[:int(indexInFieldfd / 8)])
values_first_field_dep_after.append(
value[int(indexInFieldfd / 8):])
first_field_dep_before = Field(
domain=Alt(values_first_field_dep_before),
name=first_field_dep.name + "-0")
first_field_dep_after = Field(
domain=Alt(values_first_field_dep_after),
name=first_field_dep.name + "-1")
else:
#Static field
value = values_first_field_dep[0]
value_first_field_dep_before = value[:indexInFieldfd]
value_first_field_dep_after = value[indexInFieldfd:]
first_field_dep_before = Field(
domain=Raw(value_first_field_dep_before),
name=first_field_dep.name + "-0")
first_field_dep_after = Field(
domain=Raw(value_first_field_dep_after),
name=first_field_dep.name + "-1")
first_field_dep.fields = [
first_field_dep_before, first_field_dep_after
]
self._logger.warning("[First field dependency] : " +
first_field_dep.name + "\n")
field_dep = self.__get_field_dep(symbol, first_field_dep)
#Check if static or Alt:
values_before = []
values_after = []
field_to_split_values = field_to_split.getValues()
if len(set(field_to_split_values)) > 1:
#Alt field
for value in field_to_split_values:
values_before.append(value[:int(indexInField / 8)])
values_after.append(value[int(indexInField / 8) +
size_of_size:])
#AltField before =>
altField_before_Size = Field(domain=Alt(values_before),
name=field_to_split.name +
"-Before_SizeF")
altField_after_Size = Field(domain=Alt(values_after),
name=field_to_split.name +
"-After_Size")
sizeField = Field(domain=Size(field_dep),
name=field_to_split.name + "-Size")
field_to_split.fields = [
altField_before_Size, sizeField, altField_after_Size
]
else:
#Static field
value_before = field_to_split_values[0][:int(indexInField / 8)]
value_after = field_to_split_values[0][int(indexInField / 8) +
size_of_size:]
staticField_before_Size = Field(domain=Raw(value_before),
name=field_to_split.name +
"-Before_SizeF")
staticField_after_Size = Field(domain=Raw(value_after),
name=field_to_split.name +
"-After_Size")
sizeField = Field(domain=Size(field_dep),
name=field_to_split.name + "-Size")
field_to_split.fields = [
staticField_before_Size, sizeField, staticField_after_Size
]
def create_field(self, name, did):
domain = self.get_domain(did)
field = Field(name=name, domain=domain)
self.__fields[str(field.id)] = field
return field
def initHeader(self):
"""Initialize the IP header according to the IP format definition.
"""
ip_ver = Field(name='ip.version',
domain=BitArray(value=bitarray('0100'))) # IP Version 4
ip_ihl = Field(name='ip.hdr_len', domain=BitArray(bitarray('0000')))
ip_tos = Field(name='ip.tos',
domain=Data(dataType=BitArray(nbBits=8),
originalValue=bitarray('00000000'),
svas=SVAS.PERSISTENT))
ip_tot_len = Field(name='ip.len',
domain=BitArray(bitarray('0000000000000000')))
ip_id = Field(name='ip.id', domain=BitArray(nbBits=16))
ip_flags = Field(name='ip.flags',
domain=Data(dataType=BitArray(nbBits=3),
originalValue=bitarray('000'),
svas=SVAS.PERSISTENT))
ip_frag_off = Field(name='ip.fragment',
domain=Data(
dataType=BitArray(nbBits=13),
originalValue=bitarray('0000000000000'),
svas=SVAS.PERSISTENT))
ip_ttl = Field(name='ip.ttl',
domain=Data(dataType=BitArray(nbBits=8),
originalValue=bitarray('01000000'),
svas=SVAS.PERSISTENT))
ip_proto = Field(name='ip.proto',
domain=Integer(value=self.upperProtocol,
unitSize=AbstractType.UNITSIZE_8,
endianness=AbstractType.ENDIAN_BIG,
sign=AbstractType.SIGN_UNSIGNED))
ip_checksum = Field(name='ip.checksum',
domain=BitArray(bitarray('0000000000000000')))
ip_saddr = Field(name='ip.src', domain=IPv4(self.localIP))
ip_daddr = Field(name='ip.dst', domain=IPv4(self.remoteIP))
ip_payload = Field(name='ip.payload', domain=Raw())
ip_ihl.domain = Size([
ip_ver, ip_ihl, ip_tos, ip_tot_len, ip_id, ip_flags, ip_frag_off,
ip_ttl, ip_proto, ip_checksum, ip_saddr, ip_daddr
],
dataType=BitArray(nbBits=4),
factor=1 / float(32))
ip_tot_len.domain = Size([
ip_ver, ip_ihl, ip_tos, ip_tot_len, ip_id, ip_flags, ip_frag_off,
ip_ttl, ip_proto, ip_checksum, ip_saddr, ip_daddr, ip_payload
],
dataType=Integer(
unitSize=AbstractType.UNITSIZE_16,
sign=AbstractType.SIGN_UNSIGNED),
factor=1 / float(8))
ip_checksum.domain = InternetChecksum(
fields=[
ip_ver, ip_ihl, ip_tos, ip_tot_len, ip_id, ip_flags,
ip_frag_off, ip_ttl, ip_proto, ip_checksum, ip_saddr, ip_daddr
],
dataType=Raw(nbBytes=2, unitSize=AbstractType.UNITSIZE_16))
self.header = Symbol(name='IP layer',
fields=[
ip_ver, ip_ihl, ip_tos, ip_tot_len, ip_id,
ip_flags, ip_frag_off, ip_ttl, ip_proto,
ip_checksum, ip_saddr, ip_daddr, ip_payload
])
def cluster(self, field, keyField):
"""Create and return new symbols according to a specific key
field.
>>> import binascii
>>> from netzob.all import *
>>> samples = [b"00ff2f000000", b"000020000000", b"00ff2f000000"]
>>> messages = [RawMessage(data=binascii.unhexlify(sample)) for sample in samples]
>>> f1 = Field(Raw(nbBytes=1))
>>> f2 = Field(Raw(nbBytes=2))
>>> f3 = Field(Raw(nbBytes=3))
>>> symbol = Symbol([f1, f2, f3], messages=messages)
>>> symbol.addEncodingFunction(TypeEncodingFunction(HexaString))
>>> newSymbols = Format.clusterByKeyField(symbol, f2)
>>> for sym in list(newSymbols.values()):
... sym.addEncodingFunction(TypeEncodingFunction(HexaString))
... print(sym.name + ":")
... print(sym)
Symbol_ff2f:
Field | Field | Field
----- | ------ | --------
'00' | 'ff2f' | '000000'
'00' | 'ff2f' | '000000'
----- | ------ | --------
Symbol_0020:
Field | Field | Field
----- | ------ | --------
'00' | '0020' | '000000'
----- | ------ | --------
:param field: the field we want to split in new symbols
:type field: :class:`netzob.Model.Vocabulary.AbstractField.AbstractField`
:param keyField: the field used as a key during the splitting operation
:type field: :class:`netzob.Model.Vocabulary.AbstractField.AbstractField`
:raise Exception if something bad happens
"""
# Safe checks
if field is None:
raise TypeError("'field' should not be None")
if keyField is None:
raise TypeError("'keyField' should not be None")
if keyField not in field.fields:
raise TypeError("'keyField' is not a child of 'field'")
newSymbols = collections.OrderedDict()
keyFieldMessageValues = keyField.getMessageValues(
encoded=False, styled=False)
newSymbolsSplittedMessages = {}
# we identify what would be the best type of the key field
keyFieldType = ASCII
for message, keyFieldValue in list(keyFieldMessageValues.items()):
# If the value cannot be parsed as ASCII, we convert it to HexaString
if not ASCII().canParse(
TypeConverter.convert(keyFieldValue, Raw, BitArray)):
keyFieldType = HexaString
break
# Even if the value is theoritically parsable as ASCII, some caracters cannot be encoded, so we double check
tmp_value = TypeConverter.convert(keyFieldValue, Raw, ASCII)
tmp2_value = TypeConverter.convert(tmp_value, ASCII, Raw)
if keyFieldValue != tmp2_value:
# This means we cannot retrieve the original value by encoding and then decoding in ASCII
keyFieldType = HexaString
break
# we create a symbol for each of these uniq values
for message, keyFieldValue in list(keyFieldMessageValues.items()):
keyFieldValue = TypeConverter.convert(keyFieldValue, Raw,
keyFieldType)
if keyFieldValue not in list(newSymbols.keys()):
if type(keyFieldValue) is str:
symbolName = "Symbol_{0}".format(keyFieldValue)
else:
symbolName = "Symbol_{0}".format(
keyFieldValue.decode("utf-8"))
newSymbols[keyFieldValue] = Symbol(
name=symbolName, messages=[message])
splittedMessages = DataAlignment.align(
[message.data], field, encoded=False)
newSymbolsSplittedMessages[
keyFieldValue] = [splittedMessages[0]]
else:
newSymbols[keyFieldValue].messages.append(message)
splittedMessages = DataAlignment.align(
[message.data], field, encoded=False)
newSymbolsSplittedMessages[keyFieldValue].append(
splittedMessages[0])
for newSymbolKeyValue, newSymbol in list(newSymbols.items()):
# we recreate the same fields in this new symbol as the fields that exist in the original symbol
newSymbol.clearFields()
for i, f in enumerate(field.fields):
if f == keyField:
newFieldDomain = TypeConverter.convert(newSymbolKeyValue,
keyFieldType, Raw)
else:
newFieldDomain = set()
for j in range(
len(newSymbolsSplittedMessages[
newSymbolKeyValue])):
newFieldDomain.add(newSymbolsSplittedMessages[
newSymbolKeyValue][j][i])
newFieldDomain = list(newFieldDomain)
newF = Field(name=f.name, domain=newFieldDomain)
newF.parent = newSymbol
newSymbol.fields.append(newF)
# we remove endless fields that accepts no values
cells = newSymbol.getCells(
encoded=False, styled=False, transposed=False)
max_i_cell_with_value = 0
for line in cells:
for i_cell, cell in enumerate(line):
if cell != '' and max_i_cell_with_value < i_cell:
max_i_cell_with_value = i_cell
newSymbol.clearFields()
for i, f in enumerate(field.fields[:max_i_cell_with_value + 1]):
if f == keyField:
newFieldDomain = TypeConverter.convert(newSymbolKeyValue,
keyFieldType, Raw)
else:
newFieldDomain = set()
for j in range(
len(newSymbolsSplittedMessages[
newSymbolKeyValue])):
newFieldDomain.add(newSymbolsSplittedMessages[
newSymbolKeyValue][j][i])
newFieldDomain = list(newFieldDomain)
newF = Field(name=f.name, domain=newFieldDomain)
newF.parent = newSymbol
newSymbol.fields.append(newF)
return newSymbols
def execute(self, field):
"""Executes the field edition following the specified messages.
Children of the specified field will be replaced with new fields.
:param field: the format definition that will be user
:type field: :class:`netzob.Model.Vocabulary.AbstractField.AbstractField`
:raise Exception: if something bad happens
"""
if field is None:
raise TypeError("The field cannot be None")
fieldValues = [
TypeConverter.convert(data, Raw, HexaString)
for data in field.getValues(encoded=False)
]
if len(fieldValues) == 0:
raise Exception("No value found in the field.")
# Retrieve longuest field value
maxLengthFieldValue = len(max(fieldValues, key=len))
# definies the step following specified unitsize
stepUnitsize = self.__computeStepForUnitsize()
# Vertical identification of variation
indexedValues = []
for i in range(0, maxLengthFieldValue, stepUnitsize):
currentIndexValue = []
for fieldValue in fieldValues:
if i < len(fieldValue):
currentIndexValue.append(
fieldValue[i:min(len(fieldValue), i + stepUnitsize)])
else:
currentIndexValue.append(b'')
indexedValues.append(currentIndexValue)
# If requested, merges the adjacent static fields
if self.mergeAdjacentStaticFields:
result = []
staticSequences = []
for values in indexedValues:
if len(set(values)) == 1:
# static
staticSequences.append(values[0])
else:
# dynamic
if len(staticSequences) > 0:
result.append([b''.join(staticSequences)])
staticSequences = []
result.append(values)
if len(staticSequences) > 0:
result.append([b''.join(staticSequences)])
indexedValues = result
# If requested, merges the adjacent dynamic fields
if self.mergeAdjacentDynamicFields:
result = []
dynamicSequences = []
for values in indexedValues:
if len(set(values)) > 1:
# dynamic
dynamicSequences.append(values)
else:
# static
if len(dynamicSequences) > 0:
dynValues = zip(*dynamicSequences)
tmp_result = []
for d in dynValues:
tmp_result.append(b''.join(
[x if x is not None else b'' for x in d]))
result.append(tmp_result)
dynamicSequences = []
result.append(values)
if len(dynamicSequences) > 0:
dynValues = zip(*dynamicSequences)
tmp_result = []
for d in dynValues:
tmp_result.append(b''.join(
[x if x is not None else b'' for x in d]))
result.append(tmp_result)
indexedValues = result
# Create a field for each entry
newFields = []
for (i, val) in enumerate(indexedValues):
fName = "Field-{0}".format(i)
fDomain = DomainFactory.normalizeDomain([
Raw(TypeConverter.convert(v, HexaString, BitArray))
for v in set(val)
])
newFields.append(Field(domain=fDomain, name=fName))
# attach encoding functions
for newField in newFields:
newField.encodingFunctions = list(field.encodingFunctions.values())
field.fields = newFields
def split(field, delimiter):
"""Split a field (or symbol) with a specific delimiter. The
delimiter can be passed either as an ASCII, a Raw, an
HexaString, or any objects that inherit from AbstractType.
>>> from netzob.all import *
>>> samples = [b"aaaaff000000ff10", b"bbff110010ff00000011", b"ccccccccfffe1f000000ff12"]
>>> messages = [RawMessage(data=sample) for sample in samples]
>>> symbol = Symbol(messages=messages[:3])
>>> Format.splitDelimiter(symbol, ASCII("ff"))
>>> print(symbol)
Field-0 | Field-sep-6666 | Field-2 | Field-sep-6666 | Field-4
---------- | -------------- | ------------ | -------------- | ----------
'aaaa' | 'ff' | '000000' | 'ff' | '10'
'bb' | 'ff' | '110010' | 'ff' | '00000011'
'cccccccc' | 'ff' | 'fe1f000000' | 'ff' | '12'
---------- | -------------- | ------------ | -------------- | ----------
>>> samples = [b"434d446964656e74696679230400000066726564", b"5245536964656e74696679230000000000000000", b"434d44696e666f2300000000", b"524553696e666f230000000004000000696e666f", b"434d4473746174732300000000", b"52455373746174732300000000050000007374617473", b"434d4461757468656e7469667923090000006d7950617373776421", b"52455361757468656e74696679230000000000000000", b"434d44656e6372797074230a00000031323334353674657374", b"524553656e637279707423000000000a00000073707176777436273136", b"434d4464656372797074230a00000073707176777436273136", b"5245536465637279707423000000000a00000031323334353674657374", b"434d446279652300000000", b"524553627965230000000000000000", b"434d446964656e746966792307000000526f626572746f", b"5245536964656e74696679230000000000000000", b"434d44696e666f2300000000", b"524553696e666f230000000004000000696e666f", b"434d4473746174732300000000", b"52455373746174732300000000050000007374617473", b"434d4461757468656e74696679230a000000615374726f6e67507764", b"52455361757468656e74696679230000000000000000", b"434d44656e63727970742306000000616263646566", b"524553656e6372797074230000000006000000232021262724", b"434d44646563727970742306000000232021262724", b"52455364656372797074230000000006000000616263646566", b"434d446279652300000000", b"524553627965230000000000000000"]
>>> messages = [RawMessage(data=TypeConverter.convert(sample, HexaString, Raw)) for sample in samples]
>>> symbol = Symbol(messages=messages)
>>> symbol.encodingFunctions.add(TypeEncodingFunction(ASCII)) # Change visualization to hexastring
>>> Format.splitDelimiter(symbol, ASCII("#"))
>>> print(symbol)
Field-0 | Field-sep-23 | Field-2 | Field-sep-23 | Field-4
--------------- | ------------ | -------------------- | ------------ | -------
'CMDidentify' | '#' | '....fred' | '' | ''
'RESidentify' | '#' | '........' | '' | ''
'CMDinfo' | '#' | '....' | '' | ''
'RESinfo' | '#' | '........info' | '' | ''
'CMDstats' | '#' | '....' | '' | ''
'RESstats' | '#' | '........stats' | '' | ''
'CMDauthentify' | '#' | '....myPasswd!' | '' | ''
'RESauthentify' | '#' | '........' | '' | ''
'CMDencrypt' | '#' | '....123456test' | '' | ''
'RESencrypt' | '#' | "........spqvwt6'16" | '' | ''
'CMDdecrypt' | '#' | "....spqvwt6'16" | '' | ''
'RESdecrypt' | '#' | '........123456test' | '' | ''
'CMDbye' | '#' | '....' | '' | ''
'RESbye' | '#' | '........' | '' | ''
'CMDidentify' | '#' | '....Roberto' | '' | ''
'RESidentify' | '#' | '........' | '' | ''
'CMDinfo' | '#' | '....' | '' | ''
'RESinfo' | '#' | '........info' | '' | ''
'CMDstats' | '#' | '....' | '' | ''
'RESstats' | '#' | '........stats' | '' | ''
'CMDauthentify' | '#' | '....aStrongPwd' | '' | ''
'RESauthentify' | '#' | '........' | '' | ''
'CMDencrypt' | '#' | '....abcdef' | '' | ''
'RESencrypt' | '#' | '........' | '#' | " !&'$"
'CMDdecrypt' | '#' | '....' | '#' | " !&'$"
'RESdecrypt' | '#' | '........abcdef' | '' | ''
'CMDbye' | '#' | '....' | '' | ''
'RESbye' | '#' | '........' | '' | ''
--------------- | ------------ | -------------------- | ------------ | -------
>>> print(symbol.fields[0]._str_debug())
Field-0
|-- Alt
|-- Data (Raw=b'CMDidentify' ((0, 88)))
|-- Data (Raw=b'RESidentify' ((0, 88)))
|-- Data (Raw=b'CMDinfo' ((0, 56)))
|-- Data (Raw=b'RESinfo' ((0, 56)))
|-- Data (Raw=b'CMDstats' ((0, 64)))
|-- Data (Raw=b'RESstats' ((0, 64)))
|-- Data (Raw=b'CMDauthentify' ((0, 104)))
|-- Data (Raw=b'RESauthentify' ((0, 104)))
|-- Data (Raw=b'CMDencrypt' ((0, 80)))
|-- Data (Raw=b'RESencrypt' ((0, 80)))
|-- Data (Raw=b'CMDdecrypt' ((0, 80)))
|-- Data (Raw=b'RESdecrypt' ((0, 80)))
|-- Data (Raw=b'CMDbye' ((0, 48)))
|-- Data (Raw=b'RESbye' ((0, 48)))
Below is another example of the FieldSplitDelimiter usage: it splits fields based on a Raw string.
>>> from netzob.all import *
>>> samples = [b"\\x01\\x02\\x03\\xff\\x04\\x05\\xff\\x06\\x07", b"\\x01\\x02\\xff\\x03\\x04\\x05\\x06\\xff\\x07", b"\\x01\\xff\\x02\\x03\\x04\\x05\\x06"]
>>> messages = [RawMessage(data=sample) for sample in samples]
>>> symbol = Symbol(messages=messages)
>>> Format.splitDelimiter(symbol, Raw(b"\\xff"))
>>> print(symbol)
Field-0 | Field-sep-ff | Field-2 | Field-sep-ff | Field-4
-------------- | ------------ | ---------------------- | ------------ | ----------
'\\x01\\x02\\x03' | b'\\xff' | '\\x04\\x05' | b'\\xff' | '\\x06\\x07'
'\\x01\\x02' | b'\\xff' | '\\x03\\x04\\x05\\x06' | b'\\xff' | '\\x07'
'\\x01' | b'\\xff' | '\\x02\\x03\\x04\\x05\\x06' | '' | ''
-------------- | ------------ | ---------------------- | ------------ | ----------
:param field : the field to consider when spliting
:type: :class:`netzob.Model.Vocabulary.AbstractField.AbstractField`
:param delimiter : the delimiter used to split messages of the field
:type: :class:`netzob.Model.Vocabulary.Types.AbstractType.AbstractType`
"""
if delimiter is None:
raise TypeError("Delimiter cannot be None.")
if field is None:
raise TypeError("Field cannot be None.")
if len(field.messages) < 1:
raise ValueError(
"The associated symbol does not contain any message.")
# Find message substrings after applying delimiter
splittedMessages = []
for cell in field.getValues(encoded=False, styled=False):
splittedMessage = cell.split(delimiter.value.tobytes())
splittedMessages.append(splittedMessage)
import itertools
# Inverse the array, so that columns contains observed values for each field
splittedMessages = list(itertools.zip_longest(*splittedMessages))
# If the delimiter does not create splitted fields
if len(splittedMessages) <= 1:
return
# Else, we add (2*len(splittedMessages)-1) fields
newFields = []
iField = -1
for i in range(len(splittedMessages)):
iField += 1
fieldDomain = list()
# temporary set that hosts all the observed values to prevent useless duplicate ones
observedValues = set()
has_inserted_empty_value = False
isEmptyField = True # To avoid adding an empty field
for v in splittedMessages[i]:
if v != "" and v is not None:
isEmptyField = False
if v not in observedValues:
fieldDomain.append(Raw(v))
observedValues.add(v)
else:
if not has_inserted_empty_value:
fieldDomain.append(Raw(nbBytes=0))
has_inserted_empty_value = True
if not isEmptyField:
newField = Field(
domain=DomainFactory.normalizeDomain(fieldDomain),
name="Field-" + str(iField))
newField.encodingFunctions = list(
field.encodingFunctions.values())
newFields.append(newField)
iField += 1
str_delimiter = TypeConverter.convert(delimiter.value, BitArray,
HexaString).decode('utf-8')
fieldName = "Field-sep-{}".format(str_delimiter)
newFields.append(
Field(domain=Alt([delimiter, Raw(nbBytes=0)]), name=fieldName))
newFields.pop()
# Reset the field
from netzob.Inference.Vocabulary.Format import Format
Format.resetFormat(field)
# Create a field for each entry
field.fields = newFields
def __automate_mid_field_creation(self, results, symbol, endianness):
"""
This method attempts to create a CRC field. It creates either a big or little endian CRC relation field using
the value b'\xca\xfe\x00\x00\x00\x00\xba\xbe' to compute the CRC
, the b'\x00\x00\x00\x00' is then replaced by the computed CRC in the message.
:param results: list of CRC indexes
:param symbol: The symbol in wich we want to define CRC fields
:param endianness: A string stating the endianness of the CRC, should either be "big" or "little"
"""
domain_field_list = []
subfields = []
for crcindex in results:
searchedfield, field_size, field_index = self.__getFieldFromIndex(crcindex, symbol)
# Get all the fields after the current field (to define the CRC domain Relation
afterField = False
if field_size / 8 == 4:
for i,f in enumerate(symbol.fields):
if f.name == searchedfield.name:
domain_field_list.append(symbol.fields[i-1])
domain_field_list.append(symbol.fields[i+1])
# Is the field just CRC or more?
if endianness == 'little':
searchedfield.name = "CRC32_mid_LE" + str(crcindex)
else:
searchedfield.name = "CRC32_mid_BE" + str(crcindex)
searchedfield.domain = CRC32(domain_field_list, endianness=endianness)
elif field_size > 4:
# The Field is the CRC and something else.
if field_index == 0:
# CRC at beginning of field. Create new fields after CRC
# Create a list of possible values
val_list = []
for val in searchedfield.getValues():
val_list.append(val[4:])
# Make a set
val_set = set(val_list)
newf = self.__define_field(val_set,searchedfield)
#Compute second field (the one before the CRC
val_list = [] #List containing the first four bytes used to compute CRC32 (the 4 bytes prior to the CRC)
temp_field_list = []
for prevalue in symbol.getValues():
prevalue = prevalue[crcindex - 4:crcindex]
val_list.append(prevalue)
val_list = set(val_list)
for i, f in enumerate(symbol.fields):
if f.name == searchedfield.name:
prevField,minSize,maxSize = self.__getFieldFromIndex(crcindex - 1,symbol)
if maxSize >= 4:
#The four bytes are contained in only one field. We create a subfield
newf2 = self.__define_field(val_list, temp)
else:
#The four bytes are split into several fields. We delete all thes fields and create a new field and insert it at the index of the first of these fields in symbol.fields()
for j in range(1,4):
try:
prevField, minSize, maxSize = self.__getFieldFromIndex(crcindex - j, symbol)
except:
pass
temp_field_list.append(prevField)
temp_field_set = set(temp_field_list) #All the fields containing the four bytes
for fiel in temp_field_set:
miny = 99999
stuffToDel = []
for y,fol in enumerate(symbol.fields):
if fol == fiel:
if y < miny:
miny = y
stuffToDel.append(y)
temp = symbol.fields[y]
newf2 = self.__define_field(val_list,temp)
domain_field_list.append(newf2)
domain_field_list.append(Field(domain=Raw(b'\x00\x00\x00\x00')))
domain_field_list.append(newf)
if endianness == 'little':
crc32field = Field(name="CRC32_mid_LE" + str(crcindex),
domain=CRC32(domain_field_list, endianness=endianness))
else:
crc32field = Field(name="CRC32_mid_BE" + str(crcindex),
domain=CRC32(domain_field_list, endianness=endianness))
subfields.append(crc32field)
subfields.append(newf)
elif field_index / 8 + 4 == field_size / 8:
# CRC at end of field. Create new fields before CRC
# Create a list of possible values
val_list = []
for val in searchedfield.getValues():
val_list.append(val[:field_index])
# Make a set
val_set = set(val_list)
newf = self.__define_field(val_set,searchedfield)
#Compute second field (the one after the CRC
val_list = [] #List containing the last four bytes used to compute CRC32 (the 4 bytes after the CRC)
temp_field_list = []
for prevalue in symbol.getValues():
prevalue = prevalue[crcindex:crcindex+4]
val_list.append(prevalue)
val_list = set(val_list)
for i, f in enumerate(symbol.fields):
if f.name == searchedfield.name:
prevField,minSize,maxSize = self.__getFieldFromIndex(crcindex + 1,symbol)
if maxSize >= 4:
#The four bytes are contained in only one field. We create a subfield
newf2 = self.__define_field(val_list,temp)
else:
#The four bytes are split into several fields. We delete all thes fields and create a new field and insert it at the index of the first of these fields in symbol.fields()
for j in range(1,4):
prevField, minSize, maxSize = self.__getFieldFromIndex(crcindex + j, symbol)
temp_field_list.append(prevField)
temp_field_list = set(temp_field_list) #All the fields containing the four bytes
for fiel in set(temp_field_list):
miny = 99999
for y,fol in enumerate(symbol.fields):
if fol == fiel:
if y < miny:
miny = y
temp = symbol.fields[y]
del symbol.fields[y]
symbol.fields[miny] = self.__define_field(val_list,temp)
newf2 = self.__define_field(val_list,temp)
domain_field_list.append(newf)
domain_field_list.append(Field(domain=Raw(b'\x00\x00\x00\x00')))
domain_field_list.append(newf2)
if endianness == 'little':
crc32field = Field(name="CRC32_mid_LE" + str(crcindex),
domain=CRC32(domain_field_list, endianness=endianness))
else:
crc32field = Field(name="CRC32_mid_BE" + str(crcindex),
domain=CRC32(domain_field_list, endianness=endianness))
subfields.append(newf)
subfields.append(crc32field)
else:
# CRC in the middle. Create new fields before and after CRC
val_list1 = []
val_list2 = []
for val in searchedfield.getValues():
val_list1.append(val[4:])
val_list2.append(val[:field_index])
# Make a set
val_set1 = set(val_list1)
val_set2 = set(val_list2)
newf1 = self.__define_field(val_set1,searchedfield)
newf2 = self.__define_field(val_set2,searchedfield)
domain_field_list.insert(0, newf2)
if endianness == 'little':
crc32field = Field(name="CRC32_LE" + str(crcindex),
domain=CRC32(domain_field_list, endianness=endianness))
else:
crc32field = Field(name="CRC32_BE" + str(crcindex),
domain=CRC32(domain_field_list, endianness=endianness))
subfields.append(newf1)
subfields.append(crc32field)
subfields.append(newf2)
searchedfield.fields = subfields
def __automate_field_creation(self,results,symbol,endianness):
"""
This method attempts to create a CRC field. It creates either a big or little
endian CRC relation field using all the following fields to compute the CRC.
:param results: list of CRC indexes
:param symbol: The symbol in wich we want to define CRC fields
:param endianness: A string stating the endianness of the CRC, should either be "big" or "little"
"""
domain_field_list = []
subfields = []
for crcindex in results:
searchedfield, field_size, field_index = self.__getFieldFromIndex(crcindex, symbol)
# Get all the fields after the current field (to define the CRC domain Relation
afterField = False
for f in symbol.fields:
if afterField:
domain_field_list.append(f)
elif f.name == searchedfield.name:
afterField = True
if field_size / 8 == 4:
# Is the field just CRC or more?
if endianness == 'little':
searchedfield.name = "CRC32_LE" + str(crcindex)
else:
searchedfield.name = "CRC32_BE" + str(crcindex)
searchedfield.domain = CRC32(domain_field_list, endianness=endianness)
elif field_size > 4:
# The Field is the CRC and something else.
if field_index == 0:
# CRC at beginning of field. Create new fields after CRC
# Create a list of possible values
val_list = []
for val in searchedfield.getValues():
val_list.append(val[4:])
# Make a set
val_set = set(val_list)
newf = self.__define_field(val_set,searchedfield)
domain_field_list.insert(0, newf)
if endianness == 'little':
crc32field = Field(name="CRC32_LE" + str(crcindex), domain=CRC32(domain_field_list,endianness=endianness))
else:
crc32field = Field(name="CRC32_BE" + str(crcindex), domain=CRC32(domain_field_list,endianness=endianness))
subfields.append(crc32field)
subfields.append(newf)
elif field_index / 8 + 4 == field_size / 8:
# CRC at end of field. Create new fields before CRC
# Create a list of possible values
val_list = []
for val in searchedfield.getValues():
val_list.append(val[:field_index])
# Make a set
val_set = set(val_list)
newf = self.__define_field(val_set,searchedfield)
if endianness == 'little':
crc32field = Field(name="CRC32_LE" + str(crcindex), domain=CRC32(domain_field_list,endianness=endianness))
else:
crc32field = Field(name="CRC32_BE" + str(crcindex), domain=CRC32(domain_field_list,endianness=endianness))
subfields.append(newf)
subfields(crc32field)
else:
# CRC in the middle. Create new fields before and after CRC
val_list1 = []
val_list2 = []
for val in searchedfield.getValues():
val_list1.append(val[4:])
val_list2.append(val[:field_index])
# Make a set
val_set1 = set(val_list1)
val_set2 = set(val_list2)
newf1 = self.__define_field(val_set1,searchedfield)
newf2 = self.__define_field(val_set2,searchedfield)
domain_field_list.insert(0, newf2)
if endianness == 'little':
crc32field = Field(name="CRC32_LE" + str(crcindex), domain=CRC32(domain_field_list,endianness=endianness))
else:
crc32field = Field(name="CRC32_BE" + str(crcindex), domain=CRC32(domain_field_list,endianness=endianness))
subfields.append(newf1)
subfields.append(crc32field)
subfields.append(newf2)
searchedfield.fields = subfields
def initHeader(self):
"""Initialize the IP header according to the IP format definition.
"""
# Ethernet header
# Retrieve remote MAC address
dstMacAddr = arpreq.arpreq(self.remoteIP)
if dstMacAddr is not None:
dstMacAddr = dstMacAddr.replace(':', '')
dstMacAddr = binascii.unhexlify(dstMacAddr)
else:
# Force ARP resolution
p = subprocess.Popen(["/bin/ping", "-c1", self.remoteIP])
p.wait()
time.sleep(0.1)
dstMacAddr = arpreq.arpreq(self.remoteIP)
if dstMacAddr is not None:
dstMacAddr = dstMacAddr.replace(':', '')
dstMacAddr = binascii.unhexlify(dstMacAddr)
else:
raise Exception("Cannot resolve IP address to a MAC address for IP: '{}'".format(self.remoteIP))
# Retrieve local MAC address
srcMacAddr = self.get_interface_addr(bytes(self.interface, 'utf-8'))[1]
eth_dst = Field(name='eth.dst', domain=Raw(dstMacAddr))
eth_src = Field(name='eth.src', domain=Raw(srcMacAddr))
eth_type = Field(name='eth.type', domain=Raw(b"\x08\x00"))
# IP header
ip_ver = Field(
name='ip.version', domain=BitArray(
value=bitarray('0100'))) # IP Version 4
ip_ihl = Field(name='ip.hdr_len', domain=BitArray(bitarray('0000')))
ip_tos = Field(
name='ip.tos',
domain=Data(
dataType=BitArray(nbBits=8),
originalValue=bitarray('00000000'),
svas=SVAS.PERSISTENT))
ip_tot_len = Field(
name='ip.len', domain=BitArray(bitarray('0000000000000000')))
ip_id = Field(name='ip.id', domain=BitArray(nbBits=16))
ip_flags = Field(name='ip.flags', domain=Data(dataType=BitArray(nbBits=3), originalValue=bitarray('000'), svas=SVAS.PERSISTENT))
ip_frag_off = Field(name='ip.fragment', domain=Data(dataType=BitArray(nbBits=13), originalValue=bitarray('0000000000000'), svas=SVAS.PERSISTENT))
ip_ttl = Field(name='ip.ttl', domain=Data(dataType=BitArray(nbBits=8), originalValue=bitarray('01000000'), svas=SVAS.PERSISTENT))
ip_proto = Field(name='ip.proto', domain=Integer(value=self.upperProtocol, unitSize=AbstractType.UNITSIZE_8, endianness=AbstractType.ENDIAN_BIG, sign=AbstractType.SIGN_UNSIGNED))
ip_checksum = Field(name='ip.checksum', domain=BitArray(bitarray('0000000000000000')))
ip_saddr = Field(name='ip.src', domain=IPv4(self.localIP))
ip_daddr = Field(
name='ip.dst', domain=IPv4(self.remoteIP))
ip_payload = Field(name='ip.payload', domain=Raw())
ip_ihl.domain = Size([ip_ver,
ip_ihl,
ip_tos,
ip_tot_len,
ip_id, ip_flags,
ip_frag_off,
ip_ttl, ip_proto,
ip_checksum,
ip_saddr,
ip_daddr], dataType=BitArray(nbBits=4), factor=1/float(32))
ip_tot_len.domain = Size([ip_ver,
ip_ihl,
ip_tos,
ip_tot_len,
ip_id,
ip_flags,
ip_frag_off,
ip_ttl,
ip_proto,
ip_checksum,
ip_saddr,
ip_daddr,
ip_payload], dataType=Integer(unitSize=AbstractType.UNITSIZE_16, sign=AbstractType.SIGN_UNSIGNED), factor=1/float(8))
ip_checksum.domain = InternetChecksum(fields=[ip_ver,
ip_ihl,
ip_tos,
ip_tot_len,
ip_id,
ip_flags,
ip_frag_off,
ip_ttl,
ip_proto,
ip_checksum,
ip_saddr,
ip_daddr], dataType=Raw(nbBytes=2, unitSize=AbstractType.UNITSIZE_16))
self.header = Symbol(name='Ethernet layer', fields=[eth_dst,
eth_src,
eth_type,
ip_ver,
ip_ihl,
ip_tos,
ip_tot_len,
ip_id,
ip_flags,
ip_frag_off,
ip_ttl,
ip_proto,
ip_checksum,
ip_saddr,
ip_daddr,
ip_payload])
def setUp(self) -> None:
"""
Construct AbstractFields with the same name.
Use a subclass implementing AbstractField to instantiate.
"""
self.afs = [Field(name="NEMESYS")] * 5
def split(field, delimiter):
"""Split a field (or symbol) with a specific delimiter. The
delimiter can be passed either as an ASCII, a Raw, an
HexaString, or any objects that inherit from AbstractType.
>>> from netzob.all import *
>>> samples = [b"aaaaff000000ff10", b"bbff110010ff00000011", b"ccccccccfffe1f000000ff12"]
>>> messages = [RawMessage(data=sample) for sample in samples]
>>> symbol = Symbol(messages=messages[:3])
>>> Format.splitDelimiter(symbol, ASCII("ff"))
>>> print(symbol)
Field-0 | Field-sep-6666 | Field-2 | Field-sep-6666 | Field-4
---------- | -------------- | ------------ | -------------- | ----------
'aaaa' | 'ff' | '000000' | 'ff' | '10'
'bb' | 'ff' | '110010' | 'ff' | '00000011'
'cccccccc' | 'ff' | 'fe1f000000' | 'ff' | '12'
---------- | -------------- | ------------ | -------------- | ----------
>>> samples = [b"434d446964656e74696679230400000066726564", b"5245536964656e74696679230000000000000000", b"434d44696e666f2300000000", b"524553696e666f230000000004000000696e666f", b"434d4473746174732300000000", b"52455373746174732300000000050000007374617473", b"434d4461757468656e7469667923090000006d7950617373776421", b"52455361757468656e74696679230000000000000000", b"434d44656e6372797074230a00000031323334353674657374", b"524553656e637279707423000000000a00000073707176777436273136", b"434d4464656372797074230a00000073707176777436273136", b"5245536465637279707423000000000a00000031323334353674657374", b"434d446279652300000000", b"524553627965230000000000000000", b"434d446964656e746966792307000000526f626572746f", b"5245536964656e74696679230000000000000000", b"434d44696e666f2300000000", b"524553696e666f230000000004000000696e666f", b"434d4473746174732300000000", b"52455373746174732300000000050000007374617473", b"434d4461757468656e74696679230a000000615374726f6e67507764", b"52455361757468656e74696679230000000000000000", b"434d44656e63727970742306000000616263646566", b"524553656e6372797074230000000006000000232021262724", b"434d44646563727970742306000000232021262724", b"52455364656372797074230000000006000000616263646566", b"434d446279652300000000", b"524553627965230000000000000000"]
>>> messages = [RawMessage(data=TypeConverter.convert(sample, HexaString, Raw)) for sample in samples]
>>> symbol = Symbol(messages=messages)
>>> symbol.encodingFunctions.add(TypeEncodingFunction(ASCII)) # Change visualization to hexastring
>>> Format.splitDelimiter(symbol, ASCII("#"))
>>> print(symbol)
Field-0 | Field-sep-23 | Field-2 | Field-sep-23 | Field-4
--------------- | ------------ | -------------------- | ------------ | -------
'CMDidentify' | '#' | '....fred' | '' | ''
'RESidentify' | '#' | '........' | '' | ''
'CMDinfo' | '#' | '....' | '' | ''
'RESinfo' | '#' | '........info' | '' | ''
'CMDstats' | '#' | '....' | '' | ''
'RESstats' | '#' | '........stats' | '' | ''
'CMDauthentify' | '#' | '....myPasswd!' | '' | ''
'RESauthentify' | '#' | '........' | '' | ''
'CMDencrypt' | '#' | '....123456test' | '' | ''
'RESencrypt' | '#' | "........spqvwt6'16" | '' | ''
'CMDdecrypt' | '#' | "....spqvwt6'16" | '' | ''
'RESdecrypt' | '#' | '........123456test' | '' | ''
'CMDbye' | '#' | '....' | '' | ''
'RESbye' | '#' | '........' | '' | ''
'CMDidentify' | '#' | '....Roberto' | '' | ''
'RESidentify' | '#' | '........' | '' | ''
'CMDinfo' | '#' | '....' | '' | ''
'RESinfo' | '#' | '........info' | '' | ''
'CMDstats' | '#' | '....' | '' | ''
'RESstats' | '#' | '........stats' | '' | ''
'CMDauthentify' | '#' | '....aStrongPwd' | '' | ''
'RESauthentify' | '#' | '........' | '' | ''
'CMDencrypt' | '#' | '....abcdef' | '' | ''
'RESencrypt' | '#' | '........' | '#' | " !&'$"
'CMDdecrypt' | '#' | '....' | '#' | " !&'$"
'RESdecrypt' | '#' | '........abcdef' | '' | ''
'CMDbye' | '#' | '....' | '' | ''
'RESbye' | '#' | '........' | '' | ''
--------------- | ------------ | -------------------- | ------------ | -------
>>> print(symbol.fields[0]._str_debug())
Field-0
|-- Alt
|-- Data (Raw=b'CMDidentify' ((0, 88)))
|-- Data (Raw=b'RESidentify' ((0, 88)))
|-- Data (Raw=b'CMDinfo' ((0, 56)))
|-- Data (Raw=b'RESinfo' ((0, 56)))
|-- Data (Raw=b'CMDstats' ((0, 64)))
|-- Data (Raw=b'RESstats' ((0, 64)))
|-- Data (Raw=b'CMDauthentify' ((0, 104)))
|-- Data (Raw=b'RESauthentify' ((0, 104)))
|-- Data (Raw=b'CMDencrypt' ((0, 80)))
|-- Data (Raw=b'RESencrypt' ((0, 80)))
|-- Data (Raw=b'CMDdecrypt' ((0, 80)))
|-- Data (Raw=b'RESdecrypt' ((0, 80)))
|-- Data (Raw=b'CMDbye' ((0, 48)))
|-- Data (Raw=b'RESbye' ((0, 48)))
Below is another example of the FieldSplitDelimiter usage: it splits fields based on a Raw string.
>>> from netzob.all import *
>>> samples = [b"\\x01\\x02\\x03\\xff\\x04\\x05\\xff\\x06\\x07", b"\\x01\\x02\\xff\\x03\\x04\\x05\\x06\\xff\\x07", b"\\x01\\xff\\x02\\x03\\x04\\x05\\x06"]
>>> messages = [RawMessage(data=sample) for sample in samples]
>>> symbol = Symbol(messages=messages)
>>> Format.splitDelimiter(symbol, Raw(b"\\xff"))
>>> print(symbol)
Field-0 | Field-sep-ff | Field-2 | Field-sep-ff | Field-4
-------------- | ------------ | ---------------------- | ------------ | ----------
'\\x01\\x02\\x03' | b'\\xff' | '\\x04\\x05' | b'\\xff' | '\\x06\\x07'
'\\x01\\x02' | b'\\xff' | '\\x03\\x04\\x05\\x06' | b'\\xff' | '\\x07'
'\\x01' | b'\\xff' | '\\x02\\x03\\x04\\x05\\x06' | '' | ''
-------------- | ------------ | ---------------------- | ------------ | ----------
:param field : the field to consider when spliting
:type: :class:`netzob.Model.Vocabulary.AbstractField.AbstractField`
:param delimiter : the delimiter used to split messages of the field
:type: :class:`netzob.Model.Types.AbstractType.AbstractType`
"""
if delimiter is None:
raise TypeError("Delimiter cannot be None.")
if field is None:
raise TypeError("Field cannot be None.")
if len(field.messages) < 1:
raise ValueError("The associated symbol does not contain any message.")
# Find message substrings after applying delimiter
splittedMessages = []
for cell in field.getValues(encoded=False, styled=False):
splittedMessage = cell.split(delimiter.value.tobytes())
splittedMessages.append(splittedMessage)
import itertools
# Inverse the array, so that columns contains observed values for each field
splittedMessages = list(itertools.zip_longest(*splittedMessages))
# If the delimiter does not create splitted fields
if len(splittedMessages) <= 1:
return
# Else, we add (2*len(splittedMessages)-1) fields
newFields = []
iField = -1
for i in range(len(splittedMessages)):
iField += 1
fieldDomain = list()
# temporary set that hosts all the observed values to prevent useless duplicate ones
observedValues = set()
has_inserted_empty_value = False
isEmptyField = True # To avoid adding an empty field
for v in splittedMessages[i]:
if v != "" and v is not None:
isEmptyField = False
if v not in observedValues:
fieldDomain.append(Raw(v))
observedValues.add(v)
else:
if not has_inserted_empty_value:
fieldDomain.append(Raw(nbBytes=0))
has_inserted_empty_value = True
if not isEmptyField:
newField = Field(domain=DomainFactory.normalizeDomain(fieldDomain), name="Field-"+str(iField))
newField.encodingFunctions = list(field.encodingFunctions.values())
newFields.append(newField)
iField += 1
str_delimiter = TypeConverter.convert(delimiter.value, BitArray, HexaString).decode('utf-8')
fieldName = "Field-sep-{}".format(str_delimiter)
newFields.append(Field(domain=Alt([delimiter, Raw(nbBytes=0)]), name=fieldName))
newFields.pop()
# Reset the field
from netzob.Inference.Vocabulary.Format import Format
Format.resetFormat(field)
# Create a field for each entry
field.fields = newFields
def cluster(self, field, keyField):
"""Create and return new symbols according to a specific key
field.
>>> import binascii
>>> from netzob.all import *
>>> samples = [b"00ff2f000000", b"000020000000", b"00ff2f000000"]
>>> messages = [RawMessage(data=binascii.unhexlify(sample)) for sample in samples]
>>> f1 = Field(Raw(nbBytes=1))
>>> f2 = Field(Raw(nbBytes=2))
>>> f3 = Field(Raw(nbBytes=3))
>>> symbol = Symbol([f1, f2, f3], messages=messages)
>>> symbol.addEncodingFunction(TypeEncodingFunction(HexaString))
>>> newSymbols = Format.clusterByKeyField(symbol, f2)
>>> for sym in list(newSymbols.values()):
... sym.addEncodingFunction(TypeEncodingFunction(HexaString))
... print(sym.name + ":")
... print(sym)
Symbol_ff2f:
Field | Field | Field
----- | ------ | --------
'00' | 'ff2f' | '000000'
'00' | 'ff2f' | '000000'
----- | ------ | --------
Symbol_0020:
Field | Field | Field
----- | ------ | --------
'00' | '0020' | '000000'
----- | ------ | --------
:param field: the field we want to split in new symbols
:type field: :class:`netzob.Model.Vocabulary.AbstractField.AbstractField`
:param keyField: the field used as a key during the splitting operation
:type field: :class:`netzob.Model.Vocabulary.AbstractField.AbstractField`
:raise Exception if something bad happens
"""
# Safe checks
if field is None:
raise TypeError("'field' should not be None")
if keyField is None:
raise TypeError("'keyField' should not be None")
if keyField not in field.fields:
raise TypeError("'keyField' is not a child of 'field'")
newSymbols = collections.OrderedDict()
keyFieldMessageValues = keyField.getMessageValues(encoded=False,
styled=False)
newSymbolsSplittedMessages = {}
# we identify what would be the best type of the key field
keyFieldType = ASCII
for message, keyFieldValue in list(keyFieldMessageValues.items()):
# If the value cannot be parsed as ASCII, we convert it to HexaString
if not ASCII().canParse(
TypeConverter.convert(keyFieldValue, Raw, BitArray)):
keyFieldType = HexaString
break
# Even if the value is theoritically parsable as ASCII, some caracters cannot be encoded, so we double check
tmp_value = TypeConverter.convert(keyFieldValue, Raw, ASCII)
tmp2_value = TypeConverter.convert(tmp_value, ASCII, Raw)
if keyFieldValue != tmp2_value:
# This means we cannot retrieve the original value by encoding and then decoding in ASCII
keyFieldType = HexaString
break
# we create a symbol for each of these uniq values
for message, keyFieldValue in list(keyFieldMessageValues.items()):
keyFieldValue = TypeConverter.convert(keyFieldValue, Raw,
keyFieldType)
if keyFieldValue not in list(newSymbols.keys()):
if type(keyFieldValue) is str:
symbolName = "Symbol_{0}".format(keyFieldValue)
else:
symbolName = "Symbol_{0}".format(
keyFieldValue.decode("utf-8"))
newSymbols[keyFieldValue] = Symbol(name=symbolName,
messages=[message])
splittedMessages = DataAlignment.align([message.data],
field,
encoded=False)
newSymbolsSplittedMessages[keyFieldValue] = [
splittedMessages[0]
]
else:
newSymbols[keyFieldValue].messages.append(message)
splittedMessages = DataAlignment.align([message.data],
field,
encoded=False)
newSymbolsSplittedMessages[keyFieldValue].append(
splittedMessages[0])
for newSymbolKeyValue, newSymbol in list(newSymbols.items()):
# we recreate the same fields in this new symbol as the fields that exist in the original symbol
newSymbol.clearFields()
for i, f in enumerate(field.fields):
if f == keyField:
newFieldDomain = TypeConverter.convert(
newSymbolKeyValue, keyFieldType, Raw)
else:
newFieldDomain = set()
for j in range(
len(newSymbolsSplittedMessages[newSymbolKeyValue])
):
newFieldDomain.add(
newSymbolsSplittedMessages[newSymbolKeyValue][j]
[i])
newFieldDomain = list(newFieldDomain)
newF = Field(name=f.name, domain=newFieldDomain)
newF.parent = newSymbol
newSymbol.fields.append(newF)
# we remove endless fields that accepts no values
cells = newSymbol.getCells(encoded=False,
styled=False,
transposed=False)
max_i_cell_with_value = 0
for line in cells:
for i_cell, cell in enumerate(line):
if cell != '' and max_i_cell_with_value < i_cell:
max_i_cell_with_value = i_cell
newSymbol.clearFields()
for i, f in enumerate(field.fields[:max_i_cell_with_value + 1]):
if f == keyField:
newFieldDomain = TypeConverter.convert(
newSymbolKeyValue, keyFieldType, Raw)
else:
newFieldDomain = set()
for j in range(
len(newSymbolsSplittedMessages[newSymbolKeyValue])
):
newFieldDomain.add(
newSymbolsSplittedMessages[newSymbolKeyValue][j]
[i])
newFieldDomain = list(newFieldDomain)
newF = Field(name=f.name, domain=newFieldDomain)
newF.parent = newSymbol
newSymbol.fields.append(newF)
return newSymbols
def _createSubFieldsForADynamicField(self, field, align, semanticTags):
"""Analyzes the dynamic field provided and create sub fields following
the provided semantic tags."""
if field is None:
raise TypeError("Field cannot be None")
if align is None:
raise TypeError("Align cannot be None")
if semanticTags is None:
raise TypeError("SemanticTags cannot be None")
self._logger.debug(
"Create subfields for dynamic field {0} : {1}".format(
field.name, field.regex))
subFields = []
currentTag = None
currentTagLength = 0
semanticTagsForEachMessage = field.getSemanticTagsByMessage()
for index, tag in list(semanticTags.items()):
if tag != currentTag:
# Create a sub field
if currentTagLength > 0:
values = self._getFieldValuesWithTag(
field, semanticTagsForEachMessage, currentTag)
subFields.append((currentTag, values))
currentTagLength = 0
currentTag = tag
currentTagLength += 1
if currentTagLength > 0:
values = self._getFieldValuesWithTag(field,
semanticTagsForEachMessage,
currentTag)
subFields.append((currentTag, values))
self._logger.debug("Identified subFields : {0}".format(subFields))
for iSubField, (tag, values) in enumerate(subFields):
if len(values) > 0:
if tag == b"None":
minValue = None
maxValue = None
for v in values:
if minValue is None or len(v) < minValue:
minValue = len(v)
if maxValue is None or len(v) > maxValue:
maxValue = len(v)
subField = Field(
b"{0}_{1}".format(field.getName(), iSubField),
b"(.{" + str(minValue) + b"," + str(maxValue) + b"})",
field.getSymbol())
field.addLocalField(subField)
else:
# create regex based on unique values
newRegex = '|'.join(list(set(values)))
newRegex = b"({0})".format(newRegex)
subField = Field(
b"{0}_{1}".format(field.getName(), iSubField),
newRegex, field.getSymbol())
field.addLocalField(subField)
def __seek_ip(self, ip, symbol, create_fields=False, two_terms=False):
#Convert IP to hex value:
hexipstring = binascii.hexlify(socket.inet_aton(ip))
hexipstring = binascii.unhexlify(hexipstring)
index_list = []
for message in symbol.messages:
results = self.__core_find(message.data, hexipstring, index_list,
two_terms)
#Change stdout to get message print in buffer
old_stdout = sys.stdout
sys.stdout = buffer1 = io.StringIO()
print(message)
sys.stdout = old_stdout
#Print results using click
self._logger.warning("Results for [Message] : \n" +
buffer1.getvalue() + "\n")
self._logger.warning("[Number of results found] : " +
str(results.total_length) + "\n")
self._logger.warning("Result indexes in message: \n ")
self._logger.warning("[Whole IP Big Endian] : " +
str(results.full_be) + "\n")
self._logger.warning("[Three last terms of IP Big Endian] : " +
str(results.one_less_be) + "\n")
self._logger.warning("[Two last terms of IP Big Endian] : " +
str(results.two_less_be) + "\n")
self._logger.warning("[Whole IP Little Endian] : " +
str(results.full_le) + "\n")
self._logger.warning("[Three last terms of IP Little Endian] : " +
str(results.one_less_le) + "\n")
self._logger.warning("[Two last terms of IP Little Endian] : " +
str(results.two_less_le) + "\n")
if create_fields:
self._logger.warning("Attempting to create new fields")
if symbol.fields:
self._logger.warning("Refining search to fields...")
for field in symbol.fields:
subfield_index_list = []
field_values = field.getValues()
number_of_values = len(set(field_values))
# Get field length
max_length = max([len(i) for i in field_values])
# Check if values in messages are different for need to create an alternative field or a simple raw field (above 1 if several values):
if number_of_values > 1:
# More than one value => MUST CREATE ALT FIELD
# TODO
pass
else:
mess = field_values[0]
field_result = self.__core_find(
mess, hexipstring, index_list, two_terms)
if field_result.full_be or field_result.one_less_be or field_result.two_less_be or field_result.full_le or field_result.one_less_le or field_result.two_less_le:
# Searchstring not always split in between fields => Need to create subfields
self._logger.warning(
"Searchstring inside fields, creating subfields..."
)
# Create field dict which contains fields and index
fields_dict = dict()
if number_of_values > 1:
#More than one value => MUST CREATE ALT FIELD
#TODO
pass
else:
#Can create simple static subfield
for i in field_result.full_be:
fields_dict[i] = Field(
name='FullIpBe' + str(i),
domain=Raw(hexipstring))
subfield_index_list.append(i)
subfield_index_list.append(i + 4)
for i in field_result.one_less_be:
fields_dict[i] = Field(
name='ThreeTIpBe' + str(i),
domain=Raw(hexipstring[1:]))
subfield_index_list.append(i)
subfield_index_list.append(i + 3)
for i in field_result.two_less_be:
fields_dict[i] = Field(
name='TwoTIpBe' + str(i),
domain=Raw(hexipstring[2:]))
subfield_index_list.append(i)
subfield_index_list.append(i + 2)
for i in field_result.full_le:
fields_dict[i] = Field(
name='FullIpLe' + str(i),
domain=Raw(hexipstring[::-1]))
subfield_index_list.append(i)
subfield_index_list.append(i + 4)
for i in field_result.one_less_le:
fields_dict[i] = Field(
name='ThreeTIpLe' + str(i),
domain=Raw(hexipstring[1:][::-1]))
subfield_index_list.append(i)
subfield_index_list.append(i + 3)
fields_dict[i] = Field(
name='TwoTIpLe' + str(i),
domain=Raw(hexipstring[2:][::-1]))
subfield_index_list.append(i)
subfield_index_list.append(i + 2)
#Sort list in order
subfield_index_list.append(max_length)
subfield_index_list.insert(0, 0)
#Remove duplicates (there shouldn't be any!)
subfield_index_list = list(
set(subfield_index_list))
subfield_index_list = sorted(
subfield_index_list, key=int)
#Create static field for every two elements of the subfield index_list as they represent beginning and end
for i, x in enumerate(subfield_index_list):
#Check if index is pair
if (i % 2 == 0):
#Create static field and store it's beginning index in the structure
try:
fields_dict[x] = Field(
name='Field' + str(i),
domain=Raw(field_values[0][
subfield_index_list[i]:
subfield_index_list[i +
1]]
))
except:
fields_dict[x] = Field(
name='Field' + str(i),
domain=Raw(
field_values[0]
[subfield_index_list[
i]:]))
else:
# Don't do shit
pass
#Create a list of all subfields in order
od = collections.OrderedDict(
sorted(fields_dict.items()))
field_list = list(od.values())
if not field.fields:
#No subfields
field.fields = field_list
else:
#Already contains subfields. We need to just insert our IP subfields in it.
#Get the indexes of Fields which have a name other than FieldX (where X is a number)
only_Ip_fields_dict = dict()
for index, element in fields_dict.items():
if not element.name[:5] == "Field":
only_Ip_fields_dict[
index] = element
#Find to which field these indexes belong to in the old symbol
for index, element in only_Ip_fields_dict.items(
):
new_field_dict = dict()
field_to_split, maxSize, indexInField = self.__getFieldFromIndex(
index, field)
indexInField = int(indexInField / 8)
new_field_dict[indexInField] = element
if field_to_split.getValues(
)[0][:indexInField]:
new_field_dict[0] = Field(
name='FieldBeforeIp' +
str(index),
domain=Raw(
field_to_split.getValues()
[0][:indexInField]))
new_field_dict[indexInField] = element
#Get Size of the IP Field
minSize, sizeIp = element.domain.dataType.size
sizeIp = int(sizeIp / 8)
if field_to_split.getValues(
)[0][indexInField + sizeIp:]:
new_field_dict[
indexInField + sizeIp] = Field(
name='FieldAfterIp' +
str(index),
domain=Raw(field_to_split.
getValues()[0]
[indexInField +
sizeIp:]))
od = collections.OrderedDict(
sorted(new_field_dict.items()))
field_list = list(od.values())
#DELETE EMPTY DOMAIN FIELDS
field_to_split.fields = field_list
else:
#Searchstring always split in between other fields => Delete fields and create new ones Or just return indexes, and let user redefine fields manually
# symbol.fields = []
self._logger.warning(
"Searchstring in between fields...")
self._logger.warning(
"[1] : Delete all fields and replace by IPFIELDS"
+ "\n")
self._logger.warning(
"[ANY] : Just print the index in the message"
+ "\n")
return