def check_string(opt):
"""
Return the calculated CRC sum of a string.
"""
error = False
if opt.undefined_crc_parameters:
sys.stderr.write(
"{0:s}: error: undefined parameters\n".format(progname))
sys.exit(1)
if opt.algorithm == 0:
opt.algorithm = opt.algo_bit_by_bit | opt.algo_bit_by_bit_fast | opt.algo_table_driven
alg = Crc(width=opt.width,
poly=opt.poly,
reflect_in=opt.reflect_in,
xor_in=opt.xor_in,
reflect_out=opt.reflect_out,
xor_out=opt.xor_out,
table_idx_width=opt.tbl_idx_width)
crc = None
if opt.algorithm & opt.algo_bit_by_bit:
bbb_crc = alg.bit_by_bit(opt.check_string)
if crc != None and bbb_crc != crc:
error = True
crc = bbb_crc
if opt.algorithm & opt.algo_bit_by_bit_fast:
bbf_crc = alg.bit_by_bit_fast(opt.check_string)
if crc != None and bbf_crc != crc:
error = True
crc = bbf_crc
if opt.algorithm & opt.algo_table_driven:
# no point making the python implementation slower by using less than 8 bits as index.
opt.tbl_idx_width = 8
tbl_crc = alg.table_driven(opt.check_string)
if crc != None and tbl_crc != crc:
error = True
crc = tbl_crc
if error:
sys.stderr.write(
"{0:s}: error: different checksums!\n".format(progname))
if opt.algorithm & opt.algo_bit_by_bit:
sys.stderr.write(
" bit-by-bit: {0:#x}\n".format(bbb_crc))
if opt.algorithm & opt.algo_bit_by_bit_fast:
sys.stderr.write(
" bit-by-bit-fast: {0:#x}\n".format(bbf_crc))
if opt.algorithm & opt.algo_table_driven:
sys.stderr.write(
" table_driven: {0:#x}\n".format(tbl_crc))
sys.exit(1)
return crc
def __get_crc(self, model, check_str='123456789', expected_crc=None):
"""
Get the CRC for a set of parameters from the Python reference implementation.
"""
if self.verbose:
out_str = 'Crc(width = {width:d}, poly = {poly:#x}, reflect_in = {reflect_in}, xor_in = {xor_in:#x}, reflect_out = {reflect_out}, xor_out = {xor_out:#x})'.format(
**model)
if expected_crc is not None:
out_str += ' [check = {0:#x}]'.format(expected_crc)
print(out_str)
alg = Crc(width=model['width'],
poly=model['poly'],
reflect_in=model['reflect_in'],
xor_in=model['xor_in'],
reflect_out=model['reflect_out'],
xor_out=model['xor_out'])
error = False
crc = expected_crc
if self.use_algo_bit_by_bit:
bbb_crc = alg.bit_by_bit(check_str)
if crc is None:
crc = bbb_crc
error = error or bbb_crc != crc
if self.use_algo_bit_by_bit_fast:
bbf_crc = alg.bit_by_bit_fast(check_str)
if crc is None:
crc = bbf_crc
error = error or bbf_crc != crc
if self.use_algo_table_driven:
tbl_crc = alg.table_driven(check_str)
if crc is None:
crc = tbl_crc
error = error or tbl_crc != crc
if error:
print('error: different checksums!')
if expected_crc is not None:
print(' check: {0:#x}'.format(expected_crc))
if self.use_algo_bit_by_bit:
print(' bit-by-bit: {0:#x}'.format(bbb_crc))
if self.use_algo_bit_by_bit_fast:
print(' bit-by-bit-fast: {0:#x}'.format(bbf_crc))
if self.use_algo_table_driven:
print(' table_driven: {0:#x}'.format(tbl_crc))
return None
return crc
def check_crc(code_bytes):
# Initialize the CRC algorithm
crc = Crc(width=16,
poly=0x1021,
reflect_in=True,
xor_in=0xffff,
reflect_out=True,
xor_out=0xffff)
# Separate the 128-bit number from its CRC value
received_number = code_bytes[0:16]
received_crc = int.from_bytes(code_bytes[16:18], byteorder="little")
# Compute the CRC value of the received number
computed_crc = crc.bit_by_bit_fast(received_number)
return computed_crc, received_crc
def check_string(opt):
"""
Return the calculated CRC sum of a string.
"""
error = False
if opt.undefined_crc_parameters:
sys.stderr.write("{0:s}: error: undefined parameters\n".format(progname))
sys.exit(1)
if opt.algorithm == 0:
opt.algorithm = opt.algo_bit_by_bit | opt.algo_bit_by_bit_fast | opt.algo_table_driven
alg = Crc(
width=opt.width, poly=opt.poly,
reflect_in=opt.reflect_in, xor_in=opt.xor_in,
reflect_out=opt.reflect_out, xor_out=opt.xor_out,
table_idx_width=opt.tbl_idx_width)
crc = None
if opt.algorithm & opt.algo_bit_by_bit:
bbb_crc = alg.bit_by_bit(opt.check_string)
if crc != None and bbb_crc != crc:
error = True
crc = bbb_crc
if opt.algorithm & opt.algo_bit_by_bit_fast:
bbf_crc = alg.bit_by_bit_fast(opt.check_string)
if crc != None and bbf_crc != crc:
error = True
crc = bbf_crc
if opt.algorithm & opt.algo_table_driven:
# no point making the python implementation slower by using less than 8 bits as index.
opt.tbl_idx_width = 8
tbl_crc = alg.table_driven(opt.check_string)
if crc != None and tbl_crc != crc:
error = True
crc = tbl_crc
if error:
sys.stderr.write("{0:s}: error: different checksums!\n".format(progname))
if opt.algorithm & opt.algo_bit_by_bit:
sys.stderr.write(" bit-by-bit: {0:#x}\n".format(bbb_crc))
if opt.algorithm & opt.algo_bit_by_bit_fast:
sys.stderr.write(" bit-by-bit-fast: {0:#x}\n".format(bbf_crc))
if opt.algorithm & opt.algo_table_driven:
sys.stderr.write(" table_driven: {0:#x}\n".format(tbl_crc))
sys.exit(1)
return crc
def __get_crc(self, model, check_str = '123456789', expected_crc = None):
"""
Get the CRC for a set of parameters from the Python reference implementation.
"""
if self.verbose:
out_str = 'Crc(width = {width:d}, poly = {poly:#x}, reflect_in = {reflect_in}, xor_in = {xor_in:#x}, reflect_out = {reflect_out}, xor_out = {xor_out:#x})'.format(**model)
if expected_crc is not None:
out_str += ' [check = {0:#x}]'.format(expected_crc)
print(out_str)
alg = Crc(width = model['width'], poly = model['poly'],
reflect_in = model['reflect_in'], xor_in = model['xor_in'],
reflect_out = model['reflect_out'], xor_out = model['xor_out'])
error = False
crc = expected_crc
if self.use_algo_bit_by_bit:
bbb_crc = alg.bit_by_bit(check_str)
if crc is None:
crc = bbb_crc
error = error or bbb_crc != crc
if self.use_algo_bit_by_bit_fast:
bbf_crc = alg.bit_by_bit_fast(check_str)
if crc is None:
crc = bbf_crc
error = error or bbf_crc != crc
if self.use_algo_table_driven:
tbl_crc = alg.table_driven(check_str)
if crc is None:
crc = tbl_crc
error = error or tbl_crc != crc
if error:
print('error: different checksums!')
if expected_crc is not None:
print(' check: {0:#x}'.format(expected_crc))
if self.use_algo_bit_by_bit:
print(' bit-by-bit: {0:#x}'.format(bbb_crc))
if self.use_algo_bit_by_bit_fast:
print(' bit-by-bit-fast: {0:#x}'.format(bbf_crc))
if self.use_algo_table_driven:
print(' table_driven: {0:#x}'.format(tbl_crc))
return None
return crc
class RelationFinder(object):
"""Provides multiple algorithms to find relations between messages.
>>> import binascii
>>> from netzob.all import *
>>> samples = [b"0007ff2f000000000000", b"0011ffaaaaaaaaaaaaaabbcc0010000000000000", b"0012ffddddddddddddddddddddfe1f000000000000"]
>>> messages = [RawMessage(data=binascii.unhexlify(sample)) for sample in samples]
>>> symbol = Symbol(messages=messages)
>>> Format.splitStatic(symbol)
>>> rels = RelationFinder.findOnFields(symbol.fields[1], symbol.fields[3])
>>> print(len(rels))
1
>>> for rel in rels:
... print(rel["relation_type"] + " between " + rel["x_field"].name + ":" + rel["x_attribute"] + \
" and " + rel["y_field"].name + ":" + rel["y_attribute"])
SizeRelation between Field-1:value and Field-3:size
>>> rels = RelationFinder.findOnSymbol(symbol)
>>> print(len(rels))
1
>>> for rel in rels:
... print(rel["relation_type"] + " between fields " + str([x.name for x in rel["x_fields"]]) + ":" + rel["x_attribute"] + \
" and fields " + str([y.name for y in rel["y_fields"]]) + ":" + rel["y_attribute"])
SizeRelation between fields ['Field-1']:value and fields ['Field-3']:size
The following illustrates DataRelation
>>> samples = ["Adrien > my name is Adrien", "Zoby > my name is Zoby"]
>>> messages = [RawMessage(sample) for sample in samples]
>>> symbol = Symbol(messages=messages)
>>> Format.splitAligned(symbol)
>>> results = RelationFinder.findOnSymbol(symbol)
>>> len(results)
3
>>> print(results[0]['relation_type'])
DataRelation
>>> results[0]['x_fields'][0].getValues()
[b'Adrien', b'Zoby']
"""
# Field's attributes
ATTR_VALUE = "value"
ATTR_SIZE = "size"
ATTR_CRC32 = "crc32"
ATTR_TIGER_CRC16 = "tiger_crc16"
AVAILABLE_ATTRIBUTES = [ATTR_VALUE, ATTR_SIZE]
# Relation types
REL_SIZE = "SizeRelation"
REL_DATA = "DataRelation"
REL_EQUALITY = "EqualityRelation"
REL_UNKNOWN = "Unknown"
def __init__(self):
self.tiger_crc = Crc(width=16, poly=0x1021, xor_in=0x0000, xor_out=0xFFFF, reflect_in=True, reflect_out=True)
pass
@staticmethod
@typeCheck(AbstractField)
def findOnSymbol(symbol):
"""Find exact relations between fields in the provided
symbol/field.
:param symbol: the symbol in which we are looking for relations
:type symbol: :class:`netzob.Model.Vocabulary.AbstractField.AbstractField`
"""
rf = RelationFinder()
return rf.executeOnSymbol(symbol)
@staticmethod
@typeCheck(AbstractField, AbstractField, str, str)
def findOnFields(x_field, y_field, x_attribute=None, y_attribute=None):
"""Find exact relations between the provided fields, according
to their optional specified attributes.
"""
rf = RelationFinder()
return rf.executeOnFields(x_field, y_field, x_attribute, y_attribute)
# @typeCheck(AbstractField)
# def executeOnSymbol(self, symbol):
# """
# :param symbol: the symbol in which we are looking for relations
# :type symbol: :class:`netzob.Model.Vocabulary.AbstractField.AbstractField`
# """
# cells = [field.getValues(encoded=False, styled=False)
# for field in symbol.getExtendedFields()
# #if not field.isStatic()
# ]
# if cells:
# # Invert array dimensions liks this:
# # < [[m0f0, m1f0, ...], [m0f1, m1f1, ...]]
# # > [(m0f0, m0f1, ...), (m1f0, m1f1, ...)]
# for algo, refs in _libRelation.find(zip(*cells)).items():
# for ref_idx, ref_off, ref_size, rels in refs:
# print "Relations(%s) with F%d:" % (algo, ref_idx)
# for rel_id, rel_conf in enumerate(rels):
# print " %d. F[%d][%d:%d]" % ((rel_id,) + rel_conf)
# def executeOnCells(self, cellsTable):
# if cellsTable:
# # Invert array dimensions liks this:
# # < [[m0f0, m1f0, ...], [m0f1, m1f1, ...]]
# # > [(m0f0, m0f1, ...), (m1f0, m1f1, ...)]
# for algo, refs in _libRelation.find(zip(*cellsTable)).items():
# for ref_idx, ref_off, ref_size, rels in refs:
# print "Relations(%s) with F%d:" % (algo, ref_idx)
# for rel_id, rel_conf in enumerate(rels):
# print " %d. F[%d][%d:%d]" % ((rel_id,) + rel_conf)
@typeCheck(AbstractField)
def executeOnSymbol(self, symbol):
"""Find exact relations between fields of the provided symbol.
"""
(attributeValues_headers,
attributeValues) = self._generateAttributeValuesForSymbol(symbol)
results = []
for i, x_values in enumerate(attributeValues[:-1]):
for j, y_values in enumerate(attributeValues[:]):
if j <= i:
continue
isRelation = True
for k in range(len(x_values)):
if not (x_values[k] == y_values[k]):
isRelation = False
break
if isRelation:
# TODO make configurable
# # Do no keep relations where a field's values does not change
# if len(set(x_values)) == 1 or len(set(y_values)) == 1:
# continue
(x_fields, x_attribute) = attributeValues_headers[i]
(y_fields, y_attribute) = attributeValues_headers[j]
# The relation should not apply on the same field
if len(x_fields) == 1 and len(y_fields) == 1 and x_fields[
0].id == y_fields[0].id:
continue
relation_type = self._findRelationType(x_attribute,
y_attribute, x_fields, y_fields)
# We do not consider unqualified relation (for example, the size of a field is linked to the size of another field)
if relation_type == self.REL_UNKNOWN:
continue
# DataRelation should produce an empty intersection between related fields
if relation_type == self.REL_DATA and len(
set(x_fields).intersection(set(y_fields))) > 0:
continue
# SizeRelation should a size field composed of multiple fields
if relation_type == self.REL_SIZE:
if x_attribute == self.ATTR_VALUE:
if len(x_fields) > 1:
continue
elif y_attribute == self.ATTR_VALUE:
if len(y_fields) > 1:
continue
# EqualityRelation should a field be equal to another field composed of multiple fields
if relation_type == self.REL_EQUALITY:
if x_attribute == self.ATTR_VALUE:
if len(x_fields) > 1:
continue
elif y_attribute == self.ATTR_VALUE:
if len(y_fields) > 1:
continue
self._logger.debug("Relation found between '" + str(
x_fields) + ":" + x_attribute + "' and '" + str(
y_fields) + ":" + y_attribute + "'")
id_relation = str(uuid.uuid4())
results.append({
'id': id_relation,
"relation_type": relation_type,
'x_fields': x_fields,
'x_attribute': x_attribute,
'y_fields': y_fields,
'y_attribute': y_attribute
})
return results
@typeCheck(AbstractField, AbstractField, str, str)
def executeOnFields(self,
x_field,
y_field,
x_attribute=None,
y_attribute=None):
"""Find exact relations between fields according to their
optional selected attributes.
"""
results = []
# Convert cells according to their interesting attribute (data, size or offset)
if x_attribute == self.ATTR_SIZE and y_attribute == self.ATTR_SIZE: # A relation between two size field is uncertain...
return results
x_values = x_field.getValues(encoded=False, styled=False)
y_values = y_field.getValues(encoded=False, styled=False)
# Select attributes for fields comparison
if x_attribute is None:
x_attributes = self.AVAILABLE_ATTRIBUTES
else:
x_attributes = [x_attribute]
if y_attribute is None:
y_attributes = self.AVAILABLE_ATTRIBUTES
else:
y_attributes = [y_attribute]
# Try to find a relation that matches each cell
relation_fcts = {}
# relation_fcts[self.REL_SIZE] = self._sizeRelation(x_attribute,y_attribute)
# TODO Pass correct parameters to equalRelation
relation_fcts[self.REL_SIZE] = self._sizeRelation
relation_fcts[self.REL_EQUALITY] = self._equalRelation
for x_attribute in x_attributes:
for y_attribute in y_attributes:
for (relation_name, relation_fct) in list(relation_fcts.items()):
isRelation = True
for i in range(len(x_values)):
if not relation_fct(x_values[i], x_attribute, y_values[i], y_attribute):
isRelation = False
break
if isRelation:
self._logger.debug("Relation found between '" + x_attribute + ":" + str(
x_field.name) + "' and '" + y_attribute + ":" + str(y_field.name) + "'")
self._logger.debug(" Relation: " + relation_name)
id_relation = str(uuid.uuid4())
results.append({'id': id_relation,
"relation_type": relation_name,
'x_field': x_field,
'x_attribute': x_attribute,
'y_field': y_field,
'y_attribute': y_attribute})
return results
def _findRelationType(self, x_attribute, y_attribute, x_fields, y_fields):
typeRelation = self.REL_UNKNOWN
if (x_attribute == self.ATTR_VALUE and y_attribute == self.ATTR_SIZE) or (
x_attribute == self.ATTR_SIZE and y_attribute == self.ATTR_VALUE):
typeRelation = self.REL_SIZE
elif x_attribute == y_attribute == self.ATTR_VALUE:
typeRelation = self.REL_DATA
elif self._checkEqualityRelation(x_fields, y_fields) or {self.ATTR_VALUE, self.ATTR_TIGER_CRC16} == {
x_attribute, y_attribute}:
typeRelation = self.REL_EQUALITY
return typeRelation
def _checkEqualityRelation(self, x_fields, y_fields):
x_values = []
for x_field in x_fields:
x_values += x_field.getValues(encoded=False, styled=False)
y_values = []
for y_field in y_fields:
y_values += y_field.getValues(encoded=False, styled=False)
if set(x_values) == set(y_values):
return True
else:
return False
def _equalRelation(self, x, x_attribute, y, y_attribute):
if x == y:
return True
else:
return False
def _sizeRelation(self, x, x_attribute, y, y_attribute):
if x_attribute == self.ATTR_SIZE:
if len(x) > 0:
x = len(x)
else:
if len(x) > 0:
x = TypeConverter.convert(x[:8], Raw, Integer)
else:
x = 0
if y_attribute == self.ATTR_SIZE:
if len(y) > 0:
y = len(y)
else:
if len(y) > 0:
y = TypeConverter.convert(y[:8], Raw, Integer)
else:
y = 0
if x == y:
return True
else:
return False
def _generateAttributeValuesForSymbol(self, symbol):
# First we compute the possible list of payloads
lines_data = []
line_header = []
# Compute the list of values for each field
(fields, fieldsValues) = self._getAllFieldsValues(symbol)
# Compute the table of concatenation of values
for i in range(len(fieldsValues[:])):
for j in range(i + 1, len(fieldsValues) + 1):
# We generate the data
concatCellsData = self._generateConcatData(fieldsValues[i:j])
# We generate lines and header for fields values
line_header.append((fields[i:j], self.ATTR_VALUE))
lines_data.append(self._generateDataValues(concatCellsData))
# We generate lines and header for fields values
line_header.append((fields[i:j], self.ATTR_SIZE))
lines_data.append(self._generateSizeValues(concatCellsData))
# # Generate CRC32
# line_header.append((fields[i:j], self.ATTR_CRC32))
# lines_data.append(self._generateCRC32(concatCellsData))
line_header.append((fields[i:j], self.ATTR_TIGER_CRC16))
lines_data.append(self._generateTigerCRC16(concatCellsData))
# # # Now we generate values for fields sizes
# # (multipleSize_Header, multipleSize_lines) = self._generateSizeFieldFromBeginingOfField(symbol)
# # line_header.extend(multipleSize_Header)
# # for i_line in range(0, len(lines)):
# # lines[i_line] = lines[i_line] + "," + multipleSize_lines[i_line]
# # # Now we generate values for CRC32
# # (crc32Header, crc32Lines) = self._generateCRC32(symbol)
# # line_header.extend(crc32Header)
# # for i_line in range(0, len(lines)):
# # line = lines[i_line]
# # lines[i_line] = line + "," + crc32Lines[i_line]
return (line_header, lines_data)
def _getAllFieldsValues(self, field):
# This recursive function returns a tuple containing
# (array of all fields, array of values of each field)
if len(field.fields) > 0:
fields = []
values = []
for f in field.fields:
(retFields, retValues) = self._getAllFieldsValues(f)
fields.extend(retFields)
values.extend(retValues)
return (fields, values)
else:
return ([field], [field.getValues(encoded=False, styled=False)])
def _generateConcatData(self, cellsDataList):
"""Generates the concatenation of each cell of each field.
Example:
cellsData_1 = ["a", "aa", "aaa"]
cellsData_2 = ["b", "bb", "bbb"]
res = ["ab", "aabb", "aaabbb"]
"""
if len(cellsDataList) < 1:
return []
result = [b"" for cell in cellsDataList[0]]
for cellsData in cellsDataList:
for i, data in enumerate(cellsData):
result[i] += data
return result
def _generateDataValues(self, cellsData):
result = []
for data in cellsData:
if len(data) > 0:
data = data[:8] # We take at most 8 bytes
unitSize = int(AbstractType.UNITSIZE_8) * len(data)
unitSize = int(pow(2, math.ceil(math.log(
unitSize, 2)))) # Round to the nearest upper power of 2
result.append(
Integer.encode(
data,
endianness=AbstractType.ENDIAN_BIG,
unitSize=str(unitSize)))
else:
result.append(0)
return result
def _generateSizeValues(self, cellsData):
result = []
for data in cellsData:
if len(data) > 0:
result.append(len(data)) # Size in octets
else:
result.append(0)
return result
def _generateCRC32(self, cellsData):
result = []
for data in cellsData:
valCrc32 = zlib.crc32(data) # & 0xFFFFFFFF
result.append(valCrc32)
return result
def _generateTigerCRC16(self, cellsData):
result = []
for data in cellsData:
data = data + bytes([len(data)])
valCrc32 = int.from_bytes(int.to_bytes(self.tiger_crc.bit_by_bit_fast(data), 2, byteorder="big"),
byteorder="big", signed=True)
result.append(valCrc32)
return result
class CorrelationFinder(object):
"""Correlation identification based on MINE (Maximal
Information-based Nonparametric Exploration) statistics.
TODO: implement CRC in class correctly
>>> import binascii
>>> from netzob.all import *
>>> samples = [b"0007ff2f000000000000", b"0011ffaaaaaaaaaaaaaabbcc0010000000000000", b"0012ffddddddddddddddddddddfe1f000000000000"]
>>> messages = [RawMessage(data=binascii.unhexlify(sample)) for sample in samples]
>>> symbol = Symbol(messages=messages)
>>> Format.splitStatic(symbol)
>>> rels = CorrelationFinder.find(symbol)
>>> print(len(rels))
64
"""
# Field's attributes
ATTR_VALUE = "value"
ATTR_SIZE = "size"
ATTR_CRC32 = "crc32"
ATTR_TIGER_CRC16 = "tiger_crc16"
# Relation types
REL_SIZE = "SizeRelation"
REL_DATA = "DataRelation"
@staticmethod
@typeCheck(AbstractField, float)
def find(symbol, minMic=0.7):
"""Find correlations between fields in the provided symbol,
according to a minimum threshold. The underlying work is as
follow: we compute the combination of each field's attribute
(value, size, etc.), execute MINE correlation finder on it and
parse the results.
:param symbol: the symbol in which we are looking for correlations
:type symbol: :class:`netzob.Model.Vocabulary.AbstractField.AbstractField`
:param minMic: the minimum correlation score
:type minMic: :class:`float`
"""
try:
import numpy
except:
# Fall back to classical relations
import logging
logging.warn(
"'numpy' and 'minepy' packages needed for CorrelationFinder. Fall back to RelationFinder instead."
)
return RelationFinder.findOnSymbol(symbol)
cf = CorrelationFinder(minMic)
return cf.execute(symbol)
def __init__(self, minMic=0.7):
self.minMic = minMic
self.tiger_crc = Crc(width=16,
poly=0x1021,
xor_in=0x0000,
xor_out=0x0000,
reflect_in=False,
reflect_out=False)
@typeCheck(AbstractField)
def execute(self, symbol):
"""
:param symbol: the symbol in which we are looking for correlations
:type symbol: :class:`netzob.Model.Vocabulary.AbstractField.AbstractField`
"""
(attributeValues_headers,
attributeValues) = self._generateAttributeValuesForSymbol(symbol)
symbolResults = []
# MINE computation of each field's combination
for i, values_x in enumerate(attributeValues[:-1]):
for j, values_y in enumerate(attributeValues[i + 1:]):
mine = MINE(alpha=0.6, c=15)
mine.compute_score(numpy.array(values_x),
numpy.array(values_y))
mic = round(mine.mic(), 2)
if mic > float(self.minMic):
# We add the relation to the results
(x_fields, x_attribute) = attributeValues_headers[i]
(y_fields, y_attribute) = attributeValues_headers[j]
# The relation should not apply on the same field
if len(x_fields) == 1 and len(y_fields) == 1 and x_fields[
0].id == y_fields[0].id:
continue
pearson = numpy.corrcoef(values_x, values_y)[0, 1]
if not numpy.isnan(pearson):
pearson = round(pearson, 2)
relation_type = self._findRelationType(
x_attribute, y_attribute)
self._debug_mine_stats(mine)
self._logger.debug("Correlation found between '" +
str(x_fields) + ":" + x_attribute +
"' and '" + str(y_fields) + ":" +
y_attribute + "'")
self._logger.debug(" MIC score: " + str(mic))
self._logger.debug(" Pearson score: " + str(pearson))
id_relation = str(uuid.uuid4())
symbolResults.append({
'id': id_relation,
"relation_type": relation_type,
'x_fields': x_fields,
'x_attribute': x_attribute,
'y_fields': y_fields,
'y_attribute': y_attribute,
'mic': mic,
'pearson': pearson
})
return symbolResults
def _debug_mine_stats(self, mine):
self._logger.debug("MIC: " + str(mine.mic()))
self._logger.debug("MAS: " + str(mine.mas()))
self._logger.debug("MEV: " + str(mine.mev()))
self._logger.debug("MCN (eps=0): " + str(mine.mcn(0)))
self._logger.debug("MCN (eps=1-MIC): " + str(mine.mcn_general()))
def _findRelationType(self, x_attribute, y_attribute):
typeRelation = "Unknown"
if (x_attribute == self.ATTR_VALUE and y_attribute
== self.ATTR_SIZE) or (x_attribute == self.ATTR_SIZE
and y_attribute == self.ATTR_VALUE):
typeRelation = self.REL_SIZE
elif (x_attribute == x_attribute) and x_attribute == self.ATTR_VALUE:
typeRelation = self.REL_DATA
return typeRelation
def _generateAttributeValuesForSymbol(self, symbol):
# First we compute the possible list of payloads
lines_data = []
line_header = []
# Compute the table of values
valuesTable = []
fields = symbol.fields
for field in fields:
valuesTable.append(field.getValues(encoded=False, styled=False))
# Compute the table of concatenation of values
for i in range(len(fields[:])):
for j in range(i + 1, len(fields) + 1):
# We generate the data
concatCellsData = self._generateConcatData(valuesTable[i:j])
# We generate lines and header for fields values
line_header.append((fields[i:j], self.ATTR_VALUE))
lines_data.append(self._generateDataValues(concatCellsData))
# We generate lines and header for fields values
line_header.append((fields[i:j], self.ATTR_SIZE))
lines_data.append(self._generateSizeValues(concatCellsData))
# Generate CRC32
line_header.append((fields[i:j], self.ATTR_CRC32))
lines_data.append(self._generateCRC32(concatCellsData))
# # # Now we generate values for fields sizes
# # (multipleSize_Header, multipleSize_lines) = self._generateSizeFieldFromBeginingOfField(symbol)
# # line_header.extend(multipleSize_Header)
# # for i_line in range(0, len(lines)):
# # lines[i_line] = lines[i_line] + "," + multipleSize_lines[i_line]
# # # Now we generate values for CRC32
# # (crc32Header, crc32Lines) = self._generateCRC32(symbol)
# # line_header.extend(crc32Header)
# # for i_line in range(0, len(lines)):
# # line = lines[i_line]
# # lines[i_line] = line + "," + crc32Lines[i_line]
return (line_header, lines_data)
def _generateConcatData(self, cellsDataList):
"""Generates the concatenation of each cell of each field.
Example:
cellsData_1 = ["a", "aa", "aaa"]
cellsData_2 = ["b", "bb", "bbb"]
res = ["ab", "aabb", "aaabbb"]
"""
if len(cellsDataList) < 1:
return []
result = [b"" for cell in cellsDataList[0]]
for cellsData in cellsDataList:
for i, data in enumerate(cellsData):
result[i] += data
return result
def _generateDataValues(self, cellsData):
result = []
for data in cellsData:
if len(data) > 0:
result.append(TypeConverter.convert(
data[:8], Raw, Integer)) # We take only the first 8 octets
else:
result.append(0)
return result
def _generateSizeValues(self, cellsData):
result = []
for data in cellsData:
if len(data) > 0:
result.append(len(data)) # Size in octets
else:
result.append(0)
return result
def _generateCRC32(self, cellsData):
result = []
for data in cellsData:
valCrc32 = zlib.crc32(data) & 0xFFFFFFFF
result.append(valCrc32)
return result
def _generateTigerCRC16(self, cellsData):
result = []
for data in cellsData:
data = data + bytes([len(data)])
valCrc32 = self.tiger_crc.bit_by_bit_fast(data)
result.append(valCrc32)
return result
def _generateSizeFieldFromBeginingOfField(self, symbol):
header = []
lines = []
cells = dict()
fields = symbol.fields
for field in fields:
if not field.isStatic():
header.append((self.ATTR_VALUE, field))
cells[field] = field.getCells(encoded=False, styled=False)
for i_msg in range(0, len(symbol.getMessages())):
line = []
for field in list(cells.keys()):
entry = cells[field][i_msg]
for k in range(2, 3, 2):
if len(entry) > k:
line.append(TypeConverter.netzobRawToInteger(
entry[:k]))
else:
line.append(TypeConverter.netzobRawToInteger(entry))
lines.append(b",".join(line))
return (header, lines)