def alignSymbol(self, symbol, doInternalSlick, defaultFormat):
messages = symbol.getMessages()
if messages == None or len(messages) == 0:
logging.debug("The symbol '" + symbol.getName() + "' is empty. No alignment needed")
symbol.cleanFields()
field = Field.createDefaultField()
# Use the default protocol type for representation
field.setFormat(defaultFormat)
symbol.addField(field)
else:
symbol.cleanFields()
# We execute the alignment
(alignment, score) = self.align(doInternalSlick, messages)
symbol.setAlignment(alignment)
logging.debug("Alignment : {0}".format(alignment))
# We update the regex based on the results
try:
self.buildRegexFromAlignment(symbol, alignment, defaultFormat)
except NetzobException:
logging.warn("Partitionnement error: too much fields (>100) for the symbol '" + symbol.getName() + "'")
symbol.cleanFields()
field = Field.createDefaultField()
# Use the default protocol type for representation
field.setFormat(defaultFormat)
symbol.addField(field)
def displayPopupToCreateLayer_cb(self, event):
# If fields header are selected, we get it
selectedFields = self.vocabularyController.view.selectedMessageTable.treeViewHeaderGroup.getSelectedFields()
if selectedFields is None or len(selectedFields) == 0:
# Either, we only consider the current field
selectedFields = [self.field]
# We retrieve the first and last fields selected
firstField = selectedFields[0]
lastField = selectedFields[0]
for field in selectedFields:
if field.getIndex() < firstField.getIndex():
firstField = field
if field.getIndex() > lastField.getIndex():
lastField = field
# Update selected fields to the entire range
selectedFields = []
for field in self.getSymbol().getExtendedFields():
if field.getIndex() >= firstField.getIndex() and field.getIndex() <= lastField.getIndex():
selectedFields.append(field)
# Verify that selected field range does not overlap existing layers (i.e. if the selected fields have the same parent)
parent = selectedFields[0].getParentField()
for selectedField in selectedFields:
if parent != selectedField.getParentField():
NetzobErrorMessage(_("Selected field range overlaps existing layer."))
return
# Retrieve layer's name
dialog = Gtk.Dialog(title=_("Layer creation"), flags=0, buttons=(Gtk.STOCK_CANCEL, Gtk.ResponseType.CANCEL, Gtk.STOCK_OK, Gtk.ResponseType.OK))
dialog.set_size_request(200, 50)
label = Gtk.Label("Name:")
entry = Gtk.Entry()
dialog.vbox.pack_start(label, True, True, 0)
dialog.vbox.pack_start(entry, True, True, 0)
dialog.show_all()
result = dialog.run()
if (result == Gtk.ResponseType.OK):
name = entry.get_text()
dialog.destroy()
if name == "":
return
else:
dialog.destroy()
return
# Create a new layer
regex = ""
for selectedField in selectedFields:
regex += "(?:" + selectedField.getRegex()[1:]
fieldLayer = Field(str(name), "(" + regex + ")", self.getSymbol())
index_newField = 999999
parentField = None
for selectedField in selectedFields:
parentField = selectedField.getParentField()
if parentField.getLocalFields().index(selectedField) < index_newField:
index_newField = parentField.getLocalFields().index(selectedField) # Retrieve the lowest index of the new fields
fieldLayer.addField(selectedField)
parentField.getLocalFields().remove(selectedField)
parentField.getLocalFields().insert(index_newField, fieldLayer)
# self.getSymbol().getField().addField(fieldLayer, index_newField)
self.vocabularyController.view.updateLeftPanel()
def clusterByTokenization(self, symbols):
self.ServerInference()
################################### Cluster messages according to their tokens
ll = len(self.symbols) - 1
i_equ = 0
while(ll > 0):
currentPattern = self.symbols[i_equ].getMessages()[0].getPattern()[1]
for j in range(ll):
jnext = len(self.symbols) - j - 1
cond = False
for message in self.symbols[jnext].getMessages():
if currentPattern == message.getPattern()[1]:
# score = sum([p1 == p2 for p1, p2 in zip(currentPattern, message.getPattern()[1])])
score2 = self.computeSimilarities(currentPattern, message.getPattern()[1])
# if score >= min(len(currentPattern), len(message.getPattern()[1])):
minilength = min(len(message.getData()), len(self.symbols[i_equ].getMessages()[0].getData()))
if score2 * 2.0 / minilength >= 0.40:
cond = True
if cond:
break
if(cond):
currentDst = self.symbols[i_equ].getPattern()[0]
otherDst = self.symbols[jnext].getPattern()[0]
if not self.server or (currentDst == otherDst) or (currentDst != self.server and otherDst != self.server):
self.mergeEffectiveRowCol(i_equ, jnext)
# self.log.debug("Merge the equal column/line {0} with the column/line {1}".format(str(i_equ), str(j + 1)))
i_equ -= 1
break
ll -= 1
i_equ += 1
################################## Align messages
alignment = NeedlemanAndWunsch(self.unitSize, self.cb_status)
tmpSymbols = []
for symbol in self.symbols:
# alignment.alignSymbols([symbol], self.project)
# symbol.getFields()[0].setFormat(Format.STRING)
# tmpSymbols.extend(alignment.getLastResult())
try:
# print "l"
al = self.computeAlignment(symbol)
symbol.getField().setAlignment(al)
alignment.buildRegexFromAlignment(symbol, al, self.defaultFormat)
# for (p, fields) in zip(symbol.getPattern()[1], symbol.getFields()):
# field.setFormat(p.getFormat())
except:
logging.warn("Partitionnement error: too much fields ( > 100) for the symbol '" + symbol.getName() + "' len=" + str(len(symbol.getExtendedFields())) + "len " + str(len(symbol.getPattern()[1])))
symbol.getField().removeLocalFields()
field = Field("Field 0", "(.{, })", symbol)
symbol.addField(field)
# Use the default protocol type for representation
field.setFormat(self.defaultFormat)
alignment.alignSymbols(self.symbols, self.project)
self.symbols = alignment.getLastResult()
def loadSymbol(xmlRoot, namespace_project, namespace_common, version,
project, poolOfMessages):
if version == "0.1":
idSymbol = str(xmlRoot.get("id"))
symbol = Symbol(idSymbol, "", project)
# we parse the messages
if xmlRoot.find("{" + namespace_project +
"}messages-ref") is not None:
xmlMessages = xmlRoot.find("{" + namespace_project +
"}messages-ref")
for xmlMessage in xmlMessages.findall("{" + namespace_common +
"}message-ref"):
id = xmlMessage.get("id")
message = poolOfMessages.getMessageByID(id)
if message is not None:
message.setSymbol(symbol)
symbol.addMessage(message)
# we parse the fields
if xmlRoot.find("{" + namespace_project + "}field") is not None:
xmlField = xmlRoot.find("{" + namespace_project + "}field")
field = Field.loadFromXML(xmlField, namespace_project, version,
symbol)
if field is not None:
symbol.setField(field)
return symbol
return None
def __init__(self, ID, name, project):
AbstractSymbol.__init__(self, Symbol.TYPE)
self.id = ID
self.project = project
self.messages = []
self.field = Field.createDefaultField(self)
self.field.setName(name)
self.project = project
self.pattern = None
def __init__(self, ID, name, project):
AbstractSymbol.__init__(self, Symbol.TYPE)
self.id = ID
self.project = project
self.messages = []
self.field = Field.createDefaultField(self)
self.field.setName(name)
self.project = project
self.pattern = None
def buildRegexFromAlignment(self, symbol, align, defaultFormat):
# Build regex from alignment
i = 0
start = 0
regex = []
found = False
for i in range(len(align)):
if (align[i] == "-"):
if (found == False):
start = i
found = True
else:
if (found == True):
found = False
nbTiret = i - start
regex.append("(.{," + str(nbTiret) + "})")
regex.append(align[i])
else:
if len(regex) == 0:
regex.append(align[i])
else:
regex[-1] += align[i]
if (found == True):
nbTiret = i - start + 1
regex.append("(.{," + str(nbTiret) + "})")
iField = 0
symbol.cleanFields()
logging.debug("REGEX " + str(regex))
for regexElt in regex:
field = Field("Field " + str(iField), iField, regexElt)
# Use the default protocol type for representation
field.setFormat(defaultFormat)
symbol.addField(field)
iField = iField + 1
# if len(symbol.getFields()) >= 50:
# raise NetzobException("This Python version only supports 100 named groups in regex")
# We look for useless fields
doLoop = True
# We loop until we don't pop any field
while doLoop == True:
doLoop = False
for field in symbol.getFields():
# We try to see if this field produces only empty values when applied on messages
if not field.isStatic():
cells = symbol.getCellsByField(field)
cells = "".join(cells)
if cells == "":
symbol.getFields().pop(field.getIndex()) # We remove this useless field
# Adapt index of the following fields
for fieldNext in symbol.getFields():
if fieldNext.getIndex() > field.getIndex():
fieldNext.setIndex(fieldNext.getIndex() - 1)
# Concatenate the surrounding fields (because they should be static fields)
if symbol.concatFields(field.getIndex() - 1) == 1:
doLoop = True
break
def loadSymbol(xmlRoot, namespace_project, namespace_common, version,
project, poolOfMessages):
if version == "0.1":
nameSymbol = xmlRoot.get("name")
idSymbol = xmlRoot.get("id")
symbol = TrashSymbol(idSymbol, project)
# Interpretation attributes
if xmlRoot.find("{" + namespace_project + "}format") is not None:
symbol_format = xmlRoot.find("{" + namespace_project +
"}format").text
symbol.setFormat(symbol_format)
if xmlRoot.find("{" + namespace_project + "}unitsize") is not None:
symbol_unitsize = xmlRoot.find("{" + namespace_project +
"}unitsize").text
symbol.setUnitSize(symbol_unitsize)
if xmlRoot.find("{" + namespace_project + "}sign") is not None:
symbol_sign = xmlRoot.find("{" + namespace_project +
"}sign").text
symbol.setSign(symbol_sign)
if xmlRoot.find("{" + namespace_project +
"}endianess") is not None:
symbol_endianess = xmlRoot.find("{" + namespace_project +
"}endianess").text
symbol.setEndianess(symbol_endianess)
# we parse the messages
if xmlRoot.find("{" + namespace_project +
"}messages-ref") is not None:
xmlMessages = xmlRoot.find("{" + namespace_project +
"}messages-ref")
for xmlMessage in xmlMessages.findall("{" + namespace_common +
"}message-ref"):
id = xmlMessage.get("id")
message = poolOfMessages.getMessageByID(id)
if message is not None:
message.setSymbol(symbol)
symbol.addMessage(message)
# we parse the fields
if xmlRoot.find("{" + namespace_project + "}fields") is not None:
xmlFields = xmlRoot.find("{" + namespace_project + "}fields")
for xmlField in xmlFields.findall("{" + namespace_project +
"}field"):
field = Field.loadFromXML(xmlField, namespace_project,
version)
if field is not None:
symbol.addField(field)
return symbol
return None
def importTraceInProject(self, trace, project):
percent = 0.0
inc = 1.0 / len(trace.getMessages())
# We create a symbol dedicated for the trace
symbol = Symbol(uuid.uuid4(), trace.getType(), project)
for message in trace.getMessages():
percent += inc
symbol.addMessage(message)
gobject.idle_add(self.progressbarAlignment.set_fraction, float(percent))
# We create a default field for the symbol
symbol.addField(Field.createDefaultField())
# and register the symbol in the vocabulary of the project
project.getVocabulary().addSymbol(symbol)
project.saveConfigFile(self.workspace)
def __init__(self, id=None):
AbstractSymbol.__init__(self, "TrashSymbol")
self.id = id
if self.id is None:
id = str(uuid.uuid4())
self.name = "TrashSymbol"
self.messages = []
self.encodingFilters = []
self.visualizationFilters = []
# Interpretation attributes
self.format = Format.HEX
self.unitSize = UnitSize.NONE
self.sign = Sign.UNSIGNED
self.endianess = Endianess.BIG
# Clean the symbol
self.fields = [Field.createDefaultField()]
def __init__(self, id=None):
AbstractSymbol.__init__(self, "TrashSymbol")
self.id = id
if self.id is None:
id = str(uuid.uuid4())
self.name = "TrashSymbol"
self.messages = []
self.encodingFilters = []
self.visualizationFilters = []
# Interpretation attributes
self.format = Format.HEX
self.unitSize = UnitSize.NONE
self.sign = Sign.UNSIGNED
self.endianess = Endianess.BIG
# Clean the symbol
self.fields = [Field.createDefaultField()]
def loadSymbol(xmlRoot, namespace_project, namespace_common, version, project, poolOfMessages):
if version == "0.1":
nameSymbol = xmlRoot.get("name")
idSymbol = xmlRoot.get("id")
symbol = TrashSymbol(idSymbol, project)
# Interpretation attributes
if xmlRoot.find("{" + namespace_project + "}format") is not None:
symbol_format = xmlRoot.find("{" + namespace_project + "}format").text
symbol.setFormat(symbol_format)
if xmlRoot.find("{" + namespace_project + "}unitsize") is not None:
symbol_unitsize = xmlRoot.find("{" + namespace_project + "}unitsize").text
symbol.setUnitSize(symbol_unitsize)
if xmlRoot.find("{" + namespace_project + "}sign") is not None:
symbol_sign = xmlRoot.find("{" + namespace_project + "}sign").text
symbol.setSign(symbol_sign)
if xmlRoot.find("{" + namespace_project + "}endianess") is not None:
symbol_endianess = xmlRoot.find("{" + namespace_project + "}endianess").text
symbol.setEndianess(symbol_endianess)
# we parse the messages
if xmlRoot.find("{" + namespace_project + "}messages-ref") is not None:
xmlMessages = xmlRoot.find("{" + namespace_project + "}messages-ref")
for xmlMessage in xmlMessages.findall("{" + namespace_common + "}message-ref"):
id = xmlMessage.get("id")
message = poolOfMessages.getMessageByID(id)
if message is not None:
message.setSymbol(symbol)
symbol.addMessage(message)
# we parse the fields
if xmlRoot.find("{" + namespace_project + "}fields") is not None:
xmlFields = xmlRoot.find("{" + namespace_project + "}fields")
for xmlField in xmlFields.findall("{" + namespace_project + "}field"):
field = Field.loadFromXML(xmlField, namespace_project, version)
if field is not None:
symbol.addField(field)
return symbol
return None
def loadSymbol(xmlRoot, namespace_project, namespace_common, version, project, poolOfMessages):
if version == "0.1":
idSymbol = str(xmlRoot.get("id"))
symbol = Symbol(idSymbol, "", project)
# we parse the messages
if xmlRoot.find("{" + namespace_project + "}messages-ref") is not None:
xmlMessages = xmlRoot.find("{" + namespace_project + "}messages-ref")
for xmlMessage in xmlMessages.findall("{" + namespace_common + "}message-ref"):
id = xmlMessage.get("id")
message = poolOfMessages.getMessageByID(id)
if message is not None:
message.setSymbol(symbol)
symbol.addMessage(message)
# we parse the fields
if xmlRoot.find("{" + namespace_project + "}field") is not None:
xmlField = xmlRoot.find("{" + namespace_project + "}field")
field = Field.loadFromXML(xmlField, namespace_project, version, symbol)
if field is not None:
symbol.setField(field)
return symbol
return None
def clusterByTokenization(self, symbols):
self.ServerInference()
################################### Cluster messages according to their tokens
ll = len(self.symbols) - 1
i_equ = 0
while (ll > 0):
currentPattern = self.symbols[i_equ].getMessages()[0].getPattern(
)[1]
for j in range(ll):
jnext = len(self.symbols) - j - 1
cond = False
for message in self.symbols[jnext].getMessages():
if currentPattern == message.getPattern()[1]:
# score = sum([p1 == p2 for p1, p2 in zip(currentPattern, message.getPattern()[1])])
score2 = self.computeSimilarities(
currentPattern,
message.getPattern()[1])
# if score >= min(len(currentPattern), len(message.getPattern()[1])):
minilength = min(
len(message.getData()),
len(self.symbols[i_equ].getMessages()
[0].getData()))
if score2 * 2.0 / minilength >= 0.40:
cond = True
if cond:
break
if (cond):
currentDst = self.symbols[i_equ].getPattern()[0]
otherDst = self.symbols[jnext].getPattern()[0]
if not self.server or (currentDst == otherDst) or (
currentDst != self.server
and otherDst != self.server):
self.mergeEffectiveRowCol(i_equ, jnext)
# self.log.debug("Merge the equal column/line {0} with the column/line {1}".format(str(i_equ), str(j + 1)))
i_equ -= 1
break
ll -= 1
i_equ += 1
################################## Align messages
alignment = NeedlemanAndWunsch(self.unitSize, self.cb_status)
tmpSymbols = []
for symbol in self.symbols:
# alignment.alignSymbols([symbol], self.project)
# symbol.getFields()[0].setFormat(Format.STRING)
# tmpSymbols.extend(alignment.getLastResult())
try:
# print "l"
al = self.computeAlignment(symbol)
symbol.getField().setAlignment(al)
alignment.buildRegexFromAlignment(symbol, al,
self.defaultFormat)
# for (p, fields) in zip(symbol.getPattern()[1], symbol.getFields()):
# field.setFormat(p.getFormat())
except:
logging.warn(
"Partitionnement error: too much fields ( > 100) for the symbol '"
+ symbol.getName() + "' len=" +
str(len(symbol.getExtendedFields())) + "len " +
str(len(symbol.getPattern()[1])))
symbol.getField().removeLocalFields()
field = Field("Field 0", "(.{, })", symbol)
symbol.addLocalField(field)
# Use the default protocol type for representation
field.setFormat(self.defaultFormat)
alignment.alignSymbols(self.symbols, self.project)
self.symbols = alignment.getLastResult()
def getFields(self):
self.fields = [Field.createDefaultField()]
return self.fields
def getFields(self):
self.fields = [Field.createDefaultField()]
return self.fields