예제 #1
0
 def alignSymbol(self, symbol, doInternalSlick, defaultFormat):
     messages = symbol.getMessages()
     if messages == None or len(messages) == 0:
         logging.debug("The symbol '" + symbol.getName() + "' is empty. No alignment needed")
         symbol.cleanFields()
         field = Field.createDefaultField()
         # Use the default protocol type for representation
         field.setFormat(defaultFormat)
         symbol.addField(field)
     else:
         symbol.cleanFields()
         # We execute the alignment
         (alignment, score) = self.align(doInternalSlick, messages)
         symbol.setAlignment(alignment)
         logging.debug("Alignment : {0}".format(alignment))
         # We update the regex based on the results
         try:
             self.buildRegexFromAlignment(symbol, alignment, defaultFormat)
         except NetzobException:
             logging.warn("Partitionnement error: too much fields (>100) for the symbol '" + symbol.getName() + "'")
             symbol.cleanFields()
             field = Field.createDefaultField()
             # Use the default protocol type for representation
             field.setFormat(defaultFormat)
             symbol.addField(field)
    def displayPopupToCreateLayer_cb(self, event):
        # If fields header are selected, we get it
        selectedFields = self.vocabularyController.view.selectedMessageTable.treeViewHeaderGroup.getSelectedFields()
        if selectedFields is None or len(selectedFields) == 0:
            # Either, we only consider the current field
            selectedFields = [self.field]
        # We retrieve the first and last fields selected
        firstField = selectedFields[0]
        lastField = selectedFields[0]
        for field in selectedFields:
            if field.getIndex() < firstField.getIndex():
                firstField = field
            if field.getIndex() > lastField.getIndex():
                lastField = field
        # Update selected fields to the entire range
        selectedFields = []
        for field in self.getSymbol().getExtendedFields():
            if field.getIndex() >= firstField.getIndex() and field.getIndex() <= lastField.getIndex():
                selectedFields.append(field)
        # Verify that selected field range does not overlap existing layers (i.e. if the selected fields have the same parent)
        parent = selectedFields[0].getParentField()
        for selectedField in selectedFields:
            if parent != selectedField.getParentField():
                NetzobErrorMessage(_("Selected field range overlaps existing layer."))
                return
        # Retrieve layer's name
        dialog = Gtk.Dialog(title=_("Layer creation"), flags=0, buttons=(Gtk.STOCK_CANCEL, Gtk.ResponseType.CANCEL, Gtk.STOCK_OK, Gtk.ResponseType.OK))
        dialog.set_size_request(200, 50)
        label = Gtk.Label("Name:")
        entry = Gtk.Entry()
        dialog.vbox.pack_start(label, True, True, 0)
        dialog.vbox.pack_start(entry, True, True, 0)
        dialog.show_all()
        result = dialog.run()
        if (result == Gtk.ResponseType.OK):
            name = entry.get_text()
            dialog.destroy()
            if name == "":
                return
        else:
            dialog.destroy()
            return
        # Create a new layer
        regex = ""
        for selectedField in selectedFields:
            regex += "(?:" + selectedField.getRegex()[1:]
        fieldLayer = Field(str(name), "(" + regex + ")", self.getSymbol())
        index_newField = 999999
        parentField = None
        for selectedField in selectedFields:
            parentField = selectedField.getParentField()
            if parentField.getLocalFields().index(selectedField) < index_newField:
                index_newField = parentField.getLocalFields().index(selectedField)  # Retrieve the lowest index of the new fields
            fieldLayer.addField(selectedField)
            parentField.getLocalFields().remove(selectedField)
        parentField.getLocalFields().insert(index_newField, fieldLayer)
#        self.getSymbol().getField().addField(fieldLayer, index_newField)
        self.vocabularyController.view.updateLeftPanel()
예제 #3
0
    def clusterByTokenization(self, symbols):
        self.ServerInference()
    ################################### Cluster messages according to their tokens
        ll = len(self.symbols) - 1
        i_equ = 0
        while(ll > 0):
            currentPattern = self.symbols[i_equ].getMessages()[0].getPattern()[1]
            for j in range(ll):
                jnext = len(self.symbols) - j - 1
                cond = False
                for message in self.symbols[jnext].getMessages():
                    if currentPattern == message.getPattern()[1]:
#                        score = sum([p1 == p2 for p1, p2 in zip(currentPattern, message.getPattern()[1])])
                        score2 = self.computeSimilarities(currentPattern, message.getPattern()[1])
#                        if score >= min(len(currentPattern), len(message.getPattern()[1])):
                        minilength = min(len(message.getData()), len(self.symbols[i_equ].getMessages()[0].getData()))
                        if score2 * 2.0 / minilength >= 0.40:
                            cond = True
                        if cond:
                            break

                if(cond):
                    currentDst = self.symbols[i_equ].getPattern()[0]
                    otherDst = self.symbols[jnext].getPattern()[0]
                    if not self.server or (currentDst == otherDst) or (currentDst != self.server and otherDst != self.server):
                        self.mergeEffectiveRowCol(i_equ, jnext)
#                        self.log.debug("Merge the equal column/line {0} with the column/line {1}".format(str(i_equ), str(j + 1)))
                        i_equ -= 1
                        break
            ll -= 1
            i_equ += 1

    ################################## Align messages
        alignment = NeedlemanAndWunsch(self.unitSize, self.cb_status)
        tmpSymbols = []
        for symbol in self.symbols:

#            alignment.alignSymbols([symbol], self.project)
#            symbol.getFields()[0].setFormat(Format.STRING)
#            tmpSymbols.extend(alignment.getLastResult())
            try:
#                print "l"
                al = self.computeAlignment(symbol)
                symbol.getField().setAlignment(al)
                alignment.buildRegexFromAlignment(symbol, al, self.defaultFormat)

#                for (p, fields) in zip(symbol.getPattern()[1], symbol.getFields()):
#                    field.setFormat(p.getFormat())
            except:
                logging.warn("Partitionnement error: too much fields ( > 100) for the symbol '" + symbol.getName() + "' len=" + str(len(symbol.getExtendedFields())) + "len " + str(len(symbol.getPattern()[1])))
                symbol.getField().removeLocalFields()
                field = Field("Field 0", "(.{, })", symbol)
                symbol.addField(field)
                # Use the default protocol type for representation
                field.setFormat(self.defaultFormat)

        alignment.alignSymbols(self.symbols, self.project)
        self.symbols = alignment.getLastResult()
예제 #4
0
파일: Symbol.py 프로젝트: windli4367/netzob
    def loadSymbol(xmlRoot, namespace_project, namespace_common, version,
                   project, poolOfMessages):
        if version == "0.1":
            idSymbol = str(xmlRoot.get("id"))
            symbol = Symbol(idSymbol, "", project)

            # we parse the messages
            if xmlRoot.find("{" + namespace_project +
                            "}messages-ref") is not None:
                xmlMessages = xmlRoot.find("{" + namespace_project +
                                           "}messages-ref")
                for xmlMessage in xmlMessages.findall("{" + namespace_common +
                                                      "}message-ref"):
                    id = xmlMessage.get("id")
                    message = poolOfMessages.getMessageByID(id)
                    if message is not None:
                        message.setSymbol(symbol)
                        symbol.addMessage(message)

            # we parse the fields
            if xmlRoot.find("{" + namespace_project + "}field") is not None:
                xmlField = xmlRoot.find("{" + namespace_project + "}field")
                field = Field.loadFromXML(xmlField, namespace_project, version,
                                          symbol)
                if field is not None:
                    symbol.setField(field)

            return symbol
        return None
예제 #5
0
 def __init__(self, ID, name, project):
     AbstractSymbol.__init__(self, Symbol.TYPE)
     self.id = ID
     self.project = project
     self.messages = []
     self.field = Field.createDefaultField(self)
     self.field.setName(name)
     self.project = project
     self.pattern = None
예제 #6
0
파일: Symbol.py 프로젝트: windli4367/netzob
 def __init__(self, ID, name, project):
     AbstractSymbol.__init__(self, Symbol.TYPE)
     self.id = ID
     self.project = project
     self.messages = []
     self.field = Field.createDefaultField(self)
     self.field.setName(name)
     self.project = project
     self.pattern = None
예제 #7
0
    def buildRegexFromAlignment(self, symbol, align, defaultFormat):
        # Build regex from alignment
        i = 0
        start = 0
        regex = []
        found = False
        for i in range(len(align)):
            if (align[i] == "-"):
                if (found == False):
                    start = i
                    found = True
            else:
                if (found == True):
                    found = False
                    nbTiret = i - start
                    regex.append("(.{," + str(nbTiret) + "})")
                    regex.append(align[i])
                else:
                    if len(regex) == 0:
                        regex.append(align[i])
                    else:
                        regex[-1] += align[i]
        if (found == True):
            nbTiret = i - start + 1
            regex.append("(.{," + str(nbTiret) + "})")

        iField = 0
        symbol.cleanFields()
        logging.debug("REGEX " + str(regex))
        for regexElt in regex:
            field = Field("Field " + str(iField), iField, regexElt)
            # Use the default protocol type for representation
            field.setFormat(defaultFormat)
            symbol.addField(field)
            iField = iField + 1
#        if len(symbol.getFields()) >= 50:
#            raise NetzobException("This Python version only supports 100 named groups in regex")
        # We look for useless fields
        doLoop = True
        # We loop until we don't pop any field
        while doLoop == True:
            doLoop = False
            for field in symbol.getFields():
                # We try to see if this field produces only empty values when applied on messages
                if not field.isStatic():
                    cells = symbol.getCellsByField(field)
                    cells = "".join(cells)
                    if cells == "":
                        symbol.getFields().pop(field.getIndex())  # We remove this useless field
                        # Adapt index of the following fields
                        for fieldNext in symbol.getFields():
                            if fieldNext.getIndex() > field.getIndex():
                                fieldNext.setIndex(fieldNext.getIndex() - 1)
                        # Concatenate the surrounding fields (because they should be static fields)
                        if symbol.concatFields(field.getIndex() - 1) == 1:
                            doLoop = True
                            break
예제 #8
0
    def loadSymbol(xmlRoot, namespace_project, namespace_common, version,
                   project, poolOfMessages):
        if version == "0.1":
            nameSymbol = xmlRoot.get("name")
            idSymbol = xmlRoot.get("id")
            symbol = TrashSymbol(idSymbol, project)

            # Interpretation attributes
            if xmlRoot.find("{" + namespace_project + "}format") is not None:
                symbol_format = xmlRoot.find("{" + namespace_project +
                                             "}format").text
                symbol.setFormat(symbol_format)

            if xmlRoot.find("{" + namespace_project + "}unitsize") is not None:
                symbol_unitsize = xmlRoot.find("{" + namespace_project +
                                               "}unitsize").text
                symbol.setUnitSize(symbol_unitsize)

            if xmlRoot.find("{" + namespace_project + "}sign") is not None:
                symbol_sign = xmlRoot.find("{" + namespace_project +
                                           "}sign").text
                symbol.setSign(symbol_sign)

            if xmlRoot.find("{" + namespace_project +
                            "}endianess") is not None:
                symbol_endianess = xmlRoot.find("{" + namespace_project +
                                                "}endianess").text
                symbol.setEndianess(symbol_endianess)

            # we parse the messages
            if xmlRoot.find("{" + namespace_project +
                            "}messages-ref") is not None:
                xmlMessages = xmlRoot.find("{" + namespace_project +
                                           "}messages-ref")
                for xmlMessage in xmlMessages.findall("{" + namespace_common +
                                                      "}message-ref"):
                    id = xmlMessage.get("id")
                    message = poolOfMessages.getMessageByID(id)
                    if message is not None:
                        message.setSymbol(symbol)
                        symbol.addMessage(message)

            # we parse the fields
            if xmlRoot.find("{" + namespace_project + "}fields") is not None:
                xmlFields = xmlRoot.find("{" + namespace_project + "}fields")
                for xmlField in xmlFields.findall("{" + namespace_project +
                                                  "}field"):
                    field = Field.loadFromXML(xmlField, namespace_project,
                                              version)
                    if field is not None:
                        symbol.addField(field)

            return symbol
        return None
예제 #9
0
    def importTraceInProject(self, trace, project):
        percent = 0.0
        inc = 1.0 / len(trace.getMessages())
        # We create a symbol dedicated for the trace
        symbol = Symbol(uuid.uuid4(), trace.getType(), project)
        for message in trace.getMessages():
            percent += inc
            symbol.addMessage(message)
            gobject.idle_add(self.progressbarAlignment.set_fraction, float(percent))

        # We create a default field for the symbol
        symbol.addField(Field.createDefaultField())
        # and register the symbol in the vocabulary of the project
        project.getVocabulary().addSymbol(symbol)
        project.saveConfigFile(self.workspace)
예제 #10
0
    def __init__(self, id=None):
        AbstractSymbol.__init__(self, "TrashSymbol")
        self.id = id
        if self.id is None:
            id = str(uuid.uuid4())
        self.name = "TrashSymbol"
        self.messages = []
        self.encodingFilters = []
        self.visualizationFilters = []

        # Interpretation attributes
        self.format = Format.HEX
        self.unitSize = UnitSize.NONE
        self.sign = Sign.UNSIGNED
        self.endianess = Endianess.BIG

        # Clean the symbol
        self.fields = [Field.createDefaultField()]
예제 #11
0
    def __init__(self, id=None):
        AbstractSymbol.__init__(self, "TrashSymbol")
        self.id = id
        if self.id is None:
            id = str(uuid.uuid4())
        self.name = "TrashSymbol"
        self.messages = []
        self.encodingFilters = []
        self.visualizationFilters = []

        # Interpretation attributes
        self.format = Format.HEX
        self.unitSize = UnitSize.NONE
        self.sign = Sign.UNSIGNED
        self.endianess = Endianess.BIG

        # Clean the symbol
        self.fields = [Field.createDefaultField()]
예제 #12
0
    def loadSymbol(xmlRoot, namespace_project, namespace_common, version, project, poolOfMessages):
        if version == "0.1":
            nameSymbol = xmlRoot.get("name")
            idSymbol = xmlRoot.get("id")
            symbol = TrashSymbol(idSymbol, project)

            # Interpretation attributes
            if xmlRoot.find("{" + namespace_project + "}format") is not None:
                symbol_format = xmlRoot.find("{" + namespace_project + "}format").text
                symbol.setFormat(symbol_format)

            if xmlRoot.find("{" + namespace_project + "}unitsize") is not None:
                symbol_unitsize = xmlRoot.find("{" + namespace_project + "}unitsize").text
                symbol.setUnitSize(symbol_unitsize)

            if xmlRoot.find("{" + namespace_project + "}sign") is not None:
                symbol_sign = xmlRoot.find("{" + namespace_project + "}sign").text
                symbol.setSign(symbol_sign)

            if xmlRoot.find("{" + namespace_project + "}endianess") is not None:
                symbol_endianess = xmlRoot.find("{" + namespace_project + "}endianess").text
                symbol.setEndianess(symbol_endianess)

            # we parse the messages
            if xmlRoot.find("{" + namespace_project + "}messages-ref") is not None:
                xmlMessages = xmlRoot.find("{" + namespace_project + "}messages-ref")
                for xmlMessage in xmlMessages.findall("{" + namespace_common + "}message-ref"):
                    id = xmlMessage.get("id")
                    message = poolOfMessages.getMessageByID(id)
                    if message is not None:
                        message.setSymbol(symbol)
                        symbol.addMessage(message)

            # we parse the fields
            if xmlRoot.find("{" + namespace_project + "}fields") is not None:
                xmlFields = xmlRoot.find("{" + namespace_project + "}fields")
                for xmlField in xmlFields.findall("{" + namespace_project + "}field"):
                    field = Field.loadFromXML(xmlField, namespace_project, version)
                    if field is not None:
                        symbol.addField(field)

            return symbol
        return None
예제 #13
0
    def loadSymbol(xmlRoot, namespace_project, namespace_common, version, project, poolOfMessages):
        if version == "0.1":
            idSymbol = str(xmlRoot.get("id"))
            symbol = Symbol(idSymbol, "", project)

            # we parse the messages
            if xmlRoot.find("{" + namespace_project + "}messages-ref") is not None:
                xmlMessages = xmlRoot.find("{" + namespace_project + "}messages-ref")
                for xmlMessage in xmlMessages.findall("{" + namespace_common + "}message-ref"):
                    id = xmlMessage.get("id")
                    message = poolOfMessages.getMessageByID(id)
                    if message is not None:
                        message.setSymbol(symbol)
                        symbol.addMessage(message)

            # we parse the fields
            if xmlRoot.find("{" + namespace_project + "}field") is not None:
                xmlField = xmlRoot.find("{" + namespace_project + "}field")
                field = Field.loadFromXML(xmlField, namespace_project, version, symbol)
                if field is not None:
                    symbol.setField(field)

            return symbol
        return None
예제 #14
0
    def clusterByTokenization(self, symbols):
        self.ServerInference()
        ################################### Cluster messages according to their tokens
        ll = len(self.symbols) - 1
        i_equ = 0
        while (ll > 0):
            currentPattern = self.symbols[i_equ].getMessages()[0].getPattern(
            )[1]
            for j in range(ll):
                jnext = len(self.symbols) - j - 1
                cond = False
                for message in self.symbols[jnext].getMessages():
                    if currentPattern == message.getPattern()[1]:
                        #                        score = sum([p1 == p2 for p1, p2 in zip(currentPattern, message.getPattern()[1])])
                        score2 = self.computeSimilarities(
                            currentPattern,
                            message.getPattern()[1])
                        #                        if score >= min(len(currentPattern), len(message.getPattern()[1])):
                        minilength = min(
                            len(message.getData()),
                            len(self.symbols[i_equ].getMessages()
                                [0].getData()))
                        if score2 * 2.0 / minilength >= 0.40:
                            cond = True
                        if cond:
                            break

                if (cond):
                    currentDst = self.symbols[i_equ].getPattern()[0]
                    otherDst = self.symbols[jnext].getPattern()[0]
                    if not self.server or (currentDst == otherDst) or (
                            currentDst != self.server
                            and otherDst != self.server):
                        self.mergeEffectiveRowCol(i_equ, jnext)
                        #                        self.log.debug("Merge the equal column/line {0} with the column/line {1}".format(str(i_equ), str(j + 1)))
                        i_equ -= 1
                        break
            ll -= 1
            i_equ += 1

    ################################## Align messages
        alignment = NeedlemanAndWunsch(self.unitSize, self.cb_status)
        tmpSymbols = []
        for symbol in self.symbols:

            #            alignment.alignSymbols([symbol], self.project)
            #            symbol.getFields()[0].setFormat(Format.STRING)
            #            tmpSymbols.extend(alignment.getLastResult())
            try:
                #                print "l"
                al = self.computeAlignment(symbol)
                symbol.getField().setAlignment(al)
                alignment.buildRegexFromAlignment(symbol, al,
                                                  self.defaultFormat)

#                for (p, fields) in zip(symbol.getPattern()[1], symbol.getFields()):
#                    field.setFormat(p.getFormat())
            except:
                logging.warn(
                    "Partitionnement error: too much fields ( > 100) for the symbol '"
                    + symbol.getName() + "' len=" +
                    str(len(symbol.getExtendedFields())) + "len " +
                    str(len(symbol.getPattern()[1])))
                symbol.getField().removeLocalFields()
                field = Field("Field 0", "(.{, })", symbol)
                symbol.addLocalField(field)
                # Use the default protocol type for representation
                field.setFormat(self.defaultFormat)

        alignment.alignSymbols(self.symbols, self.project)
        self.symbols = alignment.getLastResult()
예제 #15
0
 def getFields(self):
     self.fields = [Field.createDefaultField()]
     return self.fields
예제 #16
0
 def getFields(self):
     self.fields = [Field.createDefaultField()]
     return self.fields