Esempio n. 1
0
    def concatSymbolButton_clicked_cb(self, toolButton):
        if self.getCurrentProject() is None:
            NetzobErrorMessage(_("No project selected."))
            return
        # retrieve the checked symbols
        symbols = self.view.getCheckedSymbolList()

        # Create a new symbol
        newSymbol = Symbol(str(uuid.uuid4()), "Merged",
                           self.getCurrentProject())

        # fetch all their messages
        for sym in symbols:
            newSymbol.addMessages(sym.getMessages())

        #delete all selected symbols
        self.view.emptyMessageTableDisplayingSymbols(symbols)
        for sym in symbols:
            self.getCurrentProject().getVocabulary().removeSymbol(sym)

        #add the concatenate symbol
        self.getCurrentProject().getVocabulary().addSymbol(newSymbol)

        #refresh view
        self.view.updateLeftPanel()
Esempio n. 2
0
    def __init__(self, project, symbols, explodeSymbols, nbIteration, minEquivalence, doInternalSlick, defaultFormat, unitSize, cb_status=None, scores={}):
        self.project = project
        self.nbIteration = nbIteration
        self.minEquivalence = minEquivalence
        self.doInternalSlick = doInternalSlick
        self.cb_status = cb_status
        self.defaultFormat = defaultFormat
        self.unitSize = unitSize
        self.log = logging.getLogger('netzob.Inference.Vocabulary.UPGMA.py')
        self.scores = scores
        self.path = []
        if explodeSymbols == False:
            self.symbols = symbols

        else:
            # Create a symbol for each message
            self.symbols = []
            i_symbol = 1
            for symbol in symbols:
                for m in symbol.getMessages():
                    tmpSymbol = Symbol(str(uuid.uuid4()), "Symbol " + str(i_symbol), project)
                    tmpSymbol.addMessage(m)
                    self.symbols.append(tmpSymbol)
                    i_symbol += 1

        self.log.debug("A number of {0} already aligned symbols will be clustered.".format(str(len(symbols))))
Esempio n. 3
0
 def __init__(self,
              project,
              symbols,
              explodeSymbols,
              doInternalSlick,
              defaultFormat,
              unitSize,
              cb_status=None):
     self.symbols = symbols
     self.project = project
     self.doInternalSlick = doInternalSlick
     self.cb_status = cb_status
     self.defaultFormat = defaultFormat
     self.unitSize = unitSize
     self.log = logging.getLogger(
         'netzob.Inference.Vocabulary.AsciiAlign.py')
     self.server = ""
     if explodeSymbols is False:
         self.symbols = symbols
     else:
         #Create a symbol for each message and reset to constant the tokens
         self.symbols = []
         i_symbol = 1
         for symbol in symbols:
             for m in symbol.getMessages():
                 for t in m.getPattern()[1]:
                     t.setType("constant")
                 tmpSymbol = Symbol(
                     str(uuid.uuid4()), "Symbol " + str(i_symbol), project)
                 tmpSymbol.setPattern(m.getPattern())
                 tmpSymbol.addMessage(m)
                 self.symbols.append(tmpSymbol)
                 i_symbol += 1
Esempio n. 4
0
    def __init__(self, project, symbols, unitSize, cb_status=None, scores={}):
        self.project = project
        self.unitSize = unitSize
        self.cb_status = cb_status
        self.scores = scores

        # Then we retrieve all the parameters of the CLUSTERING / ALIGNMENT
        self.defaultFormat = self.project.getConfiguration().getVocabularyInferenceParameter(ProjectConfiguration.VOCABULARY_GLOBAL_FORMAT)
        self.nbIteration = self.project.getConfiguration().getVocabularyInferenceParameter(ProjectConfiguration.VOCABULARY_NB_ITERATION)
        self.minEquivalence = self.project.getConfiguration().getVocabularyInferenceParameter(ProjectConfiguration.VOCABULARY_EQUIVALENCE_THRESHOLD)
        self.doInternalSlick = self.project.getConfiguration().getVocabularyInferenceParameter(ProjectConfiguration.VOCABULARY_DO_INTERNAL_SLICK)

        self.log = logging.getLogger('netzob.Inference.Vocabulary.UPGMA.py')
        self.path = []
        self.flagStop = False
        self.currentAlignment = None

        # Create a symbol for each message
        self.symbols = []
        i_symbol = 1
        for symbol in symbols:
            for m in symbol.getMessages():
                tmpSymbol = Symbol(str(uuid.uuid4()), "Symbol " + str(i_symbol), project)
                tmpSymbol.addMessage(m)
                self.symbols.append(tmpSymbol)
                i_symbol += 1
        self.log.debug("A number of {0} already aligned symbols will be clustered.".format(str(len(symbols))))
Esempio n. 5
0
    def __init__(self, project, symbols, unitSize, cb_status=None, scores={}):
        self.project = project
        self.unitSize = unitSize
        self.cb_status = cb_status
        self.scores = scores

        # Then we retrieve all the parameters of the CLUSTERING / ALIGNMENT
        self.defaultFormat = self.project.getConfiguration().getVocabularyInferenceParameter(ProjectConfiguration.VOCABULARY_GLOBAL_FORMAT)
        self.nbIteration = self.project.getConfiguration().getVocabularyInferenceParameter(ProjectConfiguration.VOCABULARY_NB_ITERATION)
        self.minEquivalence = self.project.getConfiguration().getVocabularyInferenceParameter(ProjectConfiguration.VOCABULARY_EQUIVALENCE_THRESHOLD)
        self.doInternalSlick = self.project.getConfiguration().getVocabularyInferenceParameter(ProjectConfiguration.VOCABULARY_DO_INTERNAL_SLICK)

        self.log = logging.getLogger('netzob.Inference.Vocabulary.UPGMA.py')
        self.path = []
        self.flagStop = False
        self.currentAlignment = None

        # Create a symbol for each message
        self.symbols = []
        i_symbol = 1
        for symbol in symbols:
            for m in symbol.getMessages():
                tmpSymbol = Symbol(str(uuid.uuid4()), "Symbol " + str(i_symbol), project)
                tmpSymbol.addMessage(m)
                self.symbols.append(tmpSymbol)
                i_symbol += 1
        self.log.debug("A number of {0} already aligned symbols will be clustered.".format(str(len(symbols))))
Esempio n. 6
0
    def test_semanticAlignment_bug1(self):
        """test_semanticAlignment_bug1:
        A bug on the semantic alignment has been identified which prevent
        the computation of a valid regex. This test verifies the bug is not comming back.
        @date 18/04/2013
        """

        firstname1 = "antoine"
        email1 = "*****@*****.**"

        firstname2 = "luc"
        email2 = "*****@*****.**"

        msg1 = RawMessage(uuid.uuid4(), None, TypeConvertor.stringToNetzobRaw("6" + firstname1 + "GAHFSHQS" + email1))
        msg2 = RawMessage(uuid.uuid4(), None, TypeConvertor.stringToNetzobRaw("3" + firstname2 + "CVSDHISD" + email2))

        project = Project(uuid.uuid4(), "Experiment", datetime.now(), "")
        nwEngine = NeedlemanAndWunsch(8, project, False, None)
        symbol = Symbol(uuid.uuid4(), "Test", project)

        symbol.addMessages([msg1, msg2])
        msg1.addSemanticTag("firstname", 2, 2 + len(firstname1) * 2)
        msg1.addSemanticTag("email", 2 + len(firstname1) * 2 + 16, 2 + len(firstname1) * 2 + 16 + len(email1) * 2)

        msg2.addSemanticTag("firstname", 2, 2 + len(firstname2) * 2)
        msg2.addSemanticTag("email", 2 + len(firstname2) * 2 + 16, 2 + len(firstname2) * 2 + 16 + len(email2) * 2)

        nwEngine.alignField(symbol.getField())
        symbol.getField().setFormat(Format.STRING)

        print("Computed Regex : {0}".format(symbol.getRegex()))
        print(symbol.getCells(True))

        computedFields = symbol.getExtendedFields()
        self.assertTrue(len(computedFields) > 1, "Only one field has been computed which tells us something went wrong.")
    def traceImportInProjectAction_activate_cb(self, button):
        """This callback is called by the 'ImportInProjectAction'. It
        allows to import a trace or a session in the currently opened
        project.

        FIXME: the code doesn't allow to remove duplicated messages.
        """

        project = self.mainController.getCurrentProject()

        # We are only able to import a trace if a project is open.
        if project is None:
            self.view.showErrorMessage(
                _("No project is currently opened"),
                _("A project has to be opened before importing a trace."))
            return

        response = self.view.showImportInProjectDialog()

        if response == 1:
            self.log.info(
                "Asked to import trace '{0}' (id={1}) in the current project".
                format(self.currentTrace.name, self.currentTrace.id))

            symbolName = self.view.importInProjectNameEntry.get_text()

            if self.traceSelectionIsATrace:
                messages = self.currentTrace.getMessages()
                sessions = self.currentTrace.getSessions()
            else:
                model, path = self.view.traceTreeview.get_selection(
                ).get_selected_rows()

                # UI prevents from importing multiple sessions at a
                # time.
                assert len(path) == 1

                session = self.currentTrace.getSession(model[path.pop()][0])
                sessions = [session]
                messages = session.getMessages()

            # We add a new Symbol
            symbol = Symbol(str(uuid.uuid4()), symbolName, project)

            # We register each message in the vocabulary of the project
            for message in messages:
                project.getVocabulary().addMessage(message)
                symbol.addMessage(message)

            project.getVocabulary().addSymbol(symbol)

            for session in sessions:
                project.getVocabulary().addSession(session)
Esempio n. 8
0
    def traceImportInProjectAction_activate_cb(self, button):
        """This callback is called by the 'ImportInProjectAction'. It
        allows to import a trace or a session in the currently opened
        project.

        FIXME: the code doesn't allow to remove duplicated messages.
        """

        project = self.mainController.getCurrentProject()

        # We are only able to import a trace if a project is open.
        if project is None:
            self.view.showErrorMessage(_("No project is currently opened"),
                                       _("A project has to be opened before importing a trace."))
            return

        response = self.view.showImportInProjectDialog()

        if response == 1:
            self.log.info("Asked to import trace '{1}' (id={2}) in the current project".format(self.currentTrace.name,
                                                                                               self.currentTrace.id))

            symbolName = self.view.importInProjectNameEntry.get_text()

            if self.traceSelectionIsATrace:
                messages = self.currentTrace.getMessages()
                sessions = self.currentTrace.getSessions()
            else:
                model, path = self.view.traceTreeview.get_selection().get_selected_rows()

                # UI prevents from importing multiple sessions at a
                # time.
                assert len(path) == 1

                session = self.currentTrace.getSession(model[path.pop()][0])
                sessions = [session]
                messages = session.getMessages()

            # We add a new Symbol
            symbol = Symbol(str(uuid.uuid4()), symbolName, project)

            # We register each message in the vocabulary of the project
            for message in messages:
                project.getVocabulary().addMessage(message)
                symbol.addMessage(message)

            project.getVocabulary().addSymbol(symbol)

            for session in sessions:
                project.getVocabulary().addSession(session)
Esempio n. 9
0
    def mergeEffectiveRowCol(self, i_maximum, j_maximum):
        # Extract symbols i and j
        if i_maximum > j_maximum:
            symbol1 = self.symbols.pop(i_maximum)
            symbol2 = self.symbols.pop(j_maximum)
        else:
            symbol1 = self.symbols.pop(j_maximum)
            symbol2 = self.symbols.pop(i_maximum)

        # Merge the symbols i and j
        messages = []
        messages.extend(symbol1.getMessages())
        messages.extend(symbol2.getMessages())

        newSymbol = Symbol(str(uuid.uuid4()), symbol1.getName(), self.project)
        newSymbol.setPattern(self.mergePattern(symbol1.getPattern(), symbol2.getPattern()))
#        self.log.debug("Patterns to merge: {0} with {1}: Give Result {2}".format(symbol1.getPatternString(), symbol2.getPatternString(), newSymbol.getPatternString()))
        newSymbol.setPattern(
            self.mergePattern(symbol1.getPattern(), symbol2.getPattern()))
        #        self.log.debug("Patterns to merge: {0} with {1}: Give Result {2}".format(symbol1.getPatternString(), symbol2.getPatternString(), newSymbol.getPatternString()))
        for message in messages:
            newSymbol.addMessage(message)

        # Append th new symbol to the "symbols" structure
        self.symbols.append(newSymbol)
Esempio n. 10
0
    def test_deserialisationGroups(self):
        print("start")

        symbols = []
        nbSymbol = random.randint(2, 50)

        for iSymbol in range(0, nbSymbol):
            # We create 6 messages of 2 group
            originalSymbol = Symbol(str(uuid.uuid4()), "TestSymbol", None)
            # group1
            message1 = RawMessage(str(uuid.uuid4()), str(time.time()), TypeConvertor.stringToNetzobRaw("bonjour " + self.generateRandomString(20, 30) + " comment vas-tu ?"))
            message2 = RawMessage(str(uuid.uuid4()), str(time.time()), TypeConvertor.stringToNetzobRaw("bonjour " + self.generateRandomString(20, 30) + " comment vas-tu ?"))
            message3 = RawMessage(str(uuid.uuid4()), str(time.time()), TypeConvertor.stringToNetzobRaw("bonjour " + self.generateRandomString(20, 30) + " comment vas-tu ?"))
            # group2
            message4 = RawMessage(str(uuid.uuid4()), str(time.time()), TypeConvertor.stringToNetzobRaw("salut à toi " + self.generateRandomString(10, 15) + " what's up ?"))
            message5 = RawMessage(str(uuid.uuid4()), str(time.time()), TypeConvertor.stringToNetzobRaw("salut à toi " + self.generateRandomString(10, 15) + " what's up ?"))
            message6 = RawMessage(str(uuid.uuid4()), str(time.time()), TypeConvertor.stringToNetzobRaw("salut à toi " + self.generateRandomString(10, 15) + " what's up ?"))

            originalSymbol.addMessage(message1)
            originalSymbol.addMessage(message2)
            originalSymbol.addMessage(message3)
            originalSymbol.addMessage(message4)
            originalSymbol.addMessage(message5)
            originalSymbol.addMessage(message6)
            symbols.append(originalSymbol)

        # Start the clustering
        clusteringSolution = UPGMA(None, [originalSymbol], True, 100, 90, True)
        result = clusteringSolution.deserializeGroups(symbols)
        self.assertEqual(result, len(symbols))
Esempio n. 11
0
    def importTraceInProject(self, trace, project):
        percent = 0.0
        inc = 1.0 / len(trace.getMessages())
        # We create a symbol dedicated for the trace
        symbol = Symbol(uuid.uuid4(), trace.getType(), project)
        for message in trace.getMessages():
            percent += inc
            symbol.addMessage(message)
            gobject.idle_add(self.progressbarAlignment.set_fraction, float(percent))

        # We create a default field for the symbol
        symbol.addField(Field.createDefaultField())
        # and register the symbol in the vocabulary of the project
        project.getVocabulary().addSymbol(symbol)
        project.saveConfigFile(self.workspace)
Esempio n. 12
0
    def createSymbolButton_clicked_cb(self, toolButton):
        if self.getCurrentProject() is None:
            NetzobErrorMessage(_("No project selected."))
            return

        builder2 = Gtk.Builder()
        builder2.add_from_file(
            os.path.join(ResourcesConfiguration.getStaticResources(), "ui",
                         "dialogbox.glade"))
        dialog = builder2.get_object("createsymbol")
        dialog.set_transient_for(self.netzob.view.mainWindow)

        # Disable apply button if no text
        applybutton = builder2.get_object("button1")
        entry = builder2.get_object("entry1")
        entry.connect("changed", self.entry_disableButtonIfEmpty_cb,
                      applybutton)

        result = dialog.run()

        if (result == 0):
            newSymbolName = entry.get_text()
            newSymbolId = str(uuid.uuid4())
            self.log.debug(
                "A new symbol will be created with the given name: {0}".format(
                    newSymbolName))
            currentProject = self.netzob.getCurrentProject()
            newSymbol = Symbol(newSymbolId, newSymbolName, currentProject)
            currentProject.getVocabulary().addSymbol(newSymbol)
            self.view.updateLeftPanel()
            dialog.destroy()
        if (result == 1):
            dialog.destroy()
Esempio n. 13
0
    def loadVocabulary(xmlRoot, namespace_project, namespace_common, version,
                       project):
        vocabulary = Vocabulary()

        if version == "0.1":
            # Messages
            for xmlMessage in xmlRoot.findall("{" + namespace_project +
                                              "}messages/{" +
                                              namespace_common + "}message"):
                message = AbstractMessageFactory.loadFromXML(
                    xmlMessage, namespace_common, version)
                if message is not None:
                    vocabulary.addMessage(message)
            # Symbols
            for xmlSymbol in xmlRoot.findall("{" + namespace_project +
                                             "}symbols/{" + namespace_project +
                                             "}symbol"):
                symbol = Symbol.loadSymbol(xmlSymbol, namespace_project,
                                           namespace_common, version, project,
                                           vocabulary)
                if symbol is not None:
                    vocabulary.addSymbol(symbol)
            # Sessions
            for xmlSession in xmlRoot.findall("{" + namespace_project +
                                              "}sessions/{" +
                                              namespace_common + "}session"):
                session = Session.loadFromXML(xmlSession, namespace_project,
                                              namespace_common, version,
                                              vocabulary)
                if session is not None:
                    vocabulary.addSession(session)
        return vocabulary
Esempio n. 14
0
    def mergeEffectiveRowCol(self, i_maximum, j_maximum):
        """Merge the symbols i and j in the "symbols" structure
        @param i_maximum: id of the first symbol to merge
        @param j_maximum: id of the second symbol to merge
        @return the newly created symbol result of the merged process"""
        # Extract symbols i and j
        if i_maximum > j_maximum:
            symbol1 = self.symbols.pop(i_maximum)
            symbol2 = self.symbols.pop(j_maximum)
        else:
            symbol1 = self.symbols.pop(j_maximum)
            symbol2 = self.symbols.pop(i_maximum)

        # Merge the symbols i and j
        messages = []
        messages.extend(symbol1.getMessages())
        messages.extend(symbol2.getMessages())

        newSymbol = Symbol(str(uuid.uuid4()), symbol1.getName(), self.project)
        newSymbol.setMinEqu(self.minEquivalence)
        for message in messages:
            newSymbol.addMessage(message)

        # Append th new symbol to the "symbols" structure
        self.symbols.append(newSymbol)

        return newSymbol.getID()
Esempio n. 15
0
    def test_randomAlignmentsWithTwoCenteredMessages(self):
        workspace = self.getWorkspace()
        currentProject = workspace.getProjects()[0]

        doInternalSlick = currentProject.getConfiguration().getVocabularyInferenceParameter(ProjectConfiguration.VOCABULARY_DO_INTERNAL_SLICK)
        defaultFormat = currentProject.getConfiguration().getVocabularyInferenceParameter(ProjectConfiguration.VOCABULARY_GLOBAL_FORMAT)
        defaultUnitSize = 8

        # We generate 1000 random couples of data and try to align them
        # Objectives: just test if it executes
        nb_data = 1000
        nb_failed = 0
        nb_success = 0
        for i_test in range(0, nb_data):

            common_pattern = self.generateRandomString(30, 40)
            # Generate the content of two messages
            data1 = TypeConvertor.stringToNetzobRaw(self.generateRandomString(5, 100) + common_pattern + self.generateRandomString(5, 100))
            data2 = TypeConvertor.stringToNetzobRaw(self.generateRandomString(5, 100) + common_pattern + self.generateRandomString(5, 100))
            # Create the messages
            message1 = RawMessage(str(uuid.uuid4()), str(time.time()), data1)
            message2 = RawMessage(str(uuid.uuid4()), str(time.time()), data2)
            # Create the symbol
            symbol = Symbol(str(uuid.uuid4()), "test_randomAlignments#" + str(i_test), currentProject)
            symbol.addMessage(message1)
            symbol.addMessage(message2)
            field = symbol.getField()

            # Starts the alignment process
            alignmentProcess = NeedlemanAndWunsch(defaultUnitSize, currentProject, False, self.emptyAlignmentCB)
            alignmentProcess.alignField(field)

            if not TypeConvertor.stringToNetzobRaw(common_pattern[:]) in field.getAlignment():
                if self.debug is True:
                    print "Message 1: " + str(data1)
                    print "Message 2: " + str(data2)
                    print "Common pattern: " + TypeConvertor.stringToNetzobRaw(common_pattern)
                    print "Alignment: " + field.getAlignment()
                nb_failed += 1
            else:
                nb_success += 1
        if nb_failed > 0:
            print "A number of " + str(nb_failed) + "/" + str(nb_data) + " alignment failed !"
        self.assertEqual(0, nb_failed)
        self.assertEqual(nb_success, nb_data)
Esempio n. 16
0
    def test_deserialisationGroups(self):
        print "start"

        symbols = []
        nbSymbol = random.randint(2, 50)

        for iSymbol in range(0, nbSymbol):
            # We create 6 messages of 2 group
            originalSymbol = Symbol(str(uuid.uuid4()), "TestSymbol", None)
            # group1
            message1 = RawMessage(str(uuid.uuid4()), str(time.time()), TypeConvertor.stringToNetzobRaw("bonjour " + self.generateRandomString(20, 30) + " comment vas-tu ?"))
            message2 = RawMessage(str(uuid.uuid4()), str(time.time()), TypeConvertor.stringToNetzobRaw("bonjour " + self.generateRandomString(20, 30) + " comment vas-tu ?"))
            message3 = RawMessage(str(uuid.uuid4()), str(time.time()), TypeConvertor.stringToNetzobRaw("bonjour " + self.generateRandomString(20, 30) + " comment vas-tu ?"))
            # group2
            message4 = RawMessage(str(uuid.uuid4()), str(time.time()), TypeConvertor.stringToNetzobRaw("salut à toi " + self.generateRandomString(10, 15) + " what's up ?"))
            message5 = RawMessage(str(uuid.uuid4()), str(time.time()), TypeConvertor.stringToNetzobRaw("salut à toi " + self.generateRandomString(10, 15) + " what's up ?"))
            message6 = RawMessage(str(uuid.uuid4()), str(time.time()), TypeConvertor.stringToNetzobRaw("salut à toi " + self.generateRandomString(10, 15) + " what's up ?"))

            originalSymbol.addMessage(message1)
            originalSymbol.addMessage(message2)
            originalSymbol.addMessage(message3)
            originalSymbol.addMessage(message4)
            originalSymbol.addMessage(message5)
            originalSymbol.addMessage(message6)
            symbols.append(originalSymbol)

        # Start the clustering
        clusteringSolution = UPGMA(None, [originalSymbol], True, 100, 90, True)
        result = clusteringSolution.deserializeGroups(symbols)
        self.assertEqual(result, len(symbols))
Esempio n. 17
0
    def mergeEffectiveRowCol(self, i_maximum, j_maximum):
        # Extract symbols i and j
        if i_maximum > j_maximum:
            symbol1 = self.symbols.pop(i_maximum)
            symbol2 = self.symbols.pop(j_maximum)
        else:
            symbol1 = self.symbols.pop(j_maximum)
            symbol2 = self.symbols.pop(i_maximum)

        # Merge the symbols i and j
        messages = []
        messages.extend(symbol1.getMessages())
        messages.extend(symbol2.getMessages())

        newSymbol = Symbol(str(uuid.uuid4()), symbol1.getName(), self.project, pattern=self.mergePattern(symbol1.getPattern(), symbol2.getPattern()))
#        self.log.debug("Patterns to merge: {0} with {1}: Give Result {2}".format(symbol1.getPatternString(), symbol2.getPatternString(), newSymbol.getPatternString()))
        for message in messages:
            newSymbol.addMessage(message)

        # Append th new symbol to the "symbols" structure
        self.symbols.append(newSymbol)
Esempio n. 18
0
 def __init__(self,
              project,
              symbols,
              explodeSymbols,
              doInternalSlick,
              defaultFormat,
              unitSize,
              cb_status=None):
     self.symbols = symbols
     self.project = project
     self.doInternalSlick = doInternalSlick
     self.cb_status = cb_status
     self.defaultFormat = defaultFormat
     self.unitSize = unitSize
     self.log = logging.getLogger(
         'netzob.Inference.Vocabulary.AsciiAlign.py')
     self.server = ""
     if explodeSymbols is False:
         self.symbols = symbols
     else:
         #Create a symbol for each message and reset to constant the tokens
         self.symbols = []
         i_symbol = 1
         for symbol in symbols:
             for m in symbol.getMessages():
                 for t in m.getPattern()[1]:
                     t.setType("constant")
                 tmpSymbol = Symbol(str(uuid.uuid4()),
                                    "Symbol " + str(i_symbol), project)
                 tmpSymbol.setPattern(m.getPattern())
                 tmpSymbol.addMessage(m)
                 self.symbols.append(tmpSymbol)
                 i_symbol += 1
Esempio n. 19
0
    def mergeEffectiveRowCol(self, i_maximum, j_maximum):
        """Merge the symbols i and j in the "symbols" structure
        @param i_maximum: id of the first symbol to merge
        @param j_maximum: id of the second symbol to merge
        @return the newly created symbol result of the merged process"""
        # Extract symbols i and j
        if i_maximum > j_maximum:
            symbol1 = self.symbols.pop(i_maximum)
            symbol2 = self.symbols.pop(j_maximum)
        else:
            symbol1 = self.symbols.pop(j_maximum)
            symbol2 = self.symbols.pop(i_maximum)

        # Merge the symbols i and j
        messages = []
        messages.extend(symbol1.getMessages())
        messages.extend(symbol2.getMessages())

        newSymbol = Symbol(str(uuid.uuid4()), symbol1.getName(), self.project)
        newSymbol.setMinEqu(self.minEquivalence)
        for message in messages:
            newSymbol.addMessage(message)

        # Append th new symbol to the "symbols" structure
        self.symbols.append(newSymbol)

        return newSymbol.getID()
Esempio n. 20
0
    def mergeEffectiveRowCol(self, i_maximum, j_maximum):
        # Extract symbols i and j
        if i_maximum > j_maximum:
            symbol1 = self.symbols.pop(i_maximum)
            symbol2 = self.symbols.pop(j_maximum)
        else:
            symbol1 = self.symbols.pop(j_maximum)
            symbol2 = self.symbols.pop(i_maximum)

        # Merge the symbols i and j
        messages = []
        messages.extend(symbol1.getMessages())
        messages.extend(symbol2.getMessages())

        newSymbol = Symbol(str(uuid.uuid4()), symbol1.getName(), self.project, minEqu=self.minEquivalence)
        for message in messages:
            newSymbol.addMessage(message)

        # Append th new symbol to the "symbols" structure
        self.symbols.append(newSymbol)

        return newSymbol.getID()
Esempio n. 21
0
    def test_semanticAlignment_bug1(self):
        """test_semanticAlignment_bug1:
        A bug on the semantic alignment has been identified which prevent
        the computation of a valid regex. This test verifies the bug is not comming back.
        @date 18/04/2013
        """

        firstname1 = "antoine"
        email1 = "*****@*****.**"

        firstname2 = "luc"
        email2 = "*****@*****.**"

        msg1 = RawMessage(uuid.uuid4(), None, TypeConvertor.stringToNetzobRaw("6" + firstname1 + "GAHFSHQS" + email1))
        msg2 = RawMessage(uuid.uuid4(), None, TypeConvertor.stringToNetzobRaw("3" + firstname2 + "CVSDHISD" + email2))

        project = Project(uuid.uuid4(), "Experiment", datetime.now(), "")
        nwEngine = NeedlemanAndWunsch(8, project, False, None)
        symbol = Symbol(uuid.uuid4(), "Test", project)

        symbol.addMessages([msg1, msg2])
        msg1.addSemanticTag("firstname", 2, 2 + len(firstname1) * 2)
        msg1.addSemanticTag("email", 2 + len(firstname1) * 2 + 16, 2 + len(firstname1) * 2 + 16 + len(email1) * 2)

        msg2.addSemanticTag("firstname", 2, 2 + len(firstname2) * 2)
        msg2.addSemanticTag("email", 2 + len(firstname2) * 2 + 16, 2 + len(firstname2) * 2 + 16 + len(email2) * 2)

        nwEngine.alignField(symbol.getField())
        symbol.getField().setFormat(Format.STRING)

        print "Computed Regex : {0}".format(symbol.getRegex())
        print "======="
        print symbol.getCells(True)

        computedFields = symbol.getExtendedFields()
        self.assertTrue(len(computedFields) > 1, "Only one field has been computed which tells us something went wrong.")
Esempio n. 22
0
    def concatSymbolButton_clicked_cb(self, toolButton):
        if self.getCurrentProject() is None:
            NetzobErrorMessage(_("No project selected."))
            return
        # retrieve the checked symbols
        symbols = self.view.getCheckedSymbolList()

        # Create a new symbol
        newSymbol = Symbol(str(uuid.uuid4()), "Merged", self.getCurrentProject())

        # fetch all their messages
        for sym in symbols:
            newSymbol.addMessages(sym.getMessages())

        #delete all selected symbols
        self.view.emptyMessageTableDisplayingSymbols(symbols)
        for sym in symbols:
            self.getCurrentProject().getVocabulary().removeSymbol(sym)

        #add the concatenate symbol
        self.getCurrentProject().getVocabulary().addSymbol(newSymbol)

        #refresh view
        self.view.updateLeftPanel()
Esempio n. 23
0
    def loadVocabulary(xmlRoot, namespace_project, namespace_common, version, project):
        vocabulary = Vocabulary()

        if version == "0.1":
            # Messages
            for xmlMessage in xmlRoot.findall("{" + namespace_project + "}messages/{" + namespace_common + "}message"):
                message = AbstractMessageFactory.loadFromXML(xmlMessage, namespace_common, version)
                if message is not None:
                    vocabulary.addMessage(message)
            # Symbols
            for xmlSymbol in xmlRoot.findall("{" + namespace_project + "}symbols/{" + namespace_project + "}symbol"):
                symbol = Symbol.loadSymbol(xmlSymbol, namespace_project, namespace_common, version, project, vocabulary)
                if symbol is not None:
                    vocabulary.addSymbol(symbol)
            # Sessions
            for xmlSession in xmlRoot.findall("{" + namespace_project + "}sessions/{" + namespace_common + "}session"):
                session = Session.loadFromXML(xmlSession, namespace_project, namespace_common, version, vocabulary)
                if session is not None:
                    vocabulary.addSession(session)
        return vocabulary
    def importButton_clicked_cb(self, widget):
        """Callback executed when the user wants to import messages"""
        if self.currentProject is None:
            self.log.error("No project is open")
            return

        # retrieve symbol name
        symbolName = self._view.nameOfCreatedSymbolEntry.get_text()
        if symbolName is None or len(symbolName) < 1:
            self.displayErrorMessage(_("Specify the name of new symbol"))
            return

        found = False
        for symbol in self.currentProject.getVocabulary().getSymbols():
            if symbol.getName() == symbolName:
                found = True
                break

        if found:
            self.displayErrorMessage(_("The provided symbol name already exists."))
            return

        # Should we consider meta datas of excluded messages
        if self._view.removeDuplicatedMessagesCheckButton.get_active() and self._view.keepPropertiesOfDuplicatedMessagesCheckButton.get_active():
            # Retrieve the 'excluded' messages and retrieve their properties
            for message in self.excludedMessages:
                # search for an included message to register properties
                eq_message = None
                for importedMessage in self.importedMessages:
                    if importedMessage.getStringData() == message.getStringData():
                        eq_message = importedMessage
                        break
                if eq_message is not None:
                    for property in message.getProperties():
                        eq_message.addExtraProperty(property)

        # We create a session with each message
        session = Session(str(uuid.uuid4()), "Session 1", "")
        for message in self.importedMessages:
            session.addMessage(message)
        # We register the session in the vocabulary of the project
        self.currentProject.getVocabulary().addSession(session)

        # We register each message in the vocabulary of the project
        for message in self.importedMessages:
            self.currentProject.getVocabulary().addMessage(message)
            message.setSession(session)

        # We create a default symbol dedicated for this
        symbol = Symbol(str(uuid.uuid4()), symbolName, self.currentProject)
        for message in self.importedMessages:
            symbol.addMessage(message)
        # We register the symbol in the vocabulary of the project
        self.currentProject.getVocabulary().addSymbol(symbol)

        # Add the environmental dependencies to the project
#        if fetchEnv:
#            project.getConfiguration().setVocabularyInferenceParameter(ProjectConfiguration.VOCABULARY_ENVIRONMENTAL_DEPENDENCIES,
#                                                                       self.envDeps.getEnvData())
        # Computes current date
        date = datetime.now()
        description = "No description (yet not implemented)"

        # We also save the session and the messages in the workspace
        trace = ImportedTrace(str(uuid.uuid4()), date, self.importType, description, self.currentProject.getName())
        trace.addSession(session)
        for message in self.importedMessages:
            trace.addMessage(message)

        self.currentWorkspace.addImportedTrace(trace)

        # Now we save the workspace
        self.currentWorkspace.saveConfigFile()

        self._view.destroy()

        if self.finish_cb is not None:
            GObject.idle_add(self.finish_cb)
Esempio n. 25
0
    def test_executingClusteringWithOrphanReduction(self):

        # We create 6 messages of 2 group

        # group1
        originalSymbol1 = Symbol(str(uuid.uuid4()), "TestSymbol", None)
        message1 = RawMessage(str(uuid.uuid4()), str(time.time()), TypeConvertor.stringToNetzobRaw("bonjour " + self.generateRandomString(200, 1000)))
        originalSymbol1.addMessage(message1)

        originalSymbol2 = Symbol(str(uuid.uuid4()), "TestSymbol2", None)
        message2 = RawMessage(str(uuid.uuid4()), str(time.time()), TypeConvertor.stringToNetzobRaw("bonjour " + self.generateRandomString(200, 1000)))
        originalSymbol2.addMessage(message2)

        originalSymbol3 = Symbol(str(uuid.uuid4()), "TestSymbol3", None)
        message3 = RawMessage(str(uuid.uuid4()), str(time.time()), TypeConvertor.stringToNetzobRaw("bonjour " + self.generateRandomString(200, 1000)))
        originalSymbol3.addMessage(message3)

        # group2
        originalSymbol4 = Symbol(str(uuid.uuid4()), "TestSymbol4", None)
        message4 = RawMessage(str(uuid.uuid4()), str(time.time()), TypeConvertor.stringToNetzobRaw("salut " + self.generateRandomString(200, 1000)))
        originalSymbol4.addMessage(message4)

        originalSymbol5 = Symbol(str(uuid.uuid4()), "TestSymbol5", None)
        message5 = RawMessage(str(uuid.uuid4()), str(time.time()), TypeConvertor.stringToNetzobRaw("salut " + self.generateRandomString(200, 1000)))
        originalSymbol5.addMessage(message5)

        originalSymbol6 = Symbol(str(uuid.uuid4()), "TestSymbol6", None)
        message6 = RawMessage(str(uuid.uuid4()), str(time.time()), TypeConvertor.stringToNetzobRaw("salut " + self.generateRandomString(200, 1000)))
        originalSymbol6.addMessage(message6)

        symbols = [originalSymbol1, originalSymbol2, originalSymbol3, originalSymbol4, originalSymbol5, originalSymbol6]

        # Start the clustering
        clusteringSolution = UPGMA(None, symbols, True, 100, 80, True, Format.ASCII)
        resultBeforeOrphan = clusteringSolution.executeClustering()
        resultAfterOrphan = clusteringSolution.executeOrphanReduction()

        if (len(resultAfterOrphan) < len(resultBeforeOrphan)):
            print "Before Orphan Reduction: "
            for symbol in resultBeforeOrphan:
                print "Symbol: " + str(symbol.getName())
                for m in symbol.getMessages():
                    print " + " + str(m.getStringData())

            print "After Orphan Reduction: "
            for symbol in resultAfterOrphan:
                print "Symbol: " + str(symbol.getName())
                for m in symbol.getMessages():
                    print " + " + str(m.getStringData())

        self.assertGreaterEqual(len(resultBeforeOrphan), len(resultAfterOrphan))
Esempio n. 26
0
    def test_executingClustering(self):

        # We create 6 messages of 2 group

        # group1
        originalSymbol1 = Symbol(str(uuid.uuid4()), "TestSymbol", None)
        message1 = RawMessage(str(uuid.uuid4()), str(time.time()), TypeConvertor.stringToNetzobRaw("bonjour " + self.generateRandomString(20, 30) + " comment vas-tu ?"))
        originalSymbol1.addMessage(message1)

        originalSymbol2 = Symbol(str(uuid.uuid4()), "TestSymbol2", None)
        message2 = RawMessage(str(uuid.uuid4()), str(time.time()), TypeConvertor.stringToNetzobRaw("bonjour " + self.generateRandomString(20, 30) + " comment vas-tu ?"))
        originalSymbol2.addMessage(message2)

        originalSymbol3 = Symbol(str(uuid.uuid4()), "TestSymbol3", None)
        message3 = RawMessage(str(uuid.uuid4()), str(time.time()), TypeConvertor.stringToNetzobRaw("bonjour " + self.generateRandomString(20, 30) + " comment vas-tu ?"))
        originalSymbol3.addMessage(message3)

        # group2
        originalSymbol4 = Symbol(str(uuid.uuid4()), "TestSymbol4", None)
        message4 = RawMessage(str(uuid.uuid4()), str(time.time()), TypeConvertor.stringToNetzobRaw("salut à toi " + self.generateRandomString(10, 15) + " what's up ?"))
        originalSymbol4.addMessage(message4)

        originalSymbol5 = Symbol(str(uuid.uuid4()), "TestSymbol5", None)
        message5 = RawMessage(str(uuid.uuid4()), str(time.time()), TypeConvertor.stringToNetzobRaw("salut à toi " + self.generateRandomString(10, 15) + " what's up ?"))
        originalSymbol5.addMessage(message5)

        originalSymbol6 = Symbol(str(uuid.uuid4()), "TestSymbol6", None)
        message6 = RawMessage(str(uuid.uuid4()), str(time.time()), TypeConvertor.stringToNetzobRaw("salut à toi " + self.generateRandomString(10, 15) + " what's up ?"))
        originalSymbol6.addMessage(message6)

        symbols = [originalSymbol1, originalSymbol2, originalSymbol3, originalSymbol4, originalSymbol5, originalSymbol6]

        # Start the clustering
        clusteringSolution = UPGMA(None, symbols, True, 100, 90, True, Format.ASCII)
        result = clusteringSolution.executeClustering()

        for symbol in result:
            print "Symbol: " + str(symbol.getName())
            for m in symbol.getMessages():
                print " + " + str(m.getStringData())
Esempio n. 27
0
    def importButton_clicked_cb(self, widget):
        """Callback executed when the user wants to import messages"""
        if self.currentProject is None:
            self.log.error("No project is open")
            return

        # retrieve symbol name
        symbolName = self._view.nameOfCreatedSymbolEntry.get_text()
        if symbolName is None or len(symbolName) < 1:
            self.displayErrorMessage(_("Specify the name of new symbol"))
            return

        found = False
        for symbol in self.currentProject.getVocabulary().getSymbols():
            if symbol.getName() == symbolName:
                found = True
                break

        if found:
            self.displayErrorMessage(
                _("The provided symbol name already exists."))
            return

        # Should we consider meta datas of excluded messages
        if self._view.removeDuplicatedMessagesCheckButton.get_active(
        ) and self._view.keepPropertiesOfDuplicatedMessagesCheckButton.get_active(
        ):
            # Retrieve the 'excluded' messages and retrieve their properties
            for message in self.excludedMessages:
                # search for an included message to register properties
                eq_message = None
                for importedMessage in self.importedMessages:
                    if importedMessage.getStringData(
                    ) == message.getStringData():
                        eq_message = importedMessage
                        break
                if eq_message is not None:
                    for property in message.getProperties():
                        eq_message.addExtraProperty(property)

        # We create a session with each message
        session = Session(str(uuid.uuid4()), "Session 1", "")
        for message in self.importedMessages:
            session.addMessage(message)
        # We register the session in the vocabulary of the project
        self.currentProject.getVocabulary().addSession(session)

        # We register each message in the vocabulary of the project
        for message in self.importedMessages:
            self.currentProject.getVocabulary().addMessage(message)
            message.setSession(session)

        # We create a default symbol dedicated for this
        symbol = Symbol(str(uuid.uuid4()), symbolName, self.currentProject)
        for message in self.importedMessages:
            symbol.addMessage(message)
        # We register the symbol in the vocabulary of the project
        self.currentProject.getVocabulary().addSymbol(symbol)

        # Add the environmental dependencies to the project
        #        if fetchEnv:
        #            project.getConfiguration().setVocabularyInferenceParameter(ProjectConfiguration.VOCABULARY_ENVIRONMENTAL_DEPENDENCIES,
        #                                                                       self.envDeps.getEnvData())
        # Computes current date
        date = datetime.now()
        description = "No description (yet not implemented)"

        # We also save the session and the messages in the workspace
        trace = ImportedTrace(str(uuid.uuid4()), date, self.importType,
                              description, self.currentProject.getName())
        trace.addSession(session)
        for message in self.importedMessages:
            trace.addMessage(message)

        self.currentWorkspace.addImportedTrace(trace)

        # Now we save the workspace
        self.currentWorkspace.saveConfigFile()

        self._view.destroy()

        if self.finish_cb is not None:
            GObject.idle_add(self.finish_cb)
Esempio n. 28
0
    def test_semanticAlignment_simple(self):
        """test_semanticAlignment_simple:
        Test that messages with embedded semantic are efficiently aligned.
        Format : <random 10 bytes><random username><random 5 ASCII><random email>

        Optimal Needleman & Wunsch Parameters :
        // Cost definitions for the alignment
        static const short int MATCH = 5;
        static const short int SEMANTIC_MATCH = 30;
        static const short int MISMATCH = -5;
        static const short int GAP = 0;
        static const short int BLEN = 10;
        // Consts for the definition of a mask
        static const unsigned char END = 2;
        static const unsigned char DIFFERENT = 1;
        static const unsigned char EQUAL = 0;
        """
        project = Project(uuid.uuid4(), "Experiment", datetime.now(), "")
        symbol = Symbol(uuid.uuid4(), "Test", project)

        nbMessage = 500
        usernames = []
        emails = []
        for iMessage in range(0, nbMessage):
            str_username = self.generateRandomString(4, 10)
            username = TypeConvertor.stringToNetzobRaw(str_username)
            usernames.append(str_username)

            email_prefix = self.generateRandomString(4, 10)
            email_domain = self.generateRandomString(4, 10)
            email_extension = self.generateRandomString(2, 3)
            str_email = "{0}@{1}.{2}".format(email_prefix, email_domain, email_extension)
            emails.append(str_email)
            email = TypeConvertor.stringToNetzobRaw(str_email)
            random10Bytes = self.generateRandomBytes(10, 10)
            random5ASCII = TypeConvertor.stringToNetzobRaw(self.generateRandomString(5, 5))
            data = "{0}{1}{2}{3}".format(random10Bytes, username, random5ASCII, email)

            message = RawMessage(uuid.uuid4(), None, data)
            message.addSemanticTag("username", len(random10Bytes), len(random10Bytes) + len(username))
            message.addSemanticTag("email", len(random10Bytes) + len(username) + len(random5ASCII), len(random10Bytes) + len(username) + len(random5ASCII) + len(email))

            symbol.addMessage(message)

        nwEngine = NeedlemanAndWunsch(8, project, False, None)
        nwEngine.alignField(symbol.getField())

        symbol.getField().setFormat(Format.STRING)

        print("Number of computed fields : {0}".format(len(symbol.getExtendedFields())))
        self.assertEqual(4, len(symbol.getExtendedFields()))
        nbValidMessages = 0

        for message in symbol.getMessages():
            isValid = symbol.getField().isRegexValidForMessage(message)
            if isValid:
                nbValidMessages += 1
            self.assertTrue(isValid)

        print(symbol.getCells())

        print("Computed regex is valid for {0}/{1} messages.".format(nbValidMessages, len(symbol.getMessages())))
Esempio n. 29
0
    def test_semanticAlignment_simple(self):
        """test_semanticAlignment_simple:
        Test that messages with embedded semantic are efficiently aligned.
        Format : <random 10 bytes><random username><random 5 ASCII><random email>

        Optimal Needleman & Wunsch Parameters :
        // Cost definitions for the alignment
        static const short int MATCH = 5;
        static const short int SEMANTIC_MATCH = 30;
        static const short int MISMATCH = -5;
        static const short int GAP = 0;
        static const short int BLEN = 10;
        // Consts for the definition of a mask
        static const unsigned char END = 2;
        static const unsigned char DIFFERENT = 1;
        static const unsigned char EQUAL = 0;
        """
        project = Project(uuid.uuid4(), "Experiment", datetime.now(), "")
        symbol = Symbol(uuid.uuid4(), "Test", project)

        nbMessage = 500
        usernames = []
        emails = []
        for iMessage in range(0, nbMessage):
            str_username = self.generateRandomString(4, 10)
            username = TypeConvertor.stringToNetzobRaw(str_username)
            usernames.append(str_username)

            email_prefix = self.generateRandomString(4, 10)
            email_domain = self.generateRandomString(4, 10)
            email_extension = self.generateRandomString(2, 3)
            str_email = "{0}@{1}.{2}".format(email_prefix, email_domain, email_extension)
            emails.append(str_email)
            email = TypeConvertor.stringToNetzobRaw(str_email)
            random10Bytes = self.generateRandomBytes(10, 10)
            random5ASCII = TypeConvertor.stringToNetzobRaw(self.generateRandomString(5, 5))
            data = "{0}{1}{2}{3}".format(random10Bytes, username, random5ASCII, email)

            message = RawMessage(uuid.uuid4(), None, data)
            message.addSemanticTag("username", len(random10Bytes), len(random10Bytes) + len(username))
            message.addSemanticTag("email", len(random10Bytes) + len(username) + len(random5ASCII), len(random10Bytes) + len(username) + len(random5ASCII) + len(email))

            symbol.addMessage(message)

        nwEngine = NeedlemanAndWunsch(8, project, False, None)
        nwEngine.alignField(symbol.getField())

        symbol.getField().setFormat(Format.STRING)

        print("Number of computed fields : {0}".format(len(symbol.getExtendedFields())))
        self.assertEqual(4, len(symbol.getExtendedFields()))
        nbValidMessages = 0

        for message in symbol.getMessages():
            isValid = symbol.getField().isRegexValidForMessage(message)
            if isValid:
                nbValidMessages += 1
            self.assertTrue(isValid)

        print(symbol.getCells())

        print("Computed regex is valid for {0}/{1} messages.".format(nbValidMessages, len(symbol.getMessages())))