コード例 #1
0
    def openTextProcessing(self):
        """
        open the GUI CurrentDownload and start the text processing. Also it connects the logger
        to the edit line. This Gui informs the user about which CourseData will be textually
        processed.
        """
        self.cleanMainWindow()
        self.MainWindow = loadUi(self.pathCurrentDownload)
        self.MainWindow.showMaximized()

        # Activate logger to textbox
        logTextBox = QTextEditLogger(self.MainWindow)
        # You can format what is printed to text box
        logTextBox.setFormatter(
            logging.Formatter('%(asctime)s - %(levelname)s - %(message)s'))
        logging.getLogger().addHandler(logTextBox)
        # You can control the logging level
        logging.getLogger().setLevel(logging.INFO)

        # disable button until the text process is finished
        self.MainWindow.but_done.setEnabled(False)
        self.MainWindow.plainTextEdit.setReadOnly(True)
        self.MainWindow.show()

        # create the text mining logic
        textProcessing = TextProcessing(database=self.database,
                                        path=self.pathTempFolder,
                                        textSummarizer=self.textSummarizer)
        # get the CourseData from the database, which do not have a text abstraction and not raised an error in a past process
        fileTypes = ['pdf', 'docx', 'pptx', 'html']
        dataWithoutTextMining = self.database.getCourseDataWithoutTextMiningForParsing(
            fileTypes)

        # i is the position of the course data in the list
        i = 1
        countDataWithoutTextMining = len(dataWithoutTextMining)
        self.MainWindow.plainTextEdit.appendPlainText(
            "Es sind insgesamt %d Dateien textuell aufzuarbeiten!".replace(
                "%d", str(countDataWithoutTextMining)))
        # start for each data the text mining process
        for data in dataWithoutTextMining:
            self.MainWindow.plainTextEdit.appendPlainText(
                "Beginn mit der %d1. Datei von %d2!".replace(
                    "%d1", str(i)).replace("%d2",
                                           str(countDataWithoutTextMining)))
            textProcessing.textProcessing(data)
            self.MainWindow.plainTextEdit.appendPlainText(
                "Fertig mit der %d1. Datei von %d2!".replace(
                    "%d1", str(i)).replace("%d2",
                                           str(countDataWithoutTextMining)))
            i += 1
            QtWidgets.QApplication.processEvents()

        # enable the button for finish this
        self.MainWindow.but_done.setEnabled(True)

        # deactivate the logger
        logTextBox.deactivate()
        # go back to main page
        self.MainWindow.but_done.clicked.connect(self.goBackMainPage)
コード例 #2
0
    def test_textProcessor_ExceptionCheckLanguage(self, mock_apply_async, mock_pool):
        """
        Test if the text processing process will work right, if a ExceptionTextAbstraction pop up
        Procedure:
            1. Set the mocking results
            2. call the function
            ---------
            Verification:
            3. check if file handler was called right
            4. check if multiprocessing was called right
            5. check if text cleaner was called right
            6. check if check language was called right
            7. check if summarize text was called right
            8. check if frequency_words was called right
            9. check if update course data text fields was called right
            10. check if the course data object is right
        """

        def my_side_effect(text):
            raise ExceptionCheckLanguage("ExceptionCheckLanguage")

        dummyCourseData = CourseData(id=1, name="test.pdf", dataType="pdf")

        mock_database = Mock(spec=DatabaseManager)
        mock_fileToText = Mock(spec=FileToText)
        mock_textAbstraction = Mock(spec=TextAbstraction)
        mock_fileHandler = Mock(spec=FileHandler)
        mock_checkLanguage = Mock(spec=CheckLanguage)
        mock_textCleaner = Mock(spec=TextCleaner)
        textProcesser = TextProcessing(database=mock_database, fileHandler=mock_fileHandler, path="C:/",
                                       textSummarizer=mock_textAbstraction, fileToText=mock_fileToText,
                                       checkLanguage=mock_checkLanguage, textCleaner=mock_textCleaner)

        # mocking results
        resultFullText = "Das ist ein Volltext"
        resultCleanText = "Das ist ein sauberer Text"

        # set mocking results
        mock_get = MagicMock()
        mock_get_function = MagicMock(return_value=resultFullText)
        mock_get.get = mock_get_function
        mock_apply_async.apply_async.return_value = mock_get
        mock_pool.return_value = mock_apply_async
        mock_fileHandler.saveFile.return_value = "C:/test.pdf"
        mock_fileToText.fileToText.return_value = resultFullText
        mock_textCleaner.cleaningText.return_value = resultCleanText
        mock_checkLanguage.checkLanguage.side_effect = my_side_effect

        textProcesser.textProcessing(dummyCourseData)

        # check if FileHandler was called right
        mock_fileHandler.saveFile.assert_called_with(dummyCourseData, "C:/")
        self.assertTrue(mock_fileHandler.saveFile.called)

        # check if multiprocessing was called right
        called_args = mock_apply_async.apply_async.call_args_list[0][1]
        self.assertTrue(called_args['args'] == ("C:/test.pdf", "pdf"))
        self.assertTrue(called_args['func'] == mock_fileToText.fileToText)
        self.assertTrue(mock_apply_async.apply_async.called)

        # check if text cleaner was called right
        mock_textCleaner.cleaningText.assert_called_with(resultFullText)
        self.assertTrue(mock_textCleaner.cleaningText.called)

        # check if check language was called right
        mock_checkLanguage.checkLanguage.assert_called_with(resultCleanText)
        self.assertTrue(mock_checkLanguage.checkLanguage.called)

        # check if summarize text was not called
        self.assertFalse(mock_textAbstraction.summarize_Text.called)

        # check if frequency_words was not called
        self.assertFalse(mock_textAbstraction.frequency_Words.called)

        # check if update course data text fields was called right
        mock_database.UpdateCourseDataTextFields.assert_called_with(dummyCourseData)
        self.assertTrue(mock_database.UpdateCourseDataTextFields.called)

        # check if the course data object is right
        self.assertEqual(dummyCourseData.fullText, resultFullText)
        self.assertEqual(dummyCourseData.abstract, None)
        self.assertEqual(dummyCourseData.abstractWordFrequency, None)
        self.assertEqual(dummyCourseData.error, "ExceptionCheckLanguage")